8cf2b6c1cb
When a tag-targeted run fails, automatically escalates: 1. Tag tests fail → re-run full light suite 2. If light fails → re-run heavy suite 3. If tag fails but light passes → result = PASS (tag-only issue) Escalation path shown in Telegram notification: "Escalation: tag:genlab (29 passed, 11 failed) → light PASSED (1431)" Stops at first passing level — no wasted time. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
441 lines
18 KiB
Bash
Executable File
441 lines
18 KiB
Bash
Executable File
#!/bin/bash
|
|
# TSHARPS CI — External Test Runner
|
|
# Runs tests, feature manifests, and package checks against TSHARPS worktrees.
|
|
# READ-ONLY — never writes, commits, or pushes to TSHARPS.
|
|
# Same checks for ALL branches — no branch-specific logic.
|
|
#
|
|
# Usage:
|
|
# ci-runner.sh <branch> <commit> <actor> [tag]
|
|
#
|
|
# Optional [tag] parameter runs only tests with that pytest marker:
|
|
# ci-runner.sh mfg abc123 claude genlab → runs only genlab-tagged tests
|
|
# ci-runner.sh mfg abc123 claude → runs full light suite (default)
|
|
# ci-runner.sh mfg abc123 claude heavy → runs full heavy suite
|
|
#
|
|
# NOTE: Feature tags (genlab, scheduling, auth, etc.) need updating from time
|
|
# to time as features change. See pytest.ini for the current list of markers.
|
|
|
|
set -o pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
CONFIG="$SCRIPT_DIR/ci-config.json"
|
|
BRANCH="$1"
|
|
COMMIT="$2"
|
|
ACTOR="${3:-unknown}"
|
|
TAG="${4:-}"
|
|
START_TIME=$(date +%s)
|
|
|
|
# Load config
|
|
TIMEOUT=$(python3 -c "import json; print(json.load(open('$CONFIG')).get('test_timeout', 120))")
|
|
WORKTREE=$(python3 -c "import json; b=json.load(open('$CONFIG'))['branches']; print(b.get('$BRANCH',{}).get('worktree',''))")
|
|
|
|
if [ -z "$WORKTREE" ] || [ ! -d "$WORKTREE" ]; then
|
|
echo "ERROR: Unknown branch '$BRANCH' or worktree not found"
|
|
bash "$SCRIPT_DIR/ci-notify.sh" "$BRANCH" "$COMMIT" "error" "Unknown branch or missing worktree" "" "0s" "$ACTOR"
|
|
exit 1
|
|
fi
|
|
|
|
# Prevent concurrent runs on the same branch
|
|
LOCKFILE="/tmp/tsharps-ci-${BRANCH}.lock"
|
|
exec 200>"$LOCKFILE"
|
|
if ! flock -n 200; then
|
|
echo "SKIP: CI already running for branch $BRANCH"
|
|
exit 0
|
|
fi
|
|
|
|
# Serialize all CI runs to avoid DB connection contention
|
|
GLOBAL_LOCK="/tmp/tsharps-ci-global.lock"
|
|
exec 201>"$GLOBAL_LOCK"
|
|
flock 201
|
|
|
|
echo "=== TSHARPS CI Runner ==="
|
|
echo "Branch: $BRANCH"
|
|
echo "Commit: $COMMIT"
|
|
echo "Worktree: $WORKTREE"
|
|
echo "Timeout: ${TIMEOUT}s"
|
|
echo "========================="
|
|
|
|
PASS_COUNT=0
|
|
FAIL_COUNT=0
|
|
SKIP_COUNT=0
|
|
ERROR_COUNT=0
|
|
FEATURES_RESULT=""
|
|
OVERALL="pass"
|
|
|
|
# ─── Step 1: Feature Manifest Check ───
|
|
echo ""
|
|
echo "--- Feature Manifest Check ---"
|
|
if [ -f "$WORKTREE/ops/verify-features.py" ] && [ -d "$WORKTREE/.features" ]; then
|
|
FEATURE_OUTPUT=$(cd "$WORKTREE" && python3 ops/verify-features.py --verbose 2>&1)
|
|
FEATURE_EXIT=$?
|
|
FEATURES_RESULT=$(echo "$FEATURE_OUTPUT" | grep "RESULT:" | tail -1)
|
|
if [ -z "$FEATURES_RESULT" ]; then
|
|
FEATURES_RESULT=$(echo "$FEATURE_OUTPUT" | tail -1)
|
|
fi
|
|
echo "$FEATURE_OUTPUT"
|
|
if [ $FEATURE_EXIT -ne 0 ]; then
|
|
OVERALL="fail"
|
|
echo "FEATURE CHECK FAILED"
|
|
fi
|
|
else
|
|
FEATURES_RESULT="No manifests found"
|
|
echo "No feature manifests — skipping"
|
|
fi
|
|
|
|
# ─── Step 2: Run Test Suite ───
|
|
echo ""
|
|
echo "--- Test Suite ---"
|
|
PYTHON="$WORKTREE/.venv/bin/python3"
|
|
if [ ! -f "$PYTHON" ]; then
|
|
PYTHON="python3"
|
|
fi
|
|
|
|
TIMEOUT_FLAG=""
|
|
if $PYTHON -c "import pytest_timeout" 2>/dev/null; then
|
|
TIMEOUT_FLAG="--timeout=$TIMEOUT"
|
|
fi
|
|
|
|
# Build pytest command — use tag filter if provided
|
|
TAG_FLAG=""
|
|
RUN_MODE="light"
|
|
if [ -n "$TAG" ]; then
|
|
if [ "$TAG" = "heavy" ]; then
|
|
TAG_FLAG=""
|
|
RUN_MODE="heavy"
|
|
export RUN_HEAVY_TESTS=1
|
|
else
|
|
TAG_FLAG="-m $TAG"
|
|
RUN_MODE="tag:$TAG"
|
|
fi
|
|
fi
|
|
|
|
TEST_OUTPUT=$($PYTHON -m pytest "$WORKTREE/backend/tests/" --tb=line -q $TIMEOUT_FLAG $TAG_FLAG 2>&1)
|
|
TEST_EXIT=$?
|
|
|
|
PASS_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ passed' | grep -oP '\d+' || echo 0)
|
|
FAIL_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ failed' | grep -oP '\d+' || echo 0)
|
|
SKIP_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ skipped' | grep -oP '\d+' || echo 0)
|
|
ERROR_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ error' | grep -oP '\d+' || echo 0)
|
|
DESELECTED_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ deselected' | grep -oP '\d+' || echo 0)
|
|
|
|
if [ "$TEST_EXIT" -ne 0 ] && [ "$PASS_COUNT" = "0" ] && [ "$FAIL_COUNT" = "0" ]; then
|
|
ERROR_COUNT=1
|
|
echo "WARNING: pytest exited with code $TEST_EXIT but reported no results — likely a collection error"
|
|
echo "$TEST_OUTPUT" | tail -5
|
|
fi
|
|
|
|
if [ "$PASS_COUNT" = "0" ] && [ "$FAIL_COUNT" = "0" ] && [ "$SKIP_COUNT" = "0" ]; then
|
|
OVERALL="fail"
|
|
echo "FAIL: 0 tests ran — something is wrong"
|
|
fi
|
|
|
|
echo "Tests: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped, $ERROR_COUNT errors"
|
|
|
|
if [ "$FAIL_COUNT" != "0" ] && [ "$FAIL_COUNT" != "" ]; then
|
|
OVERALL="fail"
|
|
echo "TESTS FAILED"
|
|
fi
|
|
if [ "$ERROR_COUNT" != "0" ] && [ "$ERROR_COUNT" != "" ]; then
|
|
OVERALL="fail"
|
|
echo "TEST COLLECTION ERRORS"
|
|
fi
|
|
|
|
# ─── Auto-Escalation ───
|
|
# If a tag-targeted run fails, escalate to full light suite.
|
|
# If light fails too, escalate to heavy. Captures results at each level.
|
|
ESCALATION_LOG=""
|
|
if [ "$OVERALL" = "fail" ] && [[ "$RUN_MODE" == tag:* ]]; then
|
|
ESCALATION_LOG="Escalated from ${RUN_MODE} (${PASS_COUNT} passed, ${FAIL_COUNT} failed)"
|
|
echo ""
|
|
echo "--- ESCALATING: ${RUN_MODE} failed → running full light suite ---"
|
|
LIGHT_OUTPUT=$($PYTHON -m pytest "$WORKTREE/backend/tests/" --tb=line -q $TIMEOUT_FLAG 2>&1)
|
|
LIGHT_PASS=$(echo "$LIGHT_OUTPUT" | grep -oP '\d+ passed' | grep -oP '\d+' || echo 0)
|
|
LIGHT_FAIL=$(echo "$LIGHT_OUTPUT" | grep -oP '\d+ failed' | grep -oP '\d+' || echo 0)
|
|
LIGHT_SKIP=$(echo "$LIGHT_OUTPUT" | grep -oP '\d+ skipped' | grep -oP '\d+' || echo 0)
|
|
LIGHT_DESELECTED=$(echo "$LIGHT_OUTPUT" | grep -oP '\d+ deselected' | grep -oP '\d+' || echo 0)
|
|
echo "Light suite: $LIGHT_PASS passed, $LIGHT_FAIL failed, $LIGHT_SKIP skipped"
|
|
|
|
if [ "$LIGHT_FAIL" = "0" ] || [ -z "$LIGHT_FAIL" ]; then
|
|
ESCALATION_LOG="$ESCALATION_LOG → light PASSED ($LIGHT_PASS passed)"
|
|
RUN_MODE="tag:${TAG}→light"
|
|
PASS_COUNT="$LIGHT_PASS"; FAIL_COUNT="0"; SKIP_COUNT="$LIGHT_SKIP"
|
|
DESELECTED_COUNT="$LIGHT_DESELECTED"; OVERALL="pass"
|
|
else
|
|
ESCALATION_LOG="$ESCALATION_LOG → light FAILED ($LIGHT_PASS passed, $LIGHT_FAIL failed)"
|
|
echo ""
|
|
echo "--- ESCALATING: light failed → running heavy suite ---"
|
|
export RUN_HEAVY_TESTS=1
|
|
HEAVY_OUTPUT=$($PYTHON -m pytest "$WORKTREE/backend/tests/" --tb=line -q $TIMEOUT_FLAG 2>&1)
|
|
HEAVY_PASS=$(echo "$HEAVY_OUTPUT" | grep -oP '\d+ passed' | grep -oP '\d+' || echo 0)
|
|
HEAVY_FAIL=$(echo "$HEAVY_OUTPUT" | grep -oP '\d+ failed' | grep -oP '\d+' || echo 0)
|
|
echo "Heavy suite: $HEAVY_PASS passed, $HEAVY_FAIL failed"
|
|
|
|
if [ "$HEAVY_FAIL" = "0" ] || [ -z "$HEAVY_FAIL" ]; then
|
|
ESCALATION_LOG="$ESCALATION_LOG → heavy PASSED ($HEAVY_PASS passed)"
|
|
RUN_MODE="tag:${TAG}→light→heavy"
|
|
PASS_COUNT="$HEAVY_PASS"; FAIL_COUNT="0"; OVERALL="pass"
|
|
else
|
|
ESCALATION_LOG="$ESCALATION_LOG → heavy FAILED ($HEAVY_PASS passed, $HEAVY_FAIL failed)"
|
|
RUN_MODE="tag:${TAG}→light→heavy"
|
|
PASS_COUNT="$HEAVY_PASS"; FAIL_COUNT="$HEAVY_FAIL"
|
|
TEST_OUTPUT="$HEAVY_OUTPUT"
|
|
fi
|
|
fi
|
|
echo "Escalation: $ESCALATION_LOG"
|
|
fi
|
|
|
|
# ─── Test Count Baseline Check ───
|
|
# Baseline check only applies to full suite runs (light), not tag-targeted runs
|
|
BASELINE=$(python3 -c "import json; print(json.load(open('$CONFIG')).get('test_count_baseline', 0))")
|
|
if [ "$RUN_MODE" = "light" ] && [ "$BASELINE" -gt 0 ] && [ "$PASS_COUNT" -lt "$BASELINE" ]; then
|
|
OVERALL="fail"
|
|
echo "TEST COUNT REGRESSION: expected >= $BASELINE passed, got $PASS_COUNT"
|
|
fi
|
|
|
|
# ─── Step 3: Package Check ───
|
|
echo ""
|
|
echo "--- Package Check ---"
|
|
PKG_OUTPUT=$($PYTHON -c "
|
|
import sys
|
|
required = ['fastapi','uvicorn','sqlalchemy','psycopg2','bcrypt','jwt','pandas','openpyxl','pydantic','ortools','astral','requests','dotenv','httpx']
|
|
missing = []
|
|
for mod in required:
|
|
try: __import__(mod)
|
|
except ImportError: missing.append(mod)
|
|
if missing:
|
|
print(f'FAIL: {len(missing)} missing: {missing}')
|
|
sys.exit(1)
|
|
print(f'OK: All {len(required)} packages verified')
|
|
" 2>&1)
|
|
PKG_EXIT=$?
|
|
echo "$PKG_OUTPUT"
|
|
if [ $PKG_EXIT -ne 0 ]; then
|
|
OVERALL="fail"
|
|
fi
|
|
|
|
# ─── Results ───
|
|
END_TIME=$(date +%s)
|
|
DURATION=$((END_TIME - START_TIME))
|
|
SUMMARY="${PASS_COUNT} passed, ${FAIL_COUNT} failed, ${SKIP_COUNT} skipped"
|
|
|
|
# Collect which feature tags were present in tested files
|
|
TAGS_USED=$(grep -rhoP 'pytest\.mark\.\K\w+' "$WORKTREE/backend/tests/test_"*.py 2>/dev/null | \
|
|
sort -u | grep -vxE 'skipif|skip|parametrize|fixture|unit|integration|pipeline|e2e|slow|heavy' | \
|
|
tr '\n' ', ' | sed 's/,$//')
|
|
if [ "$RUN_MODE" = "light" ]; then
|
|
SUITE_INFO="Mode: light (full) | Tags: ${TAGS_USED} | ${DESELECTED_COUNT} not in scope"
|
|
elif [ "$RUN_MODE" = "heavy" ]; then
|
|
SUITE_INFO="Mode: heavy (all tests) | Tags: ${TAGS_USED}"
|
|
else
|
|
SUITE_INFO="Mode: ${RUN_MODE} | ${DESELECTED_COUNT} not in scope"
|
|
fi
|
|
|
|
# Append escalation info if escalation occurred
|
|
if [ -n "$ESCALATION_LOG" ]; then
|
|
SUITE_INFO="${SUITE_INFO}
|
|
Escalation: ${ESCALATION_LOG}"
|
|
fi
|
|
|
|
echo ""
|
|
echo "========================="
|
|
echo "Result: $OVERALL"
|
|
echo "Summary: $SUMMARY"
|
|
echo "Features: $FEATURES_RESULT"
|
|
echo "Duration: ${DURATION}s"
|
|
echo "========================="
|
|
|
|
# ─── Generate Failure Summary HTML ───
|
|
REPORT_FILE=""
|
|
if [ "$OVERALL" = "fail" ]; then
|
|
REPORT_FILE="/tmp/tsharps-ci-report-${BRANCH}-$(date +%Y%m%d-%H%M%S).html"
|
|
FAILED_LINES=$(echo "$TEST_OUTPUT" | grep "^FAILED " || true)
|
|
ERROR_LINES=$(echo "$TEST_OUTPUT" | grep "^ERROR " || true)
|
|
FAILURE_DETAILS=$(echo "$TEST_OUTPUT" | grep -B0 "^FAILED \|^E \|Error\|assert" | head -60 || true)
|
|
|
|
python3 -c "
|
|
import html, sys
|
|
from datetime import datetime, timezone
|
|
|
|
branch = '$BRANCH'
|
|
commit = '$COMMIT'
|
|
actor = '$ACTOR'
|
|
duration = '${DURATION}s'
|
|
passed = '$PASS_COUNT'
|
|
failed = '$FAIL_COUNT'
|
|
skipped = '$SKIP_COUNT'
|
|
errors = '$ERROR_COUNT'
|
|
features = '''$FEATURES_RESULT'''
|
|
failed_lines = '''$FAILED_LINES'''
|
|
failure_details = '''$FAILURE_DETAILS'''
|
|
pkg_output = '''$PKG_OUTPUT'''
|
|
|
|
failed_items = [l.replace('FAILED ', '') for l in failed_lines.strip().split('\n') if l.strip()]
|
|
error_items = [l for l in '''$ERROR_LINES'''.strip().split('\n') if l.strip()]
|
|
|
|
rows = ''
|
|
for item in failed_items:
|
|
parts = item.rsplit('::', 1)
|
|
file_part = parts[0] if len(parts) > 0 else item
|
|
test_part = parts[1] if len(parts) > 1 else ''
|
|
rows += f'<tr><td><code>{html.escape(file_part)}</code></td><td><code>{html.escape(test_part)}</code></td></tr>\n'
|
|
|
|
for item in error_items:
|
|
rows += f'<tr><td colspan=\"2\"><code>{html.escape(item)}</code></td></tr>\n'
|
|
|
|
detail_block = html.escape(failure_details) if failure_details.strip() else '<em>No detailed output captured</em>'
|
|
pkg_block = html.escape(pkg_output) if pkg_output.strip() else ''
|
|
ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
|
|
baseline = '$BASELINE'
|
|
|
|
# Generate suggestions based on failure patterns
|
|
tips = []
|
|
all_output = failure_details + ' ' + features + ' ' + pkg_output
|
|
if 'MISSING' in features and 'features verified' in features:
|
|
tips.append(('Feature Manifest', 'One or more feature patterns were not found in the expected files. This usually means code was moved or renamed during a refactor. Check .features/*.manifest and update the file paths to match where the code lives now.'))
|
|
if int(failed) > 0:
|
|
if 'ImportError' in all_output or 'ModuleNotFoundError' in all_output:
|
|
tips.append(('Import Error', 'A test failed because a module could not be imported. A file may have been renamed, moved, or a dependency is missing from the venv. Run: pip install -r requirements.txt'))
|
|
if 'AssertionError' in all_output or 'assert' in all_output:
|
|
tips.append(('Assertion Failure', 'A test assertion did not match expected values. Check if the API response shape changed, a default value was modified, or test fixtures are stale.'))
|
|
if '401' in all_output or 'Unauthorized' in all_output:
|
|
tips.append(('Auth Failure', 'A test got 401 Unauthorized. The endpoint may have been put behind authentication, or the test client is missing a JWT cookie.'))
|
|
if '404' in all_output:
|
|
tips.append(('Route Not Found', 'A test got 404. The endpoint may have been renamed, moved to a different router, or removed entirely.'))
|
|
if '500' in all_output or 'Internal Server Error' in all_output:
|
|
tips.append(('Server Error', 'A test triggered a 500 error. Check the backend logs for the full traceback: journalctl -u tsharp-scheduler --since \"5 minutes ago\"'))
|
|
if int(errors) > 0:
|
|
if 'fixture' in all_output.lower():
|
|
tips.append(('Fixture Error', 'A test fixture failed during setup. A conftest.py fixture may be broken or a required package (pytest-mock, pytest-asyncio, pytest-timeout) may be missing.'))
|
|
if 'collection' in all_output.lower():
|
|
tips.append(('Collection Error', 'pytest failed to collect tests. A test file may have a syntax error or a top-level import that crashes.'))
|
|
if baseline and int(passed) < int(baseline) and int(passed) > 0:
|
|
diff = int(baseline) - int(passed)
|
|
tips.append(('Test Count Regression', f'{diff} fewer tests passed than the baseline ({baseline}). Tests may have been deleted, moved out of backend/tests/, or a new skip/deselect was introduced without updating the baseline.'))
|
|
if 'FAIL' in pkg_output:
|
|
tips.append(('Missing Package', 'One or more required Python packages are not installed. Run: source .venv/bin/activate && pip install -r requirements.txt'))
|
|
if not tips:
|
|
tips.append(('Unknown', 'No specific pattern matched. Check the failure details above and the backend logs for more context.'))
|
|
|
|
suggestions = '<ul style=\"margin:0;padding-left:1.2rem\">'
|
|
for title, tip in tips:
|
|
suggestions += f'<li><strong>{html.escape(title)}:</strong> {html.escape(tip)}</li>'
|
|
suggestions += '</ul>'
|
|
|
|
report = f'''<!DOCTYPE html>
|
|
<html lang=\"en\">
|
|
<head>
|
|
<meta charset=\"UTF-8\">
|
|
<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">
|
|
<title>CI Failure Report — {html.escape(branch)} ({html.escape(commit)})</title>
|
|
<style>
|
|
body {{ font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 800px; margin: 2rem auto; padding: 0 1rem; color: #e2e8f0; background: #0d1117; }}
|
|
h1 {{ color: #ff6b6b; border-bottom: 3px solid #ff6b6b; padding-bottom: 0.5rem; }}
|
|
h2 {{ color: #58a6ff; margin-top: 1.5rem; }}
|
|
.meta {{ background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem 1.5rem; margin: 1rem 0; }}
|
|
.meta table {{ width: 100%; border-collapse: collapse; }}
|
|
.meta td {{ padding: 0.3rem 0.5rem; color: #c9d1d9; }}
|
|
.meta td:first-child {{ font-weight: 600; width: 120px; color: #8b949e; }}
|
|
.meta ul {{ color: #c9d1d9; }}
|
|
.stats {{ display: flex; gap: 1rem; flex-wrap: wrap; margin: 1rem 0; }}
|
|
.stat {{ background: #161b22; border: 1px solid #30363d; padding: 0.75rem 1.25rem; border-radius: 6px; text-align: center; }}
|
|
.stat .num {{ font-size: 1.5rem; font-weight: 700; }}
|
|
.stat .label {{ font-size: 0.75rem; color: #8b949e; text-transform: uppercase; }}
|
|
.green {{ color: #40c057; }}
|
|
.red {{ color: #ff6b6b; }}
|
|
.yellow {{ color: #f59f00; }}
|
|
table.failures {{ width: 100%; border-collapse: collapse; margin: 1rem 0; background: #161b22; border: 1px solid #30363d; border-radius: 8px; overflow: hidden; }}
|
|
table.failures th {{ background: #3d0000; color: #ff6b6b; padding: 0.6rem 0.8rem; text-align: left; }}
|
|
table.failures td {{ padding: 0.5rem 0.8rem; border-bottom: 1px solid #21262d; font-size: 0.9em; color: #c9d1d9; }}
|
|
table.failures tr:last-child td {{ border-bottom: none; }}
|
|
pre {{ background: #161b22; color: #c9d1d9; padding: 1rem; border-radius: 8px; overflow-x: auto; font-size: 0.85em; line-height: 1.5; border: 1px solid #30363d; }}
|
|
code {{ background: #21262d; padding: 2px 6px; border-radius: 3px; font-size: 0.85em; color: #79c0ff; }}
|
|
.footer {{ color: #484f58; font-size: 0.8rem; margin-top: 2rem; border-top: 1px solid #21262d; padding-top: 0.5rem; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>CI Failure Report</h1>
|
|
|
|
<div class=\"meta\">
|
|
<table>
|
|
<tr><td>Branch</td><td><strong>{html.escape(branch)}</strong></td></tr>
|
|
<tr><td>Commit</td><td><code>{html.escape(commit)}</code></td></tr>
|
|
<tr><td>Pushed by</td><td>{html.escape(actor)}</td></tr>
|
|
<tr><td>Duration</td><td>{html.escape(duration)}</td></tr>
|
|
<tr><td>Features</td><td>{html.escape(features)}</td></tr>
|
|
<tr><td>Timestamp</td><td>{ts}</td></tr>
|
|
</table>
|
|
</div>
|
|
|
|
<div class=\"stats\">
|
|
<div class=\"stat\"><div class=\"num green\">{html.escape(passed)}</div><div class=\"label\">Passed</div></div>
|
|
<div class=\"stat\"><div class=\"num red\">{html.escape(failed)}</div><div class=\"label\">Failed</div></div>
|
|
<div class=\"stat\"><div class=\"num yellow\">{html.escape(skipped)}</div><div class=\"label\">Skipped</div></div>
|
|
<div class=\"stat\"><div class=\"num red\">{html.escape(errors)}</div><div class=\"label\">Errors</div></div>
|
|
</div>
|
|
|
|
<h2>Failed Tests</h2>
|
|
<table class=\"failures\">
|
|
<tr><th>File</th><th>Test</th></tr>
|
|
{rows}
|
|
</table>
|
|
|
|
<h2>Failure Details</h2>
|
|
<pre>{detail_block}</pre>
|
|
|
|
<h2>Suggestions</h2>
|
|
<div class=\"meta\">
|
|
{suggestions}
|
|
</div>
|
|
|
|
<div class=\"footer\">Generated by TSHARPS CI Runner — {ts}</div>
|
|
</body>
|
|
</html>'''
|
|
|
|
with open('$REPORT_FILE', 'w') as f:
|
|
f.write(report)
|
|
print(f'Report written to $REPORT_FILE')
|
|
" 2>&1
|
|
echo "Failure report: $REPORT_FILE"
|
|
fi
|
|
|
|
# ─── Write Pipeline Result JSON ───
|
|
RESULTS_DIR="$SCRIPT_DIR/pipeline-results"
|
|
RESULT_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
RESULT_FILENAME=$(date -u +%Y-%m-%d_%H%M%S)_${BRANCH}.json
|
|
RESULT_FILE="$RESULTS_DIR/$RESULT_FILENAME"
|
|
mkdir -p "$RESULTS_DIR"
|
|
python3 -c "
|
|
import json, sys
|
|
|
|
result = {
|
|
'branch': '$BRANCH',
|
|
'commit': '$COMMIT',
|
|
'actor': '$ACTOR',
|
|
'timestamp': '$RESULT_TIMESTAMP',
|
|
'result': '$OVERALL',
|
|
'tests': {
|
|
'passed': int('$PASS_COUNT' or 0),
|
|
'failed': int('$FAIL_COUNT' or 0),
|
|
'skipped': int('$SKIP_COUNT' or 0),
|
|
'deselected': int('$DESELECTED_COUNT' or 0),
|
|
'errors': int('$ERROR_COUNT' or 0),
|
|
},
|
|
'features': '$FEATURES_RESULT',
|
|
'duration_seconds': int('$DURATION' or 0),
|
|
'tags_tested': '$TAGS_USED',
|
|
'trigger_source': '$ACTOR',
|
|
}
|
|
|
|
with open('$RESULT_FILE', 'w') as f:
|
|
json.dump(result, f, indent=2)
|
|
print(f'Pipeline result written: $RESULT_FILE')
|
|
" 2>&1
|
|
|
|
# ─── Send Notification ───
|
|
bash "$SCRIPT_DIR/ci-notify.sh" \
|
|
"$BRANCH" "$COMMIT" "$OVERALL" "$SUMMARY" "$FEATURES_RESULT" "${DURATION}s" "$ACTOR" "$REPORT_FILE" "$SUITE_INFO"
|
|
|
|
# Release lock
|
|
flock -u 200
|
|
exit 0
|