ab89dc05b2
When tests fail, ci-runner.sh generates an HTML report with: - Build metadata (branch, commit, actor, duration) - Test stats (passed/failed/skipped/errors) - Failed test table (file + test name) - Failure detail output ci-notify.sh sends the HTML as a document attachment alongside the text notification in the CICD Pipeline topic. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
269 lines
9.5 KiB
Bash
Executable File
269 lines
9.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# TSHARPS CI — External Test Runner
|
|
# Runs tests, feature manifests, and package checks against TSHARPS worktrees.
|
|
# READ-ONLY — never writes, commits, or pushes to TSHARPS.
|
|
# Same checks for ALL branches — no branch-specific logic.
|
|
|
|
set -o pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
CONFIG="$SCRIPT_DIR/ci-config.json"
|
|
BRANCH="$1"
|
|
COMMIT="$2"
|
|
ACTOR="${3:-unknown}"
|
|
START_TIME=$(date +%s)
|
|
|
|
# Load config
|
|
TIMEOUT=$(python3 -c "import json; print(json.load(open('$CONFIG')).get('test_timeout', 120))")
|
|
WORKTREE=$(python3 -c "import json; b=json.load(open('$CONFIG'))['branches']; print(b.get('$BRANCH',{}).get('worktree',''))")
|
|
|
|
if [ -z "$WORKTREE" ] || [ ! -d "$WORKTREE" ]; then
|
|
echo "ERROR: Unknown branch '$BRANCH' or worktree not found"
|
|
bash "$SCRIPT_DIR/ci-notify.sh" "$BRANCH" "$COMMIT" "error" "Unknown branch or missing worktree" "" "0s" "$ACTOR"
|
|
exit 1
|
|
fi
|
|
|
|
# Prevent concurrent runs on the same branch
|
|
LOCKFILE="/tmp/tsharps-ci-${BRANCH}.lock"
|
|
exec 200>"$LOCKFILE"
|
|
if ! flock -n 200; then
|
|
echo "SKIP: CI already running for branch $BRANCH"
|
|
exit 0
|
|
fi
|
|
|
|
# Serialize all CI runs to avoid DB connection contention
|
|
GLOBAL_LOCK="/tmp/tsharps-ci-global.lock"
|
|
exec 201>"$GLOBAL_LOCK"
|
|
flock 201
|
|
|
|
echo "=== TSHARPS CI Runner ==="
|
|
echo "Branch: $BRANCH"
|
|
echo "Commit: $COMMIT"
|
|
echo "Worktree: $WORKTREE"
|
|
echo "Timeout: ${TIMEOUT}s"
|
|
echo "========================="
|
|
|
|
PASS_COUNT=0
|
|
FAIL_COUNT=0
|
|
SKIP_COUNT=0
|
|
ERROR_COUNT=0
|
|
FEATURES_RESULT=""
|
|
OVERALL="pass"
|
|
|
|
# ─── Step 1: Feature Manifest Check ───
|
|
echo ""
|
|
echo "--- Feature Manifest Check ---"
|
|
if [ -f "$WORKTREE/ops/verify-features.py" ] && [ -d "$WORKTREE/.features" ]; then
|
|
FEATURE_OUTPUT=$(cd "$WORKTREE" && python3 ops/verify-features.py --verbose 2>&1)
|
|
FEATURE_EXIT=$?
|
|
FEATURES_RESULT=$(echo "$FEATURE_OUTPUT" | grep "RESULT:" | tail -1)
|
|
if [ -z "$FEATURES_RESULT" ]; then
|
|
FEATURES_RESULT=$(echo "$FEATURE_OUTPUT" | tail -1)
|
|
fi
|
|
echo "$FEATURE_OUTPUT"
|
|
if [ $FEATURE_EXIT -ne 0 ]; then
|
|
OVERALL="fail"
|
|
echo "FEATURE CHECK FAILED"
|
|
fi
|
|
else
|
|
FEATURES_RESULT="No manifests found"
|
|
echo "No feature manifests — skipping"
|
|
fi
|
|
|
|
# ─── Step 2: Run Test Suite ───
|
|
echo ""
|
|
echo "--- Test Suite ---"
|
|
PYTHON="$WORKTREE/.venv/bin/python3"
|
|
if [ ! -f "$PYTHON" ]; then
|
|
PYTHON="python3"
|
|
fi
|
|
|
|
TIMEOUT_FLAG=""
|
|
if $PYTHON -c "import pytest_timeout" 2>/dev/null; then
|
|
TIMEOUT_FLAG="--timeout=$TIMEOUT"
|
|
fi
|
|
|
|
TEST_OUTPUT=$($PYTHON -m pytest "$WORKTREE/backend/tests/" --tb=line -q $TIMEOUT_FLAG 2>&1)
|
|
TEST_EXIT=$?
|
|
|
|
PASS_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ passed' | grep -oP '\d+' || echo 0)
|
|
FAIL_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ failed' | grep -oP '\d+' || echo 0)
|
|
SKIP_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ skipped' | grep -oP '\d+' || echo 0)
|
|
ERROR_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ error' | grep -oP '\d+' || echo 0)
|
|
|
|
if [ "$TEST_EXIT" -ne 0 ] && [ "$PASS_COUNT" = "0" ] && [ "$FAIL_COUNT" = "0" ]; then
|
|
ERROR_COUNT=1
|
|
echo "WARNING: pytest exited with code $TEST_EXIT but reported no results — likely a collection error"
|
|
echo "$TEST_OUTPUT" | tail -5
|
|
fi
|
|
|
|
if [ "$PASS_COUNT" = "0" ] && [ "$FAIL_COUNT" = "0" ] && [ "$SKIP_COUNT" = "0" ]; then
|
|
OVERALL="fail"
|
|
echo "FAIL: 0 tests ran — something is wrong"
|
|
fi
|
|
|
|
echo "Tests: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped, $ERROR_COUNT errors"
|
|
|
|
if [ "$FAIL_COUNT" != "0" ] && [ "$FAIL_COUNT" != "" ]; then
|
|
OVERALL="fail"
|
|
echo "TESTS FAILED"
|
|
fi
|
|
if [ "$ERROR_COUNT" != "0" ] && [ "$ERROR_COUNT" != "" ]; then
|
|
OVERALL="fail"
|
|
echo "TEST COLLECTION ERRORS"
|
|
fi
|
|
|
|
# ─── Step 3: Package Check ───
|
|
echo ""
|
|
echo "--- Package Check ---"
|
|
PKG_OUTPUT=$($PYTHON -c "
|
|
import sys
|
|
required = ['fastapi','uvicorn','sqlalchemy','psycopg2','bcrypt','jwt','pandas','openpyxl','pydantic','ortools','astral','requests','dotenv','httpx']
|
|
missing = []
|
|
for mod in required:
|
|
try: __import__(mod)
|
|
except ImportError: missing.append(mod)
|
|
if missing:
|
|
print(f'FAIL: {len(missing)} missing: {missing}')
|
|
sys.exit(1)
|
|
print(f'OK: All {len(required)} packages verified')
|
|
" 2>&1)
|
|
PKG_EXIT=$?
|
|
echo "$PKG_OUTPUT"
|
|
if [ $PKG_EXIT -ne 0 ]; then
|
|
OVERALL="fail"
|
|
fi
|
|
|
|
# ─── Results ───
|
|
END_TIME=$(date +%s)
|
|
DURATION=$((END_TIME - START_TIME))
|
|
SUMMARY="${PASS_COUNT} passed, ${FAIL_COUNT} failed, ${SKIP_COUNT} skipped"
|
|
|
|
echo ""
|
|
echo "========================="
|
|
echo "Result: $OVERALL"
|
|
echo "Summary: $SUMMARY"
|
|
echo "Features: $FEATURES_RESULT"
|
|
echo "Duration: ${DURATION}s"
|
|
echo "========================="
|
|
|
|
# ─── Generate Failure Summary HTML ───
|
|
REPORT_FILE=""
|
|
if [ "$OVERALL" = "fail" ]; then
|
|
REPORT_FILE="/tmp/tsharps-ci-report-${BRANCH}-$(date +%Y%m%d-%H%M%S).html"
|
|
FAILED_LINES=$(echo "$TEST_OUTPUT" | grep "^FAILED " || true)
|
|
ERROR_LINES=$(echo "$TEST_OUTPUT" | grep "^ERROR " || true)
|
|
FAILURE_DETAILS=$(echo "$TEST_OUTPUT" | grep -B0 "^FAILED \|^E \|Error\|assert" | head -60 || true)
|
|
|
|
python3 -c "
|
|
import html, sys
|
|
from datetime import datetime, timezone
|
|
|
|
branch = '$BRANCH'
|
|
commit = '$COMMIT'
|
|
actor = '$ACTOR'
|
|
duration = '${DURATION}s'
|
|
passed = '$PASS_COUNT'
|
|
failed = '$FAIL_COUNT'
|
|
skipped = '$SKIP_COUNT'
|
|
errors = '$ERROR_COUNT'
|
|
features = '''$FEATURES_RESULT'''
|
|
failed_lines = '''$FAILED_LINES'''
|
|
failure_details = '''$FAILURE_DETAILS'''
|
|
pkg_output = '''$PKG_OUTPUT'''
|
|
|
|
failed_items = [l.replace('FAILED ', '') for l in failed_lines.strip().split('\n') if l.strip()]
|
|
error_items = [l for l in '''$ERROR_LINES'''.strip().split('\n') if l.strip()]
|
|
|
|
rows = ''
|
|
for item in failed_items:
|
|
parts = item.rsplit('::', 1)
|
|
file_part = parts[0] if len(parts) > 0 else item
|
|
test_part = parts[1] if len(parts) > 1 else ''
|
|
rows += f'<tr><td><code>{html.escape(file_part)}</code></td><td><code>{html.escape(test_part)}</code></td></tr>\n'
|
|
|
|
for item in error_items:
|
|
rows += f'<tr><td colspan=\"2\"><code>{html.escape(item)}</code></td></tr>\n'
|
|
|
|
detail_block = html.escape(failure_details) if failure_details.strip() else '<em>No detailed output captured</em>'
|
|
pkg_block = html.escape(pkg_output) if pkg_output.strip() else ''
|
|
ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
|
|
|
|
report = f'''<!DOCTYPE html>
|
|
<html lang=\"en\">
|
|
<head>
|
|
<meta charset=\"UTF-8\">
|
|
<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">
|
|
<title>CI Failure Report — {html.escape(branch)} ({html.escape(commit)})</title>
|
|
<style>
|
|
body {{ font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 800px; margin: 2rem auto; padding: 0 1rem; color: #1a1a2e; background: #f5f5f7; }}
|
|
h1 {{ color: #e53e3e; border-bottom: 3px solid #e53e3e; padding-bottom: 0.5rem; }}
|
|
.meta {{ background: #fff; border-radius: 8px; padding: 1rem 1.5rem; margin: 1rem 0; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}
|
|
.meta table {{ width: 100%; border-collapse: collapse; }}
|
|
.meta td {{ padding: 0.3rem 0.5rem; }}
|
|
.meta td:first-child {{ font-weight: 600; width: 120px; color: #4a5568; }}
|
|
.stats {{ display: flex; gap: 1rem; flex-wrap: wrap; margin: 1rem 0; }}
|
|
.stat {{ background: #fff; padding: 0.75rem 1.25rem; border-radius: 6px; text-align: center; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}
|
|
.stat .num {{ font-size: 1.5rem; font-weight: 700; }}
|
|
.stat .label {{ font-size: 0.75rem; color: #718096; text-transform: uppercase; }}
|
|
.green {{ color: #38a169; }}
|
|
.red {{ color: #e53e3e; }}
|
|
.yellow {{ color: #d69e2e; }}
|
|
table.failures {{ width: 100%; border-collapse: collapse; margin: 1rem 0; background: #fff; border-radius: 8px; overflow: hidden; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }}
|
|
table.failures th {{ background: #e53e3e; color: white; padding: 0.6rem 0.8rem; text-align: left; }}
|
|
table.failures td {{ padding: 0.5rem 0.8rem; border-bottom: 1px solid #e2e8f0; font-size: 0.9em; }}
|
|
table.failures tr:last-child td {{ border-bottom: none; }}
|
|
pre {{ background: #2d3748; color: #e2e8f0; padding: 1rem; border-radius: 8px; overflow-x: auto; font-size: 0.85em; line-height: 1.5; }}
|
|
code {{ background: #edf2f7; padding: 2px 6px; border-radius: 3px; font-size: 0.85em; }}
|
|
.footer {{ color: #a0aec0; font-size: 0.8rem; margin-top: 2rem; border-top: 1px solid #e2e8f0; padding-top: 0.5rem; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>CI Failure Report</h1>
|
|
|
|
<div class=\"meta\">
|
|
<table>
|
|
<tr><td>Branch</td><td><strong>{html.escape(branch)}</strong></td></tr>
|
|
<tr><td>Commit</td><td><code>{html.escape(commit)}</code></td></tr>
|
|
<tr><td>Pushed by</td><td>{html.escape(actor)}</td></tr>
|
|
<tr><td>Duration</td><td>{html.escape(duration)}</td></tr>
|
|
<tr><td>Features</td><td>{html.escape(features)}</td></tr>
|
|
<tr><td>Timestamp</td><td>{ts}</td></tr>
|
|
</table>
|
|
</div>
|
|
|
|
<div class=\"stats\">
|
|
<div class=\"stat\"><div class=\"num green\">{html.escape(passed)}</div><div class=\"label\">Passed</div></div>
|
|
<div class=\"stat\"><div class=\"num red\">{html.escape(failed)}</div><div class=\"label\">Failed</div></div>
|
|
<div class=\"stat\"><div class=\"num yellow\">{html.escape(skipped)}</div><div class=\"label\">Skipped</div></div>
|
|
<div class=\"stat\"><div class=\"num red\">{html.escape(errors)}</div><div class=\"label\">Errors</div></div>
|
|
</div>
|
|
|
|
<h2>Failed Tests</h2>
|
|
<table class=\"failures\">
|
|
<tr><th>File</th><th>Test</th></tr>
|
|
{rows}
|
|
</table>
|
|
|
|
<h2>Failure Details</h2>
|
|
<pre>{detail_block}</pre>
|
|
|
|
<div class=\"footer\">Generated by TSHARPS CI Runner — {ts}</div>
|
|
</body>
|
|
</html>'''
|
|
|
|
with open('$REPORT_FILE', 'w') as f:
|
|
f.write(report)
|
|
print(f'Report written to $REPORT_FILE')
|
|
" 2>&1
|
|
echo "Failure report: $REPORT_FILE"
|
|
fi
|
|
|
|
# ─── Send Notification ───
|
|
bash "$SCRIPT_DIR/ci-notify.sh" \
|
|
"$BRANCH" "$COMMIT" "$OVERALL" "$SUMMARY" "$FEATURES_RESULT" "${DURATION}s" "$ACTOR" "$REPORT_FILE"
|
|
|
|
# Release lock
|
|
flock -u 200
|
|
exit 0
|