TSHARPS-CI/ci-runner.sh

#!/bin/bash
# TSHARPS CI — External Test Runner
# Runs tests, feature manifests, and package checks against TSHARPS worktrees.
# READ-ONLY — never writes, commits, or pushes to TSHARPS.
# Same checks for ALL branches — no branch-specific logic.

set -o pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CONFIG="$SCRIPT_DIR/ci-config.json"
BRANCH="$1"
COMMIT="$2"
ACTOR="${3:-unknown}"
START_TIME=$(date +%s)

# Load config
TIMEOUT=$(python3 -c "import json; print(json.load(open('$CONFIG')).get('test_timeout', 120))")
WORKTREE=$(python3 -c "import json; b=json.load(open('$CONFIG'))['branches']; print(b.get('$BRANCH',{}).get('worktree',''))")

if [ -z "$WORKTREE" ] || [ ! -d "$WORKTREE" ]; then
  echo "ERROR: Unknown branch '$BRANCH' or worktree not found"
  bash "$SCRIPT_DIR/ci-notify.sh" "$BRANCH" "$COMMIT" "error" "Unknown branch or missing worktree" "" "0s" "$ACTOR"
  exit 1
fi

# Prevent concurrent runs on the same branch
LOCKFILE="/tmp/tsharps-ci-${BRANCH}.lock"
exec 200>"$LOCKFILE"
if ! flock -n 200; then
  echo "SKIP: CI already running for branch $BRANCH"
  exit 0
fi

# Serialize all CI runs to avoid DB connection contention
GLOBAL_LOCK="/tmp/tsharps-ci-global.lock"
exec 201>"$GLOBAL_LOCK"
flock 201

echo "=== TSHARPS CI Runner ==="
echo "Branch: $BRANCH"
echo "Commit: $COMMIT"
echo "Worktree: $WORKTREE"
echo "Timeout: ${TIMEOUT}s"
echo "========================="

PASS_COUNT=0
FAIL_COUNT=0
SKIP_COUNT=0
ERROR_COUNT=0
FEATURES_RESULT=""
OVERALL="pass"

# ─── Step 1: Feature Manifest Check ───
echo ""
echo "--- Feature Manifest Check ---"
if [ -f "$WORKTREE/ops/verify-features.py" ] && [ -d "$WORKTREE/.features" ]; then
  FEATURE_OUTPUT=$(cd "$WORKTREE" && python3 ops/verify-features.py --verbose 2>&1)
  FEATURE_EXIT=$?
  FEATURES_RESULT=$(echo "$FEATURE_OUTPUT" | grep "RESULT:" | tail -1)
  if [ -z "$FEATURES_RESULT" ]; then
    FEATURES_RESULT=$(echo "$FEATURE_OUTPUT" | tail -1)
  fi
  echo "$FEATURE_OUTPUT"
  if [ $FEATURE_EXIT -ne 0 ]; then
    OVERALL="fail"
    echo "FEATURE CHECK FAILED"
  fi
else
  FEATURES_RESULT="No manifests found"
  echo "No feature manifests — skipping"
fi

# ─── Step 2: Run Test Suite ───
echo ""
echo "--- Test Suite ---"
PYTHON="$WORKTREE/.venv/bin/python3"
if [ ! -f "$PYTHON" ]; then
  PYTHON="python3"
fi

TIMEOUT_FLAG=""
if $PYTHON -c "import pytest_timeout" 2>/dev/null; then
  TIMEOUT_FLAG="--timeout=$TIMEOUT"
fi

TEST_OUTPUT=$($PYTHON -m pytest "$WORKTREE/backend/tests/" --tb=line -q $TIMEOUT_FLAG 2>&1)
TEST_EXIT=$?

PASS_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ passed' | grep -oP '\d+' || echo 0)
FAIL_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ failed' | grep -oP '\d+' || echo 0)
SKIP_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ skipped' | grep -oP '\d+' || echo 0)
ERROR_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ error' | grep -oP '\d+' || echo 0)
DESELECTED_COUNT=$(echo "$TEST_OUTPUT" | grep -oP '\d+ deselected' | grep -oP '\d+' || echo 0)

if [ "$TEST_EXIT" -ne 0 ] && [ "$PASS_COUNT" = "0" ] && [ "$FAIL_COUNT" = "0" ]; then
  ERROR_COUNT=1
  echo "WARNING: pytest exited with code $TEST_EXIT but reported no results — likely a collection error"
  echo "$TEST_OUTPUT" | tail -5
fi

if [ "$PASS_COUNT" = "0" ] && [ "$FAIL_COUNT" = "0" ] && [ "$SKIP_COUNT" = "0" ]; then
  OVERALL="fail"
  echo "FAIL: 0 tests ran — something is wrong"
fi

echo "Tests: $PASS_COUNT passed, $FAIL_COUNT failed, $SKIP_COUNT skipped, $ERROR_COUNT errors"

if [ "$FAIL_COUNT" != "0" ] && [ "$FAIL_COUNT" != "" ]; then
  OVERALL="fail"
  echo "TESTS FAILED"
fi
if [ "$ERROR_COUNT" != "0" ] && [ "$ERROR_COUNT" != "" ]; then
  OVERALL="fail"
  echo "TEST COLLECTION ERRORS"
fi

# ─── Test Count Baseline Check ───
BASELINE=$(python3 -c "import json; print(json.load(open('$CONFIG')).get('test_count_baseline', 0))")
if [ "$BASELINE" -gt 0 ] && [ "$PASS_COUNT" -lt "$BASELINE" ]; then
  OVERALL="fail"
  echo "TEST COUNT REGRESSION: expected >= $BASELINE passed, got $PASS_COUNT"
fi

# ─── Step 3: Package Check ───
echo ""
echo "--- Package Check ---"
PKG_OUTPUT=$($PYTHON -c "
import sys
required = ['fastapi','uvicorn','sqlalchemy','psycopg2','bcrypt','jwt','pandas','openpyxl','pydantic','ortools','astral','requests','dotenv','httpx']
missing = []
for mod in required:
    try: __import__(mod)
    except ImportError: missing.append(mod)
if missing:
    print(f'FAIL: {len(missing)} missing: {missing}')
    sys.exit(1)
print(f'OK: All {len(required)} packages verified')
" 2>&1)
PKG_EXIT=$?
echo "$PKG_OUTPUT"
if [ $PKG_EXIT -ne 0 ]; then
  OVERALL="fail"
fi

# ─── Results ───
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
SUMMARY="${PASS_COUNT} passed, ${FAIL_COUNT} failed, ${SKIP_COUNT} skipped"

# Collect which feature tags were present in tested files
TAGS_USED=$(grep -rhoP 'pytest\.mark\.\K\w+' "$WORKTREE/backend/tests/test_"*.py 2>/dev/null | \
  sort -u | grep -vxE 'skipif|skip|parametrize|fixture|unit|integration|pipeline|e2e|slow|heavy' | \
  tr '\n' ', ' | sed 's/,$//')
SUITE_INFO="Tags: ${TAGS_USED} | ${DESELECTED_COUNT} not in scope"

echo ""
echo "========================="
echo "Result: $OVERALL"
echo "Summary: $SUMMARY"
echo "Features: $FEATURES_RESULT"
echo "Duration: ${DURATION}s"
echo "========================="

# ─── Generate Failure Summary HTML ───
REPORT_FILE=""
if [ "$OVERALL" = "fail" ]; then
  REPORT_FILE="/tmp/tsharps-ci-report-${BRANCH}-$(date +%Y%m%d-%H%M%S).html"
  FAILED_LINES=$(echo "$TEST_OUTPUT" | grep "^FAILED " || true)
  ERROR_LINES=$(echo "$TEST_OUTPUT" | grep "^ERROR " || true)
  FAILURE_DETAILS=$(echo "$TEST_OUTPUT" | grep -B0 "^FAILED \|^E   \|Error\|assert" | head -60 || true)

  python3 -c "
import html, sys
from datetime import datetime, timezone

branch = '$BRANCH'
commit = '$COMMIT'
actor = '$ACTOR'
duration = '${DURATION}s'
passed = '$PASS_COUNT'
failed = '$FAIL_COUNT'
skipped = '$SKIP_COUNT'
errors = '$ERROR_COUNT'
features = '''$FEATURES_RESULT'''
failed_lines = '''$FAILED_LINES'''
failure_details = '''$FAILURE_DETAILS'''
pkg_output = '''$PKG_OUTPUT'''

failed_items = [l.replace('FAILED ', '') for l in failed_lines.strip().split('\n') if l.strip()]
error_items = [l for l in '''$ERROR_LINES'''.strip().split('\n') if l.strip()]

rows = ''
for item in failed_items:
    parts = item.rsplit('::', 1)
    file_part = parts[0] if len(parts) > 0 else item
    test_part = parts[1] if len(parts) > 1 else ''
    rows += f'<tr><td><code>{html.escape(file_part)}</code></td><td><code>{html.escape(test_part)}</code></td></tr>\n'

for item in error_items:
    rows += f'<tr><td colspan=\"2\"><code>{html.escape(item)}</code></td></tr>\n'

detail_block = html.escape(failure_details) if failure_details.strip() else '<em>No detailed output captured</em>'
pkg_block = html.escape(pkg_output) if pkg_output.strip() else ''
ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
baseline = '$BASELINE'

# Generate suggestions based on failure patterns
tips = []
all_output = failure_details + ' ' + features + ' ' + pkg_output
if 'MISSING' in features and 'features verified' in features:
    tips.append(('Feature Manifest', 'One or more feature patterns were not found in the expected files. This usually means code was moved or renamed during a refactor. Check .features/*.manifest and update the file paths to match where the code lives now.'))
if int(failed) > 0:
    if 'ImportError' in all_output or 'ModuleNotFoundError' in all_output:
        tips.append(('Import Error', 'A test failed because a module could not be imported. A file may have been renamed, moved, or a dependency is missing from the venv. Run: pip install -r requirements.txt'))
    if 'AssertionError' in all_output or 'assert' in all_output:
        tips.append(('Assertion Failure', 'A test assertion did not match expected values. Check if the API response shape changed, a default value was modified, or test fixtures are stale.'))
    if '401' in all_output or 'Unauthorized' in all_output:
        tips.append(('Auth Failure', 'A test got 401 Unauthorized. The endpoint may have been put behind authentication, or the test client is missing a JWT cookie.'))
    if '404' in all_output:
        tips.append(('Route Not Found', 'A test got 404. The endpoint may have been renamed, moved to a different router, or removed entirely.'))
    if '500' in all_output or 'Internal Server Error' in all_output:
        tips.append(('Server Error', 'A test triggered a 500 error. Check the backend logs for the full traceback: journalctl -u tsharp-scheduler --since \"5 minutes ago\"'))
if int(errors) > 0:
    if 'fixture' in all_output.lower():
        tips.append(('Fixture Error', 'A test fixture failed during setup. A conftest.py fixture may be broken or a required package (pytest-mock, pytest-asyncio, pytest-timeout) may be missing.'))
    if 'collection' in all_output.lower():
        tips.append(('Collection Error', 'pytest failed to collect tests. A test file may have a syntax error or a top-level import that crashes.'))
if baseline and int(passed) < int(baseline) and int(passed) > 0:
    diff = int(baseline) - int(passed)
    tips.append(('Test Count Regression', f'{diff} fewer tests passed than the baseline ({baseline}). Tests may have been deleted, moved out of backend/tests/, or a new skip/deselect was introduced without updating the baseline.'))
if 'FAIL' in pkg_output:
    tips.append(('Missing Package', 'One or more required Python packages are not installed. Run: source .venv/bin/activate && pip install -r requirements.txt'))
if not tips:
    tips.append(('Unknown', 'No specific pattern matched. Check the failure details above and the backend logs for more context.'))

suggestions = '<ul style=\"margin:0;padding-left:1.2rem\">'
for title, tip in tips:
    suggestions += f'<li><strong>{html.escape(title)}:</strong> {html.escape(tip)}</li>'
suggestions += '</ul>'

report = f'''<!DOCTYPE html>
<html lang=\"en\">
<head>
<meta charset=\"UTF-8\">
<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">
<title>CI Failure Report — {html.escape(branch)} ({html.escape(commit)})</title>
<style>
  body {{ font-family: -apple-system, BlinkMacSystemFont, sans-serif; max-width: 800px; margin: 2rem auto; padding: 0 1rem; color: #e2e8f0; background: #0d1117; }}
  h1 {{ color: #ff6b6b; border-bottom: 3px solid #ff6b6b; padding-bottom: 0.5rem; }}
  h2 {{ color: #58a6ff; margin-top: 1.5rem; }}
  .meta {{ background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem 1.5rem; margin: 1rem 0; }}
  .meta table {{ width: 100%; border-collapse: collapse; }}
  .meta td {{ padding: 0.3rem 0.5rem; color: #c9d1d9; }}
  .meta td:first-child {{ font-weight: 600; width: 120px; color: #8b949e; }}
  .meta ul {{ color: #c9d1d9; }}
  .stats {{ display: flex; gap: 1rem; flex-wrap: wrap; margin: 1rem 0; }}
  .stat {{ background: #161b22; border: 1px solid #30363d; padding: 0.75rem 1.25rem; border-radius: 6px; text-align: center; }}
  .stat .num {{ font-size: 1.5rem; font-weight: 700; }}
  .stat .label {{ font-size: 0.75rem; color: #8b949e; text-transform: uppercase; }}
  .green {{ color: #40c057; }}
  .red {{ color: #ff6b6b; }}
  .yellow {{ color: #f59f00; }}
  table.failures {{ width: 100%; border-collapse: collapse; margin: 1rem 0; background: #161b22; border: 1px solid #30363d; border-radius: 8px; overflow: hidden; }}
  table.failures th {{ background: #3d0000; color: #ff6b6b; padding: 0.6rem 0.8rem; text-align: left; }}
  table.failures td {{ padding: 0.5rem 0.8rem; border-bottom: 1px solid #21262d; font-size: 0.9em; color: #c9d1d9; }}
  table.failures tr:last-child td {{ border-bottom: none; }}
  pre {{ background: #161b22; color: #c9d1d9; padding: 1rem; border-radius: 8px; overflow-x: auto; font-size: 0.85em; line-height: 1.5; border: 1px solid #30363d; }}
  code {{ background: #21262d; padding: 2px 6px; border-radius: 3px; font-size: 0.85em; color: #79c0ff; }}
  .footer {{ color: #484f58; font-size: 0.8rem; margin-top: 2rem; border-top: 1px solid #21262d; padding-top: 0.5rem; }}
</style>
</head>
<body>
<h1>CI Failure Report</h1>

<div class=\"meta\">
<table>
  <tr><td>Branch</td><td><strong>{html.escape(branch)}</strong></td></tr>
  <tr><td>Commit</td><td><code>{html.escape(commit)}</code></td></tr>
  <tr><td>Pushed by</td><td>{html.escape(actor)}</td></tr>
  <tr><td>Duration</td><td>{html.escape(duration)}</td></tr>
  <tr><td>Features</td><td>{html.escape(features)}</td></tr>
  <tr><td>Timestamp</td><td>{ts}</td></tr>
</table>
</div>

<div class=\"stats\">
  <div class=\"stat\"><div class=\"num green\">{html.escape(passed)}</div><div class=\"label\">Passed</div></div>
  <div class=\"stat\"><div class=\"num red\">{html.escape(failed)}</div><div class=\"label\">Failed</div></div>
  <div class=\"stat\"><div class=\"num yellow\">{html.escape(skipped)}</div><div class=\"label\">Skipped</div></div>
  <div class=\"stat\"><div class=\"num red\">{html.escape(errors)}</div><div class=\"label\">Errors</div></div>
</div>

<h2>Failed Tests</h2>
<table class=\"failures\">
  <tr><th>File</th><th>Test</th></tr>
  {rows}
</table>

<h2>Failure Details</h2>
<pre>{detail_block}</pre>

<h2>Suggestions</h2>
<div class=\"meta\">
{suggestions}
</div>

<div class=\"footer\">Generated by TSHARPS CI Runner — {ts}</div>
</body>
</html>'''

with open('$REPORT_FILE', 'w') as f:
    f.write(report)
print(f'Report written to $REPORT_FILE')
" 2>&1
  echo "Failure report: $REPORT_FILE"
fi

# ─── Write Pipeline Result JSON ───
RESULTS_DIR="$SCRIPT_DIR/pipeline-results"
RESULT_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
RESULT_FILENAME=$(date -u +%Y-%m-%d_%H%M%S)_${BRANCH}.json
RESULT_FILE="$RESULTS_DIR/$RESULT_FILENAME"
mkdir -p "$RESULTS_DIR"
python3 -c "
import json, sys

result = {
    'branch': '$BRANCH',
    'commit': '$COMMIT',
    'actor': '$ACTOR',
    'timestamp': '$RESULT_TIMESTAMP',
    'result': '$OVERALL',
    'tests': {
        'passed': int('$PASS_COUNT' or 0),
        'failed': int('$FAIL_COUNT' or 0),
        'skipped': int('$SKIP_COUNT' or 0),
        'deselected': int('$DESELECTED_COUNT' or 0),
        'errors': int('$ERROR_COUNT' or 0),
    },
    'features': '$FEATURES_RESULT',
    'duration_seconds': int('$DURATION' or 0),
    'tags_tested': '$TAGS_USED',
    'trigger_source': '$ACTOR',
}

with open('$RESULT_FILE', 'w') as f:
    json.dump(result, f, indent=2)
print(f'Pipeline result written: $RESULT_FILE')
" 2>&1

# ─── Send Notification ───
bash "$SCRIPT_DIR/ci-notify.sh" \
  "$BRANCH" "$COMMIT" "$OVERALL" "$SUMMARY" "$FEATURES_RESULT" "${DURATION}s" "$ACTOR" "$REPORT_FILE" "$SUITE_INFO"

# Release lock
flock -u 200
exit 0