-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathvalidate-reachability.sh
More file actions
executable file
·100 lines (89 loc) · 3.94 KB
/
Copy pathvalidate-reachability.sh
File metadata and controls
executable file
·100 lines (89 loc) · 3.94 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#!/usr/bin/env bash
set -euo pipefail
LOG="/tmp/e2e-output.log"
# 1. Verify reachability analysis completed
if grep -q "Reachability analysis completed successfully" "$LOG"; then
echo "PASS: Reachability analysis completed"
grep "Reachability analysis completed successfully" "$LOG"
grep "Results written to:" "$LOG" || true
else
echo "FAIL: Reachability analysis did not complete successfully"
cat "$LOG"
exit 1
fi
# 2. Verify scan produced a report URL
if grep -q "Full scan report URL: https://socket.dev/" "$LOG"; then
echo "PASS: Full scan report URL found"
grep "Full scan report URL:" "$LOG"
elif grep -q "Diff Url: https://socket.dev/" "$LOG"; then
echo "PASS: Diff URL found"
grep "Diff Url:" "$LOG"
else
echo "FAIL: No report URL found in scan output"
cat "$LOG"
exit 1
fi
FACTS_PATH="tests/e2e/fixtures/simple-npm/.socket.facts.json"
if [ ! -f "$FACTS_PATH" ]; then
echo "FAIL: Expected reachability facts at $FACTS_PATH after initial scan"
exit 1
fi
echo "PASS: Reachability facts file present at $FACTS_PATH"
# The tier-1 backend intermittently returns the known fixture as one orphaned
# component with zero projects, so Coana has no vulnerability to analyze even
# though manifest upload, facts generation, and scan finalization all succeed.
# After the workflow's bounded retries, classify only that explicit
# upstream signature as inconclusive. Any other empty facts result still fails,
# including the important regression case where Coana received a vulnerability
# but the CLI lost its alerted component.
if ! bash tests/e2e/reach-facts-probe.sh tests/e2e/fixtures/simple-npm; then
if grep -q "Found 1 manifest files for reachability upload" "$LOG" && \
grep -q "Found 0 projects across 0 ecosystems to analyze" "$LOG" && \
grep -q "Filtered out 1 orphaned component" "$LOG"; then
echo "::warning title=e2e-reachability inconclusive backend result::tier-1 returned the known zero-project/orphaned-component signature after retries; core reachability execution and finalization passed"
echo "e2e-reachability: inconclusive after retries — known zero-project backend signature; diagnostics uploaded" >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
exit 0
fi
echo "FAIL: no components with alerts in .socket.facts.json and the known backend signature was not present"
exit 1
fi
# 3-4. Build SARIF from the facts file produced by the initial --reach run.
# Avoid re-running reach + full scan here; duplicate API scans are slow and flaky in CI.
uv run python -c "
import json
from pathlib import Path
from socketsecurity.core.alert_selection import load_components_with_alerts
from socketsecurity.core.messages import Messages
target = 'tests/e2e/fixtures/simple-npm'
facts_file = '.socket.facts.json'
components = load_components_with_alerts(target, facts_file)
if not components:
raise SystemExit('FAIL: no components with alerts in .socket.facts.json')
for outfile, reach_filter in [
('/tmp/sarif-all.sarif', 'all'),
('/tmp/sarif-reachable.sarif', 'reachable'),
]:
sarif = Messages.create_security_comment_sarif_from_facts(
components,
reachability_filter=reach_filter,
grouping='instance',
)
Path(outfile).write_text(json.dumps(sarif, indent=2))
count = len(sarif['runs'][0]['results'])
print(f'PASS: Wrote {outfile} ({count} results, filter={reach_filter})')
"
# 5. Verify reachable-only results are a subset of all results
test -f /tmp/sarif-all.sarif
test -f /tmp/sarif-reachable.sarif
uv run python -c "
import json
with open('/tmp/sarif-all.sarif') as f:
all_data = json.load(f)
with open('/tmp/sarif-reachable.sarif') as f:
reach_data = json.load(f)
all_count = len(all_data['runs'][0]['results'])
reach_count = len(reach_data['runs'][0]['results'])
print(f'All results: {all_count}, Reachable-only results: {reach_count}')
assert reach_count <= all_count, f'FAIL: reachable ({reach_count}) > all ({all_count})'
print('PASS: Reachable-only results is a subset of all results')
"