SocketDev · lelia · Apr 8, 2026 · Apr 8, 2026 · Apr 8, 2026 · Apr 8, 2026
@@ -1,4 +1,4 @@
-name: E2E Test
+name: E2E Tests
 
 on:
   push:
@@ -10,93 +10,58 @@ permissions:
   contents: read
 
 jobs:
-  e2e-scan:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
-        with:
-          python-version: '3.12'
-
-      - name: Install CLI from local repo
-        run: |
-          python -m pip install --upgrade pip
-          pip install .
-
-      - name: Run Socket CLI scan
-        env:
-          SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          set -o pipefail
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --disable-blocking \
-            --enable-debug \
-            2>&1 | tee /tmp/scan-output.log
-
-      - name: Verify scan produced a report
-        run: |
-          if grep -q "Full scan report URL: https://socket.dev/" /tmp/scan-output.log; then
-            echo "PASS: Full scan report URL found"
-            grep "Full scan report URL:" /tmp/scan-output.log
-          elif grep -q "Diff Url: https://socket.dev/" /tmp/scan-output.log; then
-            echo "PASS: Diff URL found"
-            grep "Diff Url:" /tmp/scan-output.log
-          else
-            echo "FAIL: No report URL found in scan output"
-            cat /tmp/scan-output.log
-            exit 1
-          fi
-
-  e2e-sarif:
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
-        with:
-          python-version: '3.12'
-
-      - name: Install CLI from local repo
-        run: |
-          python -m pip install --upgrade pip
-          pip install .
-
-      - name: Run Socket CLI scan with --sarif-file
-        env:
-          SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          set -o pipefail
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --sarif-file /tmp/results.sarif \
-            --disable-blocking \
-            2>&1 | tee /tmp/sarif-output.log
-
-      - name: Verify SARIF file is valid
-        run: |
-          python3 -c "
-          import json, sys
-          with open('/tmp/results.sarif') as f:
-              data = json.load(f)
-          assert data['version'] == '2.1.0', f'Invalid version: {data[\"version\"]}'
-          assert '\$schema' in data, 'Missing \$schema'
-          count = len(data['runs'][0]['results'])
-          print(f'PASS: Valid SARIF 2.1.0 with {count} result(s)')
-          "
-
-  e2e-reachability:
+  e2e:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: scan
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --disable-blocking
+              --enable-debug
+            validate: tests/e2e/validate-scan.sh
+
+          - name: sarif
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --sarif-file /tmp/results.sarif
+              --disable-blocking
+            validate: tests/e2e/validate-sarif.sh
+
+          - name: reachability
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --reach
+              --disable-blocking
+              --enable-debug
+            validate: tests/e2e/validate-reachability.sh
+            setup-node: "true"
+
+          - name: gitlab
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --enable-gitlab-security
+              --disable-blocking
+            validate: tests/e2e/validate-gitlab.sh
+
+          - name: json
+            args: >-
+              --target-path tests/e2e/fixtures/simple-npm
+              --enable-json
+              --disable-blocking
+            validate: tests/e2e/validate-json.sh
+
+          - name: pypi
+            args: >-
+              --target-path tests/e2e/fixtures/simple-pypi
+              --disable-blocking
+              --enable-debug
+            validate: tests/e2e/validate-scan.sh
+
+    name: e2e-${{ matrix.name }}
     steps:
       - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
         with:
@@ -108,6 +73,7 @@ jobs:
           python-version: '3.12'
 
       - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af
+        if: matrix.setup-node == 'true'
         with:
           node-version: '20'
 
@@ -117,85 +83,17 @@ jobs:
           pip install .
 
       - name: Install uv
+        if: matrix.setup-node == 'true'
         run: pip install uv
 
-      - name: Run Socket CLI with reachability
+      - name: Run Socket CLI
         env:
           SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
         run: |
           set -o pipefail
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --reach \
-            --disable-blocking \
-            --enable-debug \
-            2>&1 | tee /tmp/reach-output.log
-
-      - name: Verify reachability analysis completed
-        run: |
-          if grep -q "Reachability analysis completed successfully" /tmp/reach-output.log; then
-            echo "PASS: Reachability analysis completed"
-            grep "Reachability analysis completed successfully" /tmp/reach-output.log
-            grep "Results written to:" /tmp/reach-output.log || true
-          else
-            echo "FAIL: Reachability analysis did not complete successfully"
-            cat /tmp/reach-output.log
-            exit 1
-          fi
-
-      - name: Verify scan produced a report
-        run: |
-          if grep -q "Full scan report URL: https://socket.dev/" /tmp/reach-output.log; then
-            echo "PASS: Full scan report URL found"
-            grep "Full scan report URL:" /tmp/reach-output.log
-          elif grep -q "Diff Url: https://socket.dev/" /tmp/reach-output.log; then
-            echo "PASS: Diff URL found"
-            grep "Diff Url:" /tmp/reach-output.log
-          else
-            echo "FAIL: No report URL found in scan output"
-            cat /tmp/reach-output.log
-            exit 1
-          fi
+          socketcli ${{ matrix.args }} 2>&1 | tee /tmp/e2e-output.log
 
-      - name: Run scan with --sarif-file (all results)
+      - name: Validate results
         env:
           SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --reach \
-            --sarif-file /tmp/sarif-all.sarif \
-            --sarif-scope full \
-            --sarif-reachability all \
-            --disable-blocking \
-            2>/dev/null
-
-      - name: Run scan with --sarif-file --sarif-reachability reachable (filtered results)
-        env:
-          SOCKET_SECURITY_API_KEY: ${{ secrets.SOCKET_CLI_API_TOKEN }}
-        run: |
-          socketcli \
-            --target-path tests/e2e/fixtures/simple-npm \
-            --reach \
-            --sarif-file /tmp/sarif-reachable.sarif \
-            --sarif-scope full \
-            --sarif-reachability reachable \
-            --disable-blocking \
-            2>/dev/null
-
-      - name: Verify reachable-only results are a subset of all results
-        run: |
-          test -f /tmp/sarif-all.sarif
-          test -f /tmp/sarif-reachable.sarif
-          python3 -c "
-          import json
-          with open('/tmp/sarif-all.sarif') as f:
-              all_data = json.load(f)
-          with open('/tmp/sarif-reachable.sarif') as f:
-              reach_data = json.load(f)
-          all_count = len(all_data['runs'][0]['results'])
-          reach_count = len(reach_data['runs'][0]['results'])
-          print(f'All results: {all_count}, Reachable-only results: {reach_count}')
-          assert reach_count <= all_count, f'FAIL: reachable ({reach_count}) > all ({all_count})'
-          print('PASS: Reachable-only results is a subset of all results')
-          "
+        run: bash ${{ matrix.validate }}
@@ -700,17 +700,44 @@ The GitLab report includes **actionable security alerts** based on your Socket p
 
 All alert types are included in the GitLab report if they're marked as `error` or `warn` by your Socket Security policy, ensuring the Security Dashboard shows only actionable findings.
 
+### Alert Population: GitLab vs JSON/SARIF
+
+The GitLab Security Dashboard report and the JSON/SARIF diff outputs use different alert selection strategies, reflecting their distinct purposes:
+
+| Output Format | Default Alerts | With `--strict-blocking` |
+|:---|:---|:---|
+| `--enable-gitlab-security` | **All** alerts (new + existing) | All alerts (same) |
+| `--enable-json` | New alerts only | New + existing alerts |
+| `--enable-sarif` (diff scope) | New alerts only | New + existing alerts |
+
+**Why the difference?** GitLab's Security Dashboard is designed to present the full security posture of a project. An empty dashboard on a scan with no dependency changes would be misleading -- the vulnerabilities still exist, they just didn't change. By contrast, JSON and SARIF in diff scope are designed to answer "what changed?" and only include existing alerts when `--strict-blocking` explicitly requests it.
+
+> **Tip:** If you use `--enable-json` alongside `--enable-gitlab-security`, the GitLab report may contain more vulnerabilities than the JSON output. This is expected. To make JSON output match, add `--strict-blocking`.
+
+### Alert Ignoring via PR/MR Comments
+
+When using the CLI with SCM integration (`--scm github` or `--scm gitlab`), users can ignore specific alerts by reacting to Socket's PR/MR comments. Ignored alerts are removed from `--enable-json`, `--enable-sarif`, and console output.
+
+However, the GitLab Security Dashboard report includes **all** alerts matching your security policy (new and existing), regardless of comment-based ignores. This ensures the Security Dashboard always reflects the full set of known issues. To suppress a vulnerability from the GitLab report, adjust the alert's policy in Socket's dashboard rather than ignoring it via a PR comment.
+
 ### Report Schema
 
-Socket CLI generates reports compliant with [GitLab Dependency Scanning schema version 15.0.0](https://docs.gitlab.com/ee/development/integrations/secure.html). The reports include:
+Socket CLI generates reports compliant with [GitLab Dependency Scanning schema version 15.0.0](https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/v15.0.0/dist/dependency-scanning-report-format.json). The reports include:
 
-- **Scan metadata**: Analyzer and scanner information
+- **Scan metadata**: Analyzer and scanner information with ISO 8601 timestamps
 - **Vulnerabilities**: Detailed vulnerability data with:
   - Unique deterministic UUIDs for tracking
   - Package location and dependency information
   - Severity levels mapped from Socket's analysis
   - Socket-specific alert types and CVE identifiers
   - Links to Socket.dev for detailed analysis
+- **Dependency files**: Manifest files and their dependencies discovered during the scan
+
+**Schema compatibility:** The v15.0.0 schema is supported across all GitLab versions 12.0+ (both self-hosted and cloud). The report includes the `dependency_files` field, which is required by v15.0.0 and accepted as an optional extra by newer schema versions, ensuring maximum compatibility across GitLab instances.
+
+### Performance Notes
+
+When `--enable-gitlab-security` (or `--enable-json` / `--enable-sarif`) is used with a full scan (non-diff mode), the CLI fetches package and alert data from the scan results to populate the report. This adds time proportional to the number of packages in the scan. Without these output flags, no additional data is fetched and scan performance is unchanged.
 
 ### Requirements
 
@@ -726,7 +753,9 @@ Socket CLI generates reports compliant with [GitLab Dependency Scanning schema v
 - Ensure the report file follows the correct schema format
 
 **Empty vulnerabilities array:**
-- This is normal if no new security issues were detected
+- The GitLab report includes both new and existing alerts, so repeated scans of the same repo should still populate the report as long as Socket detects actionable issues
+- If the report is empty, verify the Socket dashboard shows alerts for the scanned packages -- an empty report means no error/warn-level alerts exist
+- For full scans (non-diff mode), ensure you are using `--enable-gitlab-security` so alert data is fetched
 - Check Socket.dev dashboard for full analysis details
 
 ## Development

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketsecurity"
-version = "2.2.80"
+version = "2.2.81"
 requires-python = ">= 3.11"
 license = {"file" = "LICENSE"}
 dependencies = [

@@ -1,3 +1,3 @@
 __author__ = 'socket.dev'
-__version__ = '2.2.80'
+__version__ = '2.2.81'
 USER_AGENT = f'SocketPythonCLI/{__version__}'