-
-
Notifications
You must be signed in to change notification settings - Fork 276
Migrate Gentoo importer to advisory V2 #2090
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
1f3ad4d
f3bec2a
ea7434f
f416c5d
002300d
5ef6840
a5e7bd4
df9077b
b9c28b8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| # | ||
| # Copyright (c) nexB Inc. and others. All rights reserved. | ||
| # VulnerableCode is a trademark of nexB Inc. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. | ||
| # See https://github.com/aboutcode-org/vulnerablecode for support or download. | ||
| # See https://aboutcode.org for more information about nexB OSS projects. | ||
| # | ||
|
|
||
| import re | ||
| import xml.etree.ElementTree as ET | ||
| from pathlib import Path | ||
| from typing import Iterable | ||
|
|
||
| from fetchcode.vcs import fetch_via_vcs | ||
| from packageurl import PackageURL | ||
| from univers.version_constraint import VersionConstraint | ||
| from univers.version_range import EbuildVersionRange | ||
| from univers.versions import GentooVersion | ||
| from univers.versions import InvalidVersion | ||
|
|
||
| from vulnerabilities.importer import AdvisoryData | ||
| from vulnerabilities.importer import AffectedPackageV2 | ||
| from vulnerabilities.importer import ReferenceV2 | ||
| from vulnerabilities.importer import VulnerabilitySeverity | ||
| from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 | ||
| from vulnerabilities.severity_systems import GENERIC | ||
|
|
||
|
|
||
| class GentooImporterPipeline(VulnerableCodeBaseImporterPipelineV2): | ||
| repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git" | ||
| spdx_license_expression = "CC-BY-SA-4.0" | ||
| # the license notice is at this url https://anongit.gentoo.org/ says: | ||
| # The contents of this document, unless otherwise expressly stated, are licensed | ||
| # under the [CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/) license. | ||
| license_url = "https://creativecommons.org/licenses/by-sa/4.0/" | ||
| pipeline_id = "gentoo_importer_v2" | ||
|
|
||
| @classmethod | ||
| def steps(cls): | ||
| return ( | ||
| cls.clone, | ||
| cls.collect_and_store_advisories, | ||
| cls.clean_downloads, | ||
| ) | ||
|
|
||
| def clone(self): | ||
| self.log(f"Cloning `{self.repo_url}`") | ||
| self.vcs_response = fetch_via_vcs(self.repo_url) | ||
|
|
||
| def advisories_count(self): | ||
| advisory_dir = Path(self.vcs_response.dest_dir) | ||
| return sum(1 for _ in advisory_dir.rglob("*.xml")) | ||
|
|
||
| def collect_advisories(self) -> Iterable[AdvisoryData]: | ||
| base_path = Path(self.vcs_response.dest_dir) | ||
| for file_path in base_path.glob("**/*.xml"): | ||
| yield from self.process_file(file_path) | ||
|
|
||
| def process_file(self, file): | ||
| cves = [] | ||
| summary = "" | ||
| xml_root = ET.parse(file).getroot() | ||
| id = xml_root.attrib.get("id", "") | ||
| glsa = "GLSA-" + id | ||
| vuln_references = [ | ||
| ReferenceV2( | ||
| reference_id=glsa, | ||
| url=f"https://security.gentoo.org/glsa/{id}", | ||
| ) | ||
| ] | ||
|
|
||
| severities = [] | ||
| affected_packages = [] | ||
| for child in xml_root: | ||
| if child.tag == "references": | ||
| cves = self.cves_from_reference(child) | ||
|
|
||
| if child.tag == "synopsis": | ||
| summary = child.text | ||
|
|
||
| if child.tag == "affected": | ||
| affected_packages = [] | ||
| seen_packages = set() | ||
|
|
||
| for purl, constraint in get_affected_and_safe_purls(child, logger=self.log): | ||
| signature = (purl.to_string(), str(constraint)) | ||
|
|
||
| if signature not in seen_packages: | ||
| seen_packages.add(signature) | ||
|
|
||
| affected_package = AffectedPackageV2( | ||
| package=purl, | ||
| affected_version_range=EbuildVersionRange(constraints=[constraint]), | ||
| fixed_version_range=None, | ||
| ) | ||
| affected_packages.append(affected_package) | ||
|
|
||
| if child.tag == "impact": | ||
| severity_value = child.attrib.get("type") | ||
| if severity_value: | ||
| severities.append(VulnerabilitySeverity(system=GENERIC, value=severity_value)) | ||
|
|
||
| yield AdvisoryData( | ||
| advisory_id=glsa, | ||
| aliases=cves, | ||
| summary=summary, | ||
| references_v2=vuln_references, | ||
| severities=severities, | ||
| affected_packages=affected_packages, | ||
| url=f"https://security.gentoo.org/glsa/{id}", | ||
| original_advisory_text=file, | ||
| ) | ||
|
|
||
| def clean_downloads(self): | ||
| if self.vcs_response: | ||
| self.log("Removing cloned repository") | ||
| self.vcs_response.delete() | ||
|
|
||
| def on_failure(self): | ||
| self.clean_downloads() | ||
|
|
||
| @staticmethod | ||
| def cves_from_reference(reference): | ||
| cves = [] | ||
| for child in reference: | ||
| txt = child.text.strip() | ||
| match = re.match(r"CVE-\d{4}-\d{4,}", txt) | ||
| if match: | ||
| cves.append(match.group()) | ||
| return cves | ||
|
|
||
|
|
||
| def extract_purls_and_constraints(pkg_name, pkg_ns, constraints, invert, logger): | ||
| for comparator, version, slot_value in constraints: | ||
| qualifiers = {"slot": slot_value} if slot_value else {} | ||
| purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns, qualifiers=qualifiers) | ||
|
|
||
| try: | ||
| constraint = VersionConstraint(version=GentooVersion(version), comparator=comparator) | ||
|
|
||
| if invert: | ||
| constraint = constraint.invert() | ||
|
Comment on lines
+142
to
+143
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @ziadhany why do we invert fixed range, we should report fixed range as is.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @keshav-space we have two type of package version
we invert the the main question is that does the
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
@ziadhany Yes, unaffected is fixed range see resolution section here https://security.gentoo.org/glsa/201405-01 so if we get unaffected range it should be treated as fixed range. Also
similarly for this advisory https://security.gentoo.org/glsa/202004-13 we have these unaffected range <unaffected range="rge">2.23.3</unaffected>
<unaffected range="rge">2.24.3</unaffected>
<unaffected range="rge">2.25.4</unaffected>
<unaffected range="rge">2.26.2</unaffected>and these would be interpreted as fixed range |
||
|
|
||
| yield purl, constraint | ||
| except InvalidVersion as e: | ||
| logger(f"InvalidVersion constraints version: {version} error:{e}") | ||
|
|
||
|
|
||
| def get_affected_and_safe_purls(affected_elem, logger): | ||
| for pkg in affected_elem: | ||
| name = pkg.attrib.get("name") | ||
| if not name: | ||
| continue | ||
| pkg_ns, _, pkg_name = name.rpartition("/") | ||
|
|
||
| safe_constraints, affected_constraints = get_safe_and_affected_constraints(pkg) | ||
|
|
||
| yield from extract_purls_and_constraints( | ||
| pkg_name, pkg_ns, affected_constraints, invert=False, logger=logger | ||
| ) | ||
| yield from extract_purls_and_constraints( | ||
| pkg_name, pkg_ns, safe_constraints, invert=True, logger=logger | ||
| ) | ||
|
|
||
|
|
||
| def get_safe_and_affected_constraints(pkg): | ||
| safe_versions = set() | ||
| affected_versions = set() | ||
| for info in pkg: | ||
| # All possible values of info.attrib['range'] = | ||
| # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'} | ||
| range_value = info.attrib.get("range") | ||
| slot_value = info.attrib.get("slot") | ||
| comparator_dict = { | ||
| "gt": ">", | ||
| "lt": "<", | ||
| "ge": ">=", | ||
| "le": "<=", | ||
| "eq": "=", | ||
| "rle": "<=", | ||
| "rge": ">=", | ||
| "rgt": ">", | ||
| } | ||
| comparator = comparator_dict.get(range_value) | ||
| if info.tag == "unaffected": | ||
| safe_versions.add((comparator, info.text, slot_value)) | ||
|
|
||
| elif info.tag == "vulnerable": | ||
| affected_versions.add((comparator, info.text, slot_value)) | ||
| return safe_versions, affected_versions | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| # | ||
| # Copyright (c) nexB Inc. and others. All rights reserved. | ||
| # VulnerableCode is a trademark of nexB Inc. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. | ||
| # See https://github.com/aboutcode-org/vulnerablecode for support or download. | ||
| # See https://aboutcode.org for more information about nexB OSS projects. | ||
| # | ||
| import json | ||
| from pathlib import Path | ||
| from unittest.mock import Mock | ||
| from unittest.mock import patch | ||
|
|
||
| import pytest | ||
|
|
||
| from vulnerabilities.pipelines.v2_importers.gentoo_importer import GentooImporterPipeline | ||
| from vulnerabilities.tests import util_tests | ||
|
|
||
| TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "gentoo_v2" | ||
|
|
||
| TEST_CVE_FILES = [ | ||
| TEST_DATA / "glsa-201709-09.xml", | ||
| TEST_DATA / "glsa-202511-02.xml", | ||
| TEST_DATA / "glsa-202512-01.xml", | ||
| ] | ||
|
|
||
|
|
||
| @pytest.mark.django_db | ||
| @pytest.mark.parametrize("xml_file", TEST_CVE_FILES) | ||
| def test_gentoo_advisories_per_file(xml_file): | ||
| pipeline = GentooImporterPipeline() | ||
| pipeline.vcs_response = Mock(dest_dir=TEST_DATA) | ||
|
|
||
| with patch.object(Path, "glob", return_value=[xml_file]): | ||
| results = [adv.to_dict() for adv in pipeline.collect_advisories()] | ||
|
|
||
| for adv in results: | ||
| adv["affected_packages"].sort(key=lambda x: json.dumps(x, sort_keys=True)) | ||
|
|
||
| expected_file = xml_file.with_name(xml_file.stem + "-expected.json") | ||
| util_tests.check_results_against_json(results, expected_file) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| [ | ||
| { | ||
| "advisory_id": "GLSA-201709-09", | ||
| "aliases": [ | ||
| "CVE-2017-9800" | ||
| ], | ||
| "summary": "A command injection vulnerability in Subversion may allow remote\n attackers to execute arbitrary code.", | ||
| "affected_packages": [ | ||
| { | ||
| "package": { | ||
| "type": "ebuild", | ||
| "namespace": "dev-vcs", | ||
| "name": "subversion", | ||
| "version": "", | ||
| "qualifiers": "", | ||
| "subpath": "" | ||
| }, | ||
| "affected_version_range": "vers:ebuild/0.1.1", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| }, | ||
| { | ||
| "package": { | ||
| "type": "ebuild", | ||
| "namespace": "dev-vcs", | ||
| "name": "subversion", | ||
| "version": "", | ||
| "qualifiers": "", | ||
| "subpath": "" | ||
| }, | ||
| "affected_version_range": "vers:ebuild/<1.9.7", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| }, | ||
| { | ||
| "package": { | ||
| "type": "ebuild", | ||
| "namespace": "dev-vcs", | ||
| "name": "subversion", | ||
| "version": "", | ||
| "qualifiers": "", | ||
| "subpath": "" | ||
| }, | ||
| "affected_version_range": "vers:ebuild/<=1.8.18", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| } | ||
|
Comment on lines
+37
to
+50
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not right, for the |
||
| ], | ||
| "references_v2": [ | ||
| { | ||
| "reference_id": "GLSA-201709-09", | ||
| "reference_type": "", | ||
| "url": "https://security.gentoo.org/glsa/201709-09" | ||
| } | ||
| ], | ||
| "patches": [], | ||
| "severities": [ | ||
| { | ||
| "system": "generic_textual", | ||
| "value": "normal", | ||
| "scoring_elements": "" | ||
| } | ||
| ], | ||
| "date_published": null, | ||
| "weaknesses": [], | ||
| "url": "https://security.gentoo.org/glsa/201709-09" | ||
| } | ||
| ] | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It does not make sense to report
unaffectedrange inaffected_version_range.