Detect versions reported as both affected and fixed

vulnerabilities/pipelines/v2_improvers/compute_advisory_todo.py

@@ -43,6 +43,7 @@ class ComputeToDo(VulnerableCodePipeline):
     def steps(cls):
         return (
             cls.compute_individual_advisory_todo,
+            cls.detect_inconsistent_package_versions,
             cls.detect_conflicting_package_versions,
             cls.detect_conflicting_cvss_scores,
             cls.detect_conflicting_weakness,
@@ -112,6 +113,60 @@ def compute_individual_advisory_todo(self):
             f"Successfully created {new_todos_count} ToDos for missing summary, affected and fixed packages"
         )
 
+    def detect_inconsistent_package_versions(self):
+        """Create ToDos for versions reported as both affected and fixed in one advisory."""
+        advisories = (
+            AdvisoryV2.objects.todo_excluded()
+            .latest_per_avid()
+            .exclude(
+                advisory_todos__in=AdvisoryToDoV2.objects.filter(
+                    issue_type="CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES",
+                    advisories_count=1,
+                )
+            )
+            .prefetch_related(
+                "impacted_packages",
+                "impacted_packages__affecting_packages",
+                "impacted_packages__fixed_by_packages",
+            )
+        )
+        advisories_count = advisories.count()
+        advisory_relation_to_create = {}
+        todo_to_create = []
+        new_todos_count = 0
+        batch_size = 1000
+
+        self.log(
+            f"Checking inconsistent affected and fixed versions in {advisories_count} Advisories"
+        )
+        progress = LoopProgress(
+            total_iterations=advisories_count,
+            logger=self.log,
+            progress_step=10,
+        )
+        for advisory in progress.iter(advisories.iterator(chunk_size=500)):
+            check_inconsistent_affected_and_fixed_by_packages(
+                advisory=advisory,
+                todo_to_create=todo_to_create,
+                advisory_relation_to_create=advisory_relation_to_create,
+            )
+
+            if len(todo_to_create) >= batch_size:
+                new_todos_count += bulk_create_with_m2m(
+                    todos=todo_to_create,
+                    advisories=advisory_relation_to_create,
+                    logger=self.log,
+                )
+                advisory_relation_to_create.clear()
+                todo_to_create.clear()
+
+        new_todos_count += bulk_create_with_m2m(
+            todos=todo_to_create,
+            advisories=advisory_relation_to_create,
+            logger=self.log,
+        )
+        self.log(f"Successfully created {new_todos_count} ToDos for inconsistent package versions")
+
     def detect_conflicting_package_versions(self):
         """
         Create ToDos for advisories with conflicting opinions on fixed and affected
@@ -908,6 +963,78 @@ def check_missing_affected_and_fixed_by_packages(
         advisory_relation_to_create[todo_id] = [advisory]
 
 
+def check_inconsistent_affected_and_fixed_by_packages(
+    advisory,
+    todo_to_create,
+    advisory_relation_to_create,
+):
+    """
+    Flag overlapping affected and fixed versions, including across impacts for the same PURL.
+
+    Known package relationships can establish a conflict before range unfurling completes.
+    Leave conflicting versions out of partial curation so a human must decide their status.
+    """
+    packages_by_purl = defaultdict(lambda: {"affected": set(), "fixed": set()})
+    for impact in advisory.impacted_packages.all():
+        packages = packages_by_purl[impact.base_purl]
+        packages["affected"].update(p.version for p in impact.affecting_packages.all())
+        packages["fixed"].update(p.version for p in impact.fixed_by_packages.all())
+
+    conflict_details = {}
+    curation_items = []
+    for purl, packages in sorted(packages_by_purl.items()):
+        affected = packages["affected"]
+        fixed = packages["fixed"]
+        conflicting_versions = sorted(affected & fixed)
+        if not conflicting_versions:
+            continue
+
+        detail = {
+            "avids": [advisory.avid],
+            "conflicting_versions": conflicting_versions,
+            "affected_union": sorted(affected),
+            "fixed_union": sorted(fixed),
+            "affected_intersection": sorted(affected - fixed),
+            "fixed_intersection": sorted(fixed - affected),
+            "affected_disagreement": conflicting_versions,
+            "fixed_disagreement": conflicting_versions,
+        }
+        conflict_details[purl] = detail
+        curation_item = get_grouped_curation_advisories_for_dashboard_ui(
+            purl=purl,
+            adv_map={advisory.avid: packages},
+            conflict_detail=detail,
+            advisories={advisory.avid: advisory},
+        )
+        curation_item["conflict_reason"] = (
+            "Advisory reports the following version(s) as both affected and fixed: "
+            f"{', '.join(conflicting_versions)}."
+        )
+        curation_items.append(curation_item)
+
+    if not conflict_details:
+        return
+
+    todo_id = advisories_checksum(advisory)
+    alias = advisory.advisory_id.rsplit("/", 1)[-1]
+    todo_to_create.append(
+        AdvisoryToDoV2(
+            related_advisories_id=todo_id,
+            issue_type="CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES",
+            issue_detail={
+                "alias": alias,
+                "conflict_checksum": sha256_digest(canonical_value(conflict_details)),
+                "conflict_details": conflict_details,
+                "curation_items": curation_items,
+            },
+            alias=alias,
+            advisories_count=1,
+            oldest_advisory_date=advisory.date_published or advisory.date_collected,
+        )
+    )
+    advisory_relation_to_create[todo_id] = [advisory]
+
+
 def compute_version_range_disagreement(adv_map):
     """Compute differences in affected and fixed version sets across advisories."""
 
@@ -1092,12 +1219,14 @@ def get_grouped_curation_advisories_for_dashboard_ui(purl, adv_map, conflict_det
         "advisories": [],
     }
 
-    all_versions = conflict_detail["affected_union"] + conflict_detail["fixed_union"]
+    all_versions = set(conflict_detail["affected_union"] + conflict_detail["fixed_union"])
     package_url = PackageURL.from_string(purl)
-    range_class = RANGE_CLASS_BY_SCHEMES[package_url.type]
-    version_class = range_class.version_class
-    sorted_versions = sorted([version_class(v) for v in all_versions])
-    curation_item["all_versions"] = [str(v) for v in sorted_versions]
+    range_class = RANGE_CLASS_BY_SCHEMES.get(package_url.type)
+    if range_class:
+        sorted_versions = sorted(range_class.version_class(v) for v in all_versions)
+        curation_item["all_versions"] = [str(v) for v in sorted_versions]
+    else:
+        curation_item["all_versions"] = sorted(all_versions)
     curation_item["conflict_reason"] = get_disagreement_message(
         fixed_disagreement=conflict_detail["fixed_disagreement"],
         affected_disagreement=conflict_detail["affected_disagreement"],

vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_todo_v2.py

@@ -9,6 +9,8 @@
 
 from collections.abc import Mapping
 from datetime import datetime
+from pathlib import Path
+from unittest.mock import patch
 
 from django.test import TestCase
 from django.utils import timezone
@@ -23,10 +25,13 @@
 from vulnerabilities.models import AdvisoryToDoV2
 from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.models import ToDoRelatedAdvisoryV2
 from vulnerabilities.pipelines.v2_improvers.compute_advisory_todo import ComputeToDo
+from vulnerabilities.pipes.advisory import advisories_checksum
 from vulnerabilities.pipes.advisory import insert_advisory_v2
 from vulnerabilities.tests.pipelines import TestLogger
 from vulnerabilities.utils import canonical_value
+from vulnerabilities.utils import load_json
 
 
 class TestComputeToDo(TestCase):
@@ -762,6 +767,185 @@ def test_todo_conflicting_severity_with_no_common_cvss(self):
         self.assertEqual(0, AdvisoryToDoV2.objects.count())
 
 
+class TestInconsistentPackageVersions(TestCase):
+    def setUp(self):
+        fixture = Path(__file__).parents[2] / "test_data/compute_advisory_todo/npm-92.json"
+        self.advisory_data = AdvisoryDataV2.from_dict(load_json(fixture))
+        self.log = TestLogger()
+
+    def insert_advisory(self):
+        return insert_advisory_v2(
+            advisory=self.advisory_data,
+            pipeline_id="test_pipeline",
+            datasource_id="test_npm",
+            logger=self.log.write,
+        )
+
+    def test_creates_todo_for_affected_and_fixed_version_in_one_advisory(self):
+        advisory = self.insert_advisory()
+        impact = advisory.impacted_packages.get()
+        self.assertTrue(impact.affecting_packages.filter(version="3.0.0").exists())
+        self.assertTrue(impact.fixed_by_packages.filter(version="3.0.0").exists())
+        self.assertFalse(advisory.aliases.exists())
+
+        ComputeToDo().execute()
+
+        todo = AdvisoryToDoV2.objects.get()
+        self.assertEqual("CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES", todo.issue_type)
+        self.assertEqual([advisory], list(todo.advisories.all()))
+        self.assertEqual(1, todo.advisories_count)
+        self.assertEqual("92", todo.alias)
+        self.assertEqual(advisory.date_published, todo.oldest_advisory_date)
+        details = todo.issue_detail["conflict_details"]
+        self.assertEqual(["3.0.0"], details[impact.base_purl]["conflicting_versions"])
+        (item,) = todo.issue_detail["curation_items"]
+        self.assertEqual(impact.base_purl, item["purl"])
+        self.assertIn("both affected and fixed: 3.0.0", item["conflict_reason"])
+        self.assertEqual(1, item["all_versions"].count("3.0.0"))
+        self.assertNotIn("3.0.0", item["partial_curation"]["affected"])
+        self.assertNotIn("3.0.0", item["partial_curation"]["fixing"])
+        self.assertEqual(advisory.avid, item["advisories"][0]["primary"]["advisory_uid"])
+        impact.refresh_from_db()
+        self.assertTrue(impact.affecting_packages.filter(version="3.0.0").exists())
+        self.assertTrue(impact.fixed_by_packages.filter(version="3.0.0").exists())
+
+    def test_distinct_affected_and_fixed_versions_do_not_create_todo(self):
+        self.advisory_data.affected_packages[0].fixed_version_range = VersionRange.from_string(
+            "vers:npm/3.1.0"
+        )
+        self.insert_advisory()
+
+        ComputeToDo().execute()
+
+        self.assertFalse(AdvisoryToDoV2.objects.exists())
+
+    def test_same_version_for_different_packages_does_not_create_todo(self):
+        self.advisory_data.affected_packages = [
+            AffectedPackageV2(
+                package=PackageURL(type="npm", name="express-restify-mongoose"),
+                affected_version_range=VersionRange.from_string("vers:npm/3.0.0"),
+            ),
+            AffectedPackageV2(
+                package=PackageURL(type="npm", name="another-package"),
+                fixed_version_range=VersionRange.from_string("vers:npm/3.0.0"),
+            ),
+        ]
+        self.insert_advisory()
+
+        ComputeToDo().execute()
+
+        self.assertFalse(AdvisoryToDoV2.objects.exists())
+
+    def test_detects_conflict_across_impacts_for_the_same_package(self):
+        package = PackageURL(type="npm", name="express-restify-mongoose")
+        self.advisory_data.affected_packages = [
+            AffectedPackageV2(
+                package=package,
+                affected_version_range=VersionRange.from_string("vers:npm/3.0.0"),
+            ),
+            AffectedPackageV2(
+                package=package,
+                fixed_version_range=VersionRange.from_string("vers:npm/3.0.0"),
+            ),
+        ]
+        self.insert_advisory()
+
+        ComputeToDo().execute()
+
+        todo = AdvisoryToDoV2.objects.get()
+        self.assertEqual(
+            ["3.0.0"],
+            todo.issue_detail["conflict_details"][str(package)]["conflicting_versions"],
+        )
+
+    def test_repeated_pipeline_runs_do_not_duplicate_todo_or_relations(self):
+        self.insert_advisory()
+        ComputeToDo().execute()
+        todo = AdvisoryToDoV2.objects.get()
+        original_detail = todo.issue_detail
+        created_at = todo.created_at
+
+        ComputeToDo().execute()
+        self.assertEqual(1, AdvisoryToDoV2.objects.count())
+        self.assertEqual(1, ToDoRelatedAdvisoryV2.objects.count())
+
+        todo.is_resolved = True
+        todo.save()
+        ComputeToDo().execute()
+
+        self.assertEqual(1, AdvisoryToDoV2.objects.count())
+        self.assertEqual(1, ToDoRelatedAdvisoryV2.objects.count())
+        todo.refresh_from_db()
+        self.assertTrue(todo.is_resolved)
+        self.assertEqual(original_detail, todo.issue_detail)
+        self.assertEqual(created_at, todo.created_at)
+
+    def test_other_todo_does_not_hide_version_conflict(self):
+        self.advisory_data.summary = ""
+        advisory = self.insert_advisory()
+        advisory._all_impacts_unfurled_at = timezone.now()
+        advisory.save()
+
+        ComputeToDo().execute()
+
+        self.assertEqual(
+            {"MISSING_SUMMARY", "CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES"},
+            set(advisory.advisory_todos.values_list("issue_type", flat=True)),
+        )
+
+    def test_non_latest_advisory_does_not_create_todo(self):
+        advisory = self.insert_advisory()
+        advisory.is_latest = False
+        advisory.save()
+
+        ComputeToDo().execute()
+
+        self.assertFalse(AdvisoryToDoV2.objects.exists())
+
+    @patch("vulnerabilities.importers.TODO_EXCLUDED_PIPELINES", ["test_pipeline"])
+    def test_excluded_pipeline_does_not_create_todo(self):
+        self.insert_advisory()
+
+        ComputeToDo().execute()
+
+        self.assertFalse(AdvisoryToDoV2.objects.exists())
+
+    def test_group_todo_does_not_hide_conflicts_within_individual_advisories(self):
+        advisory = self.insert_advisory()
+        self.advisory_data.advisory_id = "93"
+        other_advisory = self.insert_advisory()
+        group_todo = AdvisoryToDoV2.objects.create(
+            related_advisories_id=advisories_checksum([advisory, other_advisory]),
+            issue_type="CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES",
+            advisories_count=2,
+            alias="test_alias",
+        )
+        group_todo.advisories.add(advisory, other_advisory)
+
+        ComputeToDo().execute()
+
+        self.assertEqual(3, AdvisoryToDoV2.objects.count())
+        self.assertEqual(2, AdvisoryToDoV2.objects.filter(advisories_count=1).count())
+        self.assertEqual(2, group_todo.advisories.count())
+
+    def test_package_type_without_range_scheme_still_creates_todo(self):
+        self.advisory_data.affected_packages = [
+            AffectedPackageV2(
+                package=PackageURL(type="docker", name="example"),
+                affected_version_range=VersionRange.from_string("vers:generic/3.0.0"),
+                fixed_version_range=VersionRange.from_string("vers:generic/3.0.0"),
+            )
+        ]
+        self.insert_advisory()
+
+        ComputeToDo().execute()
+
+        todo = AdvisoryToDoV2.objects.get()
+        (item,) = todo.issue_detail["curation_items"]
+        self.assertEqual("pkg:docker/example", item["purl"])
+        self.assertEqual(["3.0.0"], item["all_versions"])
+
+
 def normalize(obj):
     if isinstance(obj, Mapping):
         return {k: normalize(v) for k, v in sorted(obj.items())}