Flag advisory commits not reachable from repository branches

docs/source/reference_improver_overview.rst

@@ -32,3 +32,31 @@ like *Time Travel* and *finding fix commits* could be implemented as well.
 
 You can find more in-code documentation about improvers in :file:`vulnerabilites/improver.py` and
 the framework responsible for invoking these improvers in :file:`vulnerabilites/improve_runner.py`.
+
+Advisory commit branch reachability
+-----------------------------------
+
+The ``collect_ref_fix_commits_v2`` pipeline collects commit references and then checks all
+``PackageCommitPatch`` records, including commits collected directly by importers. Commits
+and their advisory links are retained regardless of the check's result.
+
+The ``PackageCommitPatch.branch_reachability`` field records one of three outcomes:
+
+- ``reachable``: the commit is reachable from a branch in the referenced repository.
+- ``unreachable``: a complete snapshot of all branch histories does not contain the commit.
+  A commit reachable only from a tag, a detached HEAD, or a pull request ref does not count
+  as belonging to a repository branch.
+- ``unknown``: the commit has not been checked, the repository could not be cloned, or the
+  clone has incomplete history. Shallow and partial clones cannot establish unreachability.
+
+The check creates one temporary full bare clone per repository per run and removes it after
+use. Clones include all branches and have a five-minute timeout; large repositories may
+therefore remain unknown. A successful walk of all branch histories can establish
+unreachability even when the referenced commit object is absent from the clone.
+
+The advisory's commit details page displays the result and highlights unreachable commits.
+Pipeline logs include the repository URL and hash for these commits, for upstream review.
+These results describe the branches observed during the last check and may change when
+branches are added, deleted, or rewritten. Rerunning the pipeline refreshes existing records;
+a failed check sets their status to unknown. The flag is local enrichment and is not part of
+the source advisory's serialized patch data.

vulnerabilities/migrations/0145_packagecommitpatch_branch_reachability.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.2.11 on 2026-11-12 16:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0144_remove_advisoryv2_advisory_latest_by_avid_idx_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="packagecommitpatch",
+            name="branch_reachability",
+            field=models.CharField(
+                choices=[
+                    ("unknown", "Branch reachability unknown"),
+                    ("reachable", "Reachable from a repository branch"),
+                    ("unreachable", "Not reachable from any repository branch"),
+                ],
+                default="unknown",
+                help_text="Reachability from repository branches at the last check; unknown if incomplete.",
+                max_length=11,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -2898,6 +2898,11 @@ class PackageCommitPatch(models.Model):
     A specific patch implementation for structured Package/VCS data
     """
 
+    class BranchReachability(models.TextChoices):
+        UNKNOWN = "unknown", "Branch reachability unknown"
+        REACHABLE = "reachable", "Reachable from a repository branch"
+        UNREACHABLE = "unreachable", "Not reachable from any repository branch"
+
     commit_hash = models.CharField(
         max_length=128,
         help_text="The commit hash of the patch in the VCS.",
@@ -2908,6 +2913,13 @@ class PackageCommitPatch(models.Model):
         help_text="The Version Control System URL (e.g., git repo URL).",
     )
 
+    branch_reachability = models.CharField(
+        max_length=11,
+        choices=BranchReachability.choices,
+        default=BranchReachability.UNKNOWN,
+        help_text="Reachability from repository branches at the last check; unknown if incomplete.",
+    )
+
     patch_text = models.TextField(blank=True, null=True)
     patch_checksum = models.CharField(max_length=128, blank=True, null=True)
 

vulnerabilities/pipelines/v2_improvers/reference_collect_commits.py

@@ -6,11 +6,13 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from collections import defaultdict
+from tempfile import TemporaryDirectory
 
 from aboutcode.federated import get_core_purl
 from aboutcode.pipeline import LoopProgress
 from django.db.models import Prefetch
+from git import Repo
+from git.exc import GitError
 from packageurl.contrib.purl2url import purl2url
 from packageurl.contrib.url2purl import url2purl
 
@@ -20,6 +22,7 @@
 from vulnerabilities.models import PackageCommitPatch
 from vulnerabilities.models import Patch
 from vulnerabilities.pipelines import VulnerableCodePipeline
+from vulnerabilities.pipes.commit_reachability import get_branch_reachability
 from vulnerabilities.utils import is_commit
 
 
@@ -32,14 +35,60 @@ class CollectReferencesFixCommitsPipeline(VulnerableCodePipeline):
 
     @classmethod
     def steps(cls):
-        return (cls.collect_and_store_fix_commits,)
+        return (cls.collect_and_store_fix_commits, cls.check_commit_branch_reachability)
+
+    def check_commit_branch_reachability(self):
+        """Check all collected commits, cloning each repository once per run.
+
+        Collection remains independent of repository availability. A fresh, full
+        bare clone includes all branch heads and their history, so even an object
+        absent from that clone can be reported as unreachable from its branches.
+        """
+        vcs_urls = (
+            PackageCommitPatch.objects.order_by("vcs_url")
+            .values_list("vcs_url", flat=True)
+            .distinct()
+        )
+        for vcs_url in vcs_urls.iterator():
+            commits = list(
+                PackageCommitPatch.objects.filter(vcs_url=vcs_url).only("id", "commit_hash")
+            )
+            results = dict.fromkeys(
+                (commit.commit_hash for commit in commits),
+                PackageCommitPatch.BranchReachability.UNKNOWN,
+            )
+            try:
+                with TemporaryDirectory(prefix="vcio-commit-branches-") as directory:
+                    with Repo.clone_from(
+                        vcs_url,
+                        directory,
+                        bare=True,
+                        single_branch=False,
+                        no_local=True,
+                        no_tags=True,
+                        env={"GIT_TERMINAL_PROMPT": "0"},
+                        kill_after_timeout=300,
+                    ) as repo:
+                        results = get_branch_reachability(repo, results, all_branches_fetched=True)
+            except (GitError, OSError, ValueError) as error:
+                self.log(f"Branch reachability unknown for {vcs_url}: {error}")
+
+            for commit in commits:
+                commit.branch_reachability = results[commit.commit_hash]
+                if commit.branch_reachability == PackageCommitPatch.BranchReachability.UNREACHABLE:
+                    self.log(
+                        "Commit not reachable from repository branches: "
+                        f"{vcs_url}@{commit.commit_hash}"
+                    )
+
+            PackageCommitPatch.objects.bulk_update(
+                commits, fields=["branch_reachability"], batch_size=10000
+            )
 
     def get_vcs_data(self, url):
-        """Extracts a VCS URL and commit hash from URL.
-        >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/commit/98e516011d6e096e25247b82fc5f196bbeecff10')
-        ("pkg:github/aboutcode-org/vulnerablecode", 'https://github.com/aboutcode-org/vulnerablecode', '98e516011d6e096e25247b82fc5f196bbeecff10')
-        >> get_vcs_commit('https://github.com/aboutcode-org/vulnerablecode/pull/1974')
-        None
+        """Extract a core purl, VCS URL, and commit hash from a commit URL.
+
+        Return None for non-commit URLs, including pull requests.
         """
         try:
             purl = url2purl(url)
@@ -54,7 +103,7 @@ def get_vcs_data(self, url):
             if base_purl and vcs_url and version:
                 return base_purl, vcs_url, version
         except Exception as e:
-            self.log(f"Invalid URL: url:{url} error:{e}")
+            self.log(f"Invalid URL: url: {url} error: {e}")
 
     def collect_and_store_fix_commits(self):
         advisories = AdvisoryV2.objects.only("id").prefetch_related(
@@ -85,7 +134,8 @@ def collect_and_store_fix_commits(self):
             updated_pkg_patch_commit_count += self.bulk_commit_batch_update(commit_batch)
             commit_batch.clear()
 
-        self.log(f"Successfully processed pkg patch commit {updated_pkg_patch_commit_count:,d}")
+        processed_count = format(updated_pkg_patch_commit_count, ",d")
+        self.log(f"Successfully processed pkg patch commit {processed_count}")
 
     def bulk_commit_batch_update(self, vcs_data_table):
         impact_data = {(row[0], row[3]) for row in vcs_data_table}  # base_purl, adv_id

vulnerabilities/pipes/commit_reachability.py

@@ -0,0 +1,49 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from vulnerabilities.models import PackageCommitPatch
+
+
+def get_branch_reachability(repo, commit_hashes, *, all_branches_fetched=False):
+    """Return reachability for commits against a complete snapshot of branch heads.
+
+    The caller must establish that all repository branches were fetched. Shallow
+    and partial clones cannot prove absence and yield unknown results. Tags,
+    detached HEADs, and pull request refs do not count as repository branches.
+    Git errors propagate so the caller can report unknown and retry later.
+    """
+    status = PackageCommitPatch.BranchReachability
+    results = dict.fromkeys(commit_hashes, status.UNKNOWN)
+    if not all_branches_fetched or repo.git.rev_parse("--is-shallow-repository") == "true":
+        return results
+
+    with repo.config_reader() as config:
+        if config.has_option("extensions", "partialclone"):
+            return results
+        for section in config.sections():
+            if section.startswith('remote "') and config.get_value(section, "promisor", False):
+                return results
+
+    # Traverse branch histories once, without fetching objects mentioned only in
+    # advisories. A missing object is unreachable only after this walk succeeds.
+    reachable = set(repo.git.rev_list("--branches", "--missing=error").splitlines())
+    for commit_hash in results:
+        normalized = commit_hash.lower()
+        if len(normalized) in (40, 64):
+            results[commit_hash] = (
+                status.REACHABLE if normalized in reachable else status.UNREACHABLE
+            )
+        else:
+            matches = sum(commit.startswith(normalized) for commit in reachable)
+            if matches == 1:
+                results[commit_hash] = status.REACHABLE
+            elif matches == 0:
+                results[commit_hash] = status.UNREACHABLE
+            # An ambiguous abbreviated hash remains unknown.
+    return results

vulnerabilities/templates/advisory_package_commit_details.html

@@ -37,8 +37,11 @@
                             <tr>
                                 <td>
                                     <a href="{{ pkg_commit_patch.vcs_url }}" target="_self">
-                                        {{ pkg_commit_patch.base_purl }}@{{ pkg_commit_patch.commit_hash }}
+                                        {{ impact.base_purl }}@{{ pkg_commit_patch.commit_hash }}
                                     </a>
+                                    <span class="tag {% if pkg_commit_patch.branch_reachability == 'unreachable' %}is-warning{% endif %}">
+                                        {{ pkg_commit_patch.get_branch_reachability_display }}
+                                    </span>
                                 </td>
                                 <td></td>
                             </tr>
@@ -51,6 +54,9 @@
                                     <a href="{{ pkg_commit_patch.vcs_url }}" target="_self">
                                         {{ impact.base_purl }}@{{ pkg_commit_patch.commit_hash }}
                                     </a>
+                                    <span class="tag {% if pkg_commit_patch.branch_reachability == 'unreachable' %}is-warning{% endif %}">
+                                        {{ pkg_commit_patch.get_branch_reachability_display }}
+                                    </span>
                                 </td>
                             </tr>
                         {% endfor %}
@@ -72,4 +78,4 @@
 
 <script src="{% static 'js/main.js' %}" crossorigin="anonymous"></script>
 
-{% endblock %}
+{% endblock %}