From 57c8d58549e799fc0a7c0bc052a34d51f4bccade Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Mar 2026 14:45:08 +0530 Subject: [PATCH 01/65] Change API design Signed-off-by: Tushar Goel --- vulnerabilities/api_v2.py | 569 ++++++++++++++------------------------ vulnerabilities/models.py | 30 ++ vulnerablecode/urls.py | 8 + 3 files changed, 240 insertions(+), 367 deletions(-) diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index 74975b819..8cbd8ccc0 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -8,7 +8,10 @@ # +from urllib.parse import urlencode + from django.db.models import Prefetch +from django.db.models import Q from django_filters import rest_framework as filters from drf_spectacular.utils import OpenApiParameter from drf_spectacular.utils import extend_schema @@ -20,6 +23,7 @@ from rest_framework import viewsets from rest_framework.authentication import SessionAuthentication from rest_framework.decorators import action +from rest_framework.exceptions import MethodNotAllowed from rest_framework.permissions import BasePermission from rest_framework.response import Response from rest_framework.reverse import reverse @@ -141,7 +145,7 @@ def get_aliases(self, obj): return [alias.alias for alias in obj.aliases.all()] -class AdvisoryV2Serializer(serializers.ModelSerializer): +class AdvisoryV3Serializer(serializers.ModelSerializer): aliases = serializers.SerializerMethodField() weaknesses = AdvisoryWeaknessSerializer(many=True) references = AdvisoryReferenceSerializer(many=True) @@ -337,7 +341,9 @@ class PackageV3Serializer(serializers.ModelSerializer): purl = serializers.CharField(source="package_url") risk_score = serializers.FloatField(read_only=True) affected_by_vulnerabilities = serializers.SerializerMethodField() + affected_by_vulnerabilities_url = serializers.SerializerMethodField() fixing_vulnerabilities = serializers.SerializerMethodField() + fixing_vulnerabilities_url = serializers.SerializerMethodField() next_non_vulnerable_version = serializers.SerializerMethodField() latest_non_vulnerable_version = serializers.SerializerMethodField() @@ -346,80 +352,118 @@ class Meta: fields = [ "purl", "affected_by_vulnerabilities", + "affected_by_vulnerabilities_url", "fixing_vulnerabilities", + "fixing_vulnerabilities_url", "next_non_vulnerable_version", "latest_non_vulnerable_version", "risk_score", ] + def to_representation(self, instance): + data = super().to_representation(instance) + + if data.get("affected_by_vulnerabilities") is None: + data.pop("affected_by_vulnerabilities", None) + else: + data.pop("affected_by_vulnerabilities_url", None) + + if data.get("fixing_vulnerabilities") is None: + data.pop("fixing_vulnerabilities", None) + else: + data.pop("fixing_vulnerabilities_url", None) + + return data + + def get_affected_by_vulnerabilities_url(self, obj): + request = self.context.get("request") + if not request: + return None + + base = reverse("affected-by-advisories-list") + url = request.build_absolute_uri(base) + + return f"{url}?{urlencode({'purl': obj.package_url})}" + + def get_fixing_vulnerabilities_url(self, obj): + request = self.context.get("request") + if not request: + return None + + base = reverse("fixing-advisories-list") + url = request.build_absolute_uri(base) + + return f"{url}?{urlencode({'purl': obj.package_url})}" + def get_affected_by_vulnerabilities(self, package): """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" - impacts = package.affected_in_impacts.select_related("advisory").prefetch_related( - "fixed_by_packages" - ) + advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) - avids = {impact.advisory.avid for impact in impacts if impact.advisory_id} + advisories = list(advisories_qs[:101]) + if len(advisories) > 100: + return None - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - advisory_by_avid = {adv.avid: adv for adv in latest_advisories} - impact_by_avid = {} - - advisories = [] - for impact in impacts: - avid = impact.advisory.avid - advisory = advisory_by_avid.get(avid) - if not advisory: - continue - advisories.append(advisory) - impact_by_avid[avid] = impact + advisory_by_avid = {adv.avid: adv for adv in advisories} + avids = advisory_by_avid.keys() - grouped_advisories = group_advisories_by_content(advisories=advisories) + impacts = ( + package.affected_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) - advs = [] + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - for hash in grouped_advisories: - advs.append(grouped_advisories[hash]) + grouped = group_advisories_by_content(advisories) result = [] - - for advisory in advs: - primary_advisory = advisory["primary"] - avid = primary_advisory.avid - impact = impact_by_avid.get(avid) + for entry in grouped.values(): + primary = entry["primary"] + impact = impact_by_avid.get(primary.avid) if not impact: continue + result.append( { - "advisory_id": primary_advisory.avid, + "advisory_id": primary.avid, "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], - "duplicate_advisory_ids": [adv.avid for adv in advisory["secondary"]], + "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], } ) return result def get_fixing_vulnerabilities(self, package): - impacts = package.fixed_in_impacts.select_related("advisory") + advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - avids = {impact.advisory.avid for impact in impacts if impact.advisory_id} + advisories = list(advisories_qs[:101]) + if len(advisories) > 100: + return None - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) + advisory_by_avid = {adv.avid: adv for adv in advisories} + avids = advisory_by_avid.keys() - grouped_advisories = group_advisories_by_content(advisories=latest_advisories) + impacts = ( + package.fixed_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) - advs = [] + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - for hash in grouped_advisories: - advs.append(grouped_advisories[hash]) + grouped = group_advisories_by_content(advisories) result = [] + for entry in grouped.values(): + primary = entry["primary"] + impact = impact_by_avid.get(primary.avid) + if not impact: + continue - for advisory in advs: - primary_advisory = advisory["primary"] result.append( { - "advisory_id": primary_advisory.avid, - "duplicate_advisory_ids": [adv.avid for adv in advisory["secondary"]], + "advisory_id": primary.avid, + "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], } ) @@ -462,27 +506,6 @@ class PackageV2FilterSet(filters.FilterSet): purl = filters.CharFilter(field_name="package_url") -class AdvisoryPackageV2FilterSet(filters.FilterSet): - affected_by_advisory = filters.CharFilter( - field_name="affected_in_impacts__advisory__avid", - label="Affected By Advisory ID", - help_text="Filter packages affected by a specific Advisory ID.", - ) - - fixing_advisory = filters.CharFilter( - field_name="fixed_in_impacts__advisory__avid", - label="Fixed By Advisory ID", - help_text="Filter packages fixed by a specific Advisory ID.", - ) - - purls = CharInFilter( - field_name="package_url", - lookup_expr="in", - label="Package URL", - help_text="Filter by one or more Package URLs. Multi-value supported (comma-separated).", - ) - - class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet): queryset = Package.objects.all().prefetch_related( Prefetch( @@ -1064,337 +1087,149 @@ def get_view_name(self): return "Pipeline Jobs" -class PackageV3ViewSet(viewsets.ReadOnlyModelViewSet): - queryset = PackageV2.objects.all() - serializer_class = PackageV3Serializer - filter_backends = [filters.DjangoFilterBackend] - filterset_class = AdvisoryPackageV2FilterSet - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def get_queryset(self): - return ( - super() - .get_queryset() - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( - "fixed_by_packages", - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - - def list(self, request, *args, **kwargs): - queryset = self.filter_queryset(self.get_queryset()) - page = self.paginate_queryset(queryset) - - packages = page if page is not None else queryset - - avids = set() - - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - - advisory_data = {adv.avid: AdvisoryV2Serializer(adv).data for adv in latest_advisories} - - serializer = self.get_serializer(packages, many=True) - - if page is not None: - return self.get_paginated_response( - { - "packages": serializer.data, - "advisories_by_id": advisory_data, - } - ) - - return Response( - { - "packages": serializer.data, - "advisories_by_id": advisory_data, - } - ) - - @extend_schema( - request=PackageurlListSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageurlListSerializer, - filter_backends=[], - pagination_class=None, +class PackageQuerySerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, ) - def bulk_lookup(self, request): - """ - Return the response for exact PackageURLs requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - - purls = serializer.validated_data.get("purls") - - packages = ( - PackageV2.objects.for_purls(purls) - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( - "fixed_by_packages" - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - - avids = set() - - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) + details = serializers.BooleanField(default=False) + approximate = serializers.BooleanField(default=False) + + def validate(self, data): + if not data["purls"]: + if data["details"] or data["approximate"]: + raise serializers.ValidationError( + "details and approximate must be false when purls is empty" + ) + return data - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) +class AdvisoryQuerySerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, + ) - advisory_data = { - adv.avid: AdvisoryV2Serializer(adv, context={"request": request}).data - for adv in latest_advisories - } + def validate(self, data): + if not data["purls"]: + raise serializers.ValidationError("purls is required") + return data - package_data = PackageV3Serializer( - packages, - many=True, - context={"request": request}, - ).data - return Response( - { - "packages": package_data, - "advisories_by_id": advisory_data, - } - ) +class PackageV3ViewSet(viewsets.GenericViewSet): + queryset = PackageV2.objects.all() + serializer_class = PackageV3Serializer + filter_backends = [filters.DjangoFilterBackend] + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - @extend_schema( - request=PackageBulkSearchRequestSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageBulkSearchRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_search(self, request): - """ - Lookup for vulnerable packages using many Package URLs at once. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, + def create(self, request, *args, **kwargs): + serializer = PackageQuerySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + purls = serializer.validated_data["purls"] + details = serializer.validated_data["details"] + approximate = serializer.validated_data["approximate"] + + if not purls: + vulnerable_purls = ( + PackageV2.objects.vulnerable() + .only("package_url") + .distinct() + .values_list("package_url", flat=True) + .order_by("package_url") ) - - validated_data = serializer.validated_data - purls = validated_data.get("purls") - purl_only = validated_data.get("purl_only", False) - plain_purl = validated_data.get("plain_purl", False) - - if plain_purl: - purl_objects = [PackageURL.from_string(purl) for purl in purls] - plain_purl_objects = [ - PackageURL( - type=purl.type, - namespace=purl.namespace, - name=purl.name, - version=purl.version, + page = self.paginate_queryset(vulnerable_purls) + return self.get_paginated_response(page) + + plain_purls = None + + if approximate: + plain_purls = [ + str( + PackageURL( + type=p.type, + namespace=p.namespace, + name=p.name, + version=p.version, + ) ) - for purl in purl_objects + for p in map(PackageURL.from_string, purls) ] - plain_purls = [str(purl) for purl in plain_purl_objects] + if not details: + if approximate: + query = ( + PackageV2.objects.filter(plain_package_url__in=plain_purls) + .values_list("plain_package_url", flat=True) + .distinct() + .order_by("plain_package_url") + ) + else: + query = ( + PackageV2.objects.filter(package_url__in=purls) + .values_list("package_url", flat=True) + .distinct() + .order_by("package_url") + ) + + page = self.paginate_queryset(query) + return self.get_paginated_response(page) + + if approximate: query = ( PackageV2.objects.filter(plain_package_url__in=plain_purls) .order_by("plain_package_url") .distinct("plain_package_url") - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related( - "advisory" - ).prefetch_related( - "fixed_by_packages", - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() + ) + else: + query = ( + PackageV2.objects.filter(package_url__in=purls) + .order_by("package_url") + .distinct("package_url") ) - packages = query + page = self.paginate_queryset(query) + serializer = self.get_serializer(page, many=True, context={"request": request}) + return self.get_paginated_response(serializer.data) - avids = set() - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - advisory_data = { - adv.avid: AdvisoryV2Serializer(adv, context={"request": request}).data - for adv in latest_advisories - } - if not purl_only: - package_data = PackageV3Serializer( - packages, - many=True, - context={"request": request}, - ).data - return Response( - { - "packages": package_data, - "advisories_by_id": advisory_data, - } - ) +class AdvisoryV3ViewSet(viewsets.GenericViewSet): + queryset = AdvisoryV2.objects.all() + serializer_class = AdvisoryV3Serializer + filter_backends = [filters.DjangoFilterBackend] + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - # Using order by and distinct because there will be - # many fully qualified purl for a single plain purl - vulnerable_purls = query.vulnerable().only("plain_package_url") - vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) + def create(self, request, *args, **kwargs): + serializer = PackageQuerySerializer(data=request.data) + serializer.is_valid(raise_exception=True) - query = ( - PackageV2.objects.filter(package_url__in=purls) - .order_by("plain_package_url") - .distinct("plain_package_url") - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory").prefetch_related( - "fixed_by_packages", - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - .with_is_vulnerable() - ) - packages = query + purls = serializer.validated_data["purls"] - avids = set() - for package in packages: - for impact in package.affected_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - for impact in package.fixed_in_impacts.all(): - if impact.advisory_id: - avids.add(impact.advisory.avid) - - latest_advisories = AdvisoryV2.objects.latest_for_avids(avids) - advisory_data = { - adv.avid: AdvisoryV2Serializer(adv, context={"request": request}).data - for adv in latest_advisories - } + latest_advisories = AdvisoryV2.objects.latest_advisories_for_purls(purls=purls) - if not purl_only: - package_data = PackageV3Serializer( - packages, - many=True, - context={"request": request}, - ).data - return Response( - { - "packages": package_data, - "advisories_by_id": advisory_data, - } - ) + page = self.paginate_queryset(latest_advisories) + serializer = self.get_serializer(page, many=True, context={"request": request}) + return self.get_paginated_response(serializer.data) - vulnerable_purls = query.vulnerable().only("package_url") - vulnerable_purls = [str(package.package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - @action(detail=False, methods=["get"]) - def all(self, request): - """ - Return a list of Package URLs of vulnerable packages. - """ - vulnerable_purls = ( - PackageV2.objects.vulnerable() - .only("package_url") - .order_by("package_url") - .distinct() - .values_list("package_url", flat=True) - ) - return Response(vulnerable_purls) +class PackageAdvisoriesViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = AdvisoryV3Serializer + relation = None - @extend_schema( - request=LookupRequestSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=LookupRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def lookup(self, request): - """ - Return the response for exact PackageURL requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A 'purl' is required.", - }, - ) - validated_data = serializer.validated_data - purl = validated_data.get("purl") + def get_queryset(self): + purl = self.request.query_params.get("purl") - qs = self.get_queryset().for_purls([purl]).with_is_vulnerable() - return Response(PackageV3Serializer(qs, many=True, context={"request": request}).data) + if not purl: + return AdvisoryV2.objects.none() + + return AdvisoryV2.objects.filter(**{self.relation: purl}).latest_per_avid() + + +class FixingAdvisoriesViewSet(PackageAdvisoriesViewSet): + relation = "impacted_packages__fixed_by_packages__package_url" + + +class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): + relation = "impacted_packages__affecting_packages__package_url" diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 2e69be49a..c4de3611a 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2876,6 +2876,36 @@ def latest_per_avid(self): def latest_for_avids(self, avids): return self.filter(avid__in=avids).latest_per_avid() + def latest_affecting_advisories_for_purl(self, purl): + return self.filter( + impacted_packages__affecting_packages__package_url=purl + ).latest_per_avid() + + def latest_affecting_advisories_for_purls(self, purls): + return self.filter( + impacted_packages__affecting_packages__package_url__in=purls + ).latest_per_avid() + + def latest_fixed_by_advisories_for_purl(self, purl): + return self.filter(impacted_packages__fixed_by_packages__package_url=purl).latest_per_avid() + + def latest_fixed_by_advisories_for_purls(self, purls): + return self.filter( + impacted_packages__fixed_by_packages__package_url__in=purls + ).latest_per_avid() + + def latest_advisories_for_purl(self, purl): + return self.filter( + Q(impacted_packages__affecting_packages__package_url=purl) + | Q(impacted_packages__fixed_by_packages__package_url=purl) + ).latest_per_avid() + + def latest_advisories_for_purls(self, purls): + return self.filter( + Q(impacted_packages__affecting_packages__package_url__in=purls) + | Q(impacted_packages__fixed_by_packages__package_url__in=purls) + ).latest_per_avid() + class AdvisoryV2(models.Model): """ diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 49948a3b9..0fcee200b 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -20,8 +20,11 @@ from vulnerabilities.api import CPEViewSet from vulnerabilities.api import PackageViewSet from vulnerabilities.api import VulnerabilityViewSet +from vulnerabilities.api_v2 import AdvisoryV3ViewSet +from vulnerabilities.api_v2 import AffectedByAdvisoriesViewSet from vulnerabilities.api_v2 import CodeFixV2ViewSet from vulnerabilities.api_v2 import CodeFixViewSet +from vulnerabilities.api_v2 import FixingAdvisoriesViewSet from vulnerabilities.api_v2 import PackageV2ViewSet from vulnerabilities.api_v2 import PackageV3ViewSet from vulnerabilities.api_v2 import PipelineScheduleV2ViewSet @@ -70,6 +73,11 @@ def __init__(self, *args, **kwargs): api_v3_router = OptionalSlashRouter() api_v3_router.register("packages", PackageV3ViewSet, basename="package-v3") +api_v3_router.register("advisories", AdvisoryV3ViewSet, basename="advisory-v3") +api_v3_router.register( + "affected-by-advisories", AffectedByAdvisoriesViewSet, basename="affected-by-advisories" +) +api_v3_router.register("fixing-advisories", FixingAdvisoriesViewSet, basename="fixing-advisories") urlpatterns = [ path("admin/login/", AdminLoginView.as_view(), name="admin-login"), From 3172cabb9833445041b0b0cbc2e3c9be683b8ac2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 11 Mar 2026 15:06:42 +0530 Subject: [PATCH 02/65] Restructure V3 API Signed-off-by: Tushar Goel --- vulnerabilities/api_v2.py | 384 ------------------------------------ vulnerabilities/api_v3.py | 400 ++++++++++++++++++++++++++++++++++++++ vulnerablecode/urls.py | 8 +- 3 files changed, 404 insertions(+), 388 deletions(-) create mode 100644 vulnerabilities/api_v3.py diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index 8cbd8ccc0..6e0ab9213 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -8,8 +8,6 @@ # -from urllib.parse import urlencode - from django.db.models import Prefetch from django.db.models import Q from django_filters import rest_framework as filters @@ -23,21 +21,14 @@ from rest_framework import viewsets from rest_framework.authentication import SessionAuthentication from rest_framework.decorators import action -from rest_framework.exceptions import MethodNotAllowed from rest_framework.permissions import BasePermission from rest_framework.response import Response from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle -from vulnerabilities.models import AdvisoryReference -from vulnerabilities.models import AdvisorySeverity -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import AdvisoryWeakness from vulnerabilities.models import CodeFix from vulnerabilities.models import CodeFixV2 -from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import Package -from vulnerabilities.models import PackageV2 from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.models import Vulnerability @@ -45,7 +36,6 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.throttling import PermissionBasedUserRateThrottle -from vulnerabilities.utils import group_advisories_by_content class CharInFilter(filters.BaseInFilter, filters.CharFilter): @@ -62,16 +52,6 @@ class Meta: fields = ["cwe_id", "name", "description"] -class AdvisoryWeaknessSerializer(serializers.ModelSerializer): - cwe_id = serializers.CharField() - name = serializers.CharField() - description = serializers.CharField() - - class Meta: - model = AdvisoryWeakness - fields = ["cwe_id", "name", "description"] - - class VulnerabilityReferenceV2Serializer(serializers.ModelSerializer): url = serializers.CharField() reference_type = serializers.CharField() @@ -82,29 +62,6 @@ class Meta: fields = ["url", "reference_type", "reference_id"] -class AdvisoryReferenceSerializer(serializers.ModelSerializer): - url = serializers.CharField() - reference_type = serializers.CharField() - reference_id = serializers.CharField() - - class Meta: - model = AdvisoryReference - fields = ["url", "reference_type", "reference_id"] - - -class AdvisorySeveritySerializer(serializers.ModelSerializer): - class Meta: - model = AdvisorySeverity - fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] - - def to_representation(self, instance): - data = super().to_representation(instance) - published_at = data.get("published_at", None) - if not published_at: - data.pop("published_at") - return data - - class VulnerabilitySeverityV2Serializer(serializers.ModelSerializer): class Meta: model = VulnerabilitySeverity @@ -145,58 +102,6 @@ def get_aliases(self, obj): return [alias.alias for alias in obj.aliases.all()] -class AdvisoryV3Serializer(serializers.ModelSerializer): - aliases = serializers.SerializerMethodField() - weaknesses = AdvisoryWeaknessSerializer(many=True) - references = AdvisoryReferenceSerializer(many=True) - severities = AdvisorySeveritySerializer(many=True) - advisory_id = serializers.CharField(source="avid", read_only=True) - related_ssvc_trees = serializers.SerializerMethodField() - - def get_related_ssvc_trees(self, obj): - related_ssvcs = obj.related_ssvcs.all().select_related("source_advisory") - source_ssvcs = obj.source_ssvcs.all().select_related("source_advisory") - - seen = set() - result = [] - - for ssvc in list(related_ssvcs) + list(source_ssvcs): - key = (ssvc.vector, ssvc.source_advisory_id) - if key in seen: - continue - seen.add(key) - - result.append( - { - "vector": ssvc.vector, - "decision": ssvc.decision, - "options": ssvc.options, - "source_url": ssvc.source_advisory.url, - } - ) - - return result - - class Meta: - model = AdvisoryV2 - fields = [ - "advisory_id", - "url", - "aliases", - "summary", - "severities", - "weaknesses", - "references", - "exploitability", - "weighted_severity", - "risk_score", - "related_ssvc_trees", - ] - - def get_aliases(self, obj): - return [alias.alias for alias in obj.aliases.all()] - - class VulnerabilityListSerializer(serializers.ModelSerializer): url = serializers.SerializerMethodField() @@ -337,147 +242,6 @@ def get_fixing_vulnerabilities(self, obj): return [vuln.vulnerability_id for vuln in obj.fixing_vulnerabilities.all()] -class PackageV3Serializer(serializers.ModelSerializer): - purl = serializers.CharField(source="package_url") - risk_score = serializers.FloatField(read_only=True) - affected_by_vulnerabilities = serializers.SerializerMethodField() - affected_by_vulnerabilities_url = serializers.SerializerMethodField() - fixing_vulnerabilities = serializers.SerializerMethodField() - fixing_vulnerabilities_url = serializers.SerializerMethodField() - next_non_vulnerable_version = serializers.SerializerMethodField() - latest_non_vulnerable_version = serializers.SerializerMethodField() - - class Meta: - model = Package - fields = [ - "purl", - "affected_by_vulnerabilities", - "affected_by_vulnerabilities_url", - "fixing_vulnerabilities", - "fixing_vulnerabilities_url", - "next_non_vulnerable_version", - "latest_non_vulnerable_version", - "risk_score", - ] - - def to_representation(self, instance): - data = super().to_representation(instance) - - if data.get("affected_by_vulnerabilities") is None: - data.pop("affected_by_vulnerabilities", None) - else: - data.pop("affected_by_vulnerabilities_url", None) - - if data.get("fixing_vulnerabilities") is None: - data.pop("fixing_vulnerabilities", None) - else: - data.pop("fixing_vulnerabilities_url", None) - - return data - - def get_affected_by_vulnerabilities_url(self, obj): - request = self.context.get("request") - if not request: - return None - - base = reverse("affected-by-advisories-list") - url = request.build_absolute_uri(base) - - return f"{url}?{urlencode({'purl': obj.package_url})}" - - def get_fixing_vulnerabilities_url(self, obj): - request = self.context.get("request") - if not request: - return None - - base = reverse("fixing-advisories-list") - url = request.build_absolute_uri(base) - - return f"{url}?{urlencode({'purl': obj.package_url})}" - - def get_affected_by_vulnerabilities(self, package): - """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" - advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) - - advisories = list(advisories_qs[:101]) - if len(advisories) > 100: - return None - - advisory_by_avid = {adv.avid: adv for adv in advisories} - avids = advisory_by_avid.keys() - - impacts = ( - package.affected_in_impacts.filter(advisory__avid__in=avids) - .select_related("advisory") - .prefetch_related("fixed_by_packages") - ) - - impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - - grouped = group_advisories_by_content(advisories) - - result = [] - for entry in grouped.values(): - primary = entry["primary"] - impact = impact_by_avid.get(primary.avid) - if not impact: - continue - - result.append( - { - "advisory_id": primary.avid, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], - "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], - } - ) - - return result - - def get_fixing_vulnerabilities(self, package): - advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - - advisories = list(advisories_qs[:101]) - if len(advisories) > 100: - return None - - advisory_by_avid = {adv.avid: adv for adv in advisories} - avids = advisory_by_avid.keys() - - impacts = ( - package.fixed_in_impacts.filter(advisory__avid__in=avids) - .select_related("advisory") - .prefetch_related("fixed_by_packages") - ) - - impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - - grouped = group_advisories_by_content(advisories) - - result = [] - for entry in grouped.values(): - primary = entry["primary"] - impact = impact_by_avid.get(primary.avid) - if not impact: - continue - - result.append( - { - "advisory_id": primary.avid, - "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], - } - ) - - return result - - def get_next_non_vulnerable_version(self, package): - if next_non_vulnerable := package.get_non_vulnerable_versions()[0]: - return next_non_vulnerable.version - - def get_latest_non_vulnerable_version(self, package): - if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]: - return latest_non_vulnerable.version - - class PackageurlListSerializer(serializers.Serializer): purls = serializers.ListField( child=serializers.CharField(), @@ -1085,151 +849,3 @@ def get_view_name(self): if self.detail: return "Pipeline Instance" return "Pipeline Jobs" - - -class PackageQuerySerializer(serializers.Serializer): - purls = serializers.ListField( - child=serializers.CharField(), - required=False, - default=list, - ) - details = serializers.BooleanField(default=False) - approximate = serializers.BooleanField(default=False) - - def validate(self, data): - if not data["purls"]: - if data["details"] or data["approximate"]: - raise serializers.ValidationError( - "details and approximate must be false when purls is empty" - ) - return data - - -class AdvisoryQuerySerializer(serializers.Serializer): - purls = serializers.ListField( - child=serializers.CharField(), - required=False, - default=list, - ) - - def validate(self, data): - if not data["purls"]: - raise serializers.ValidationError("purls is required") - return data - - -class PackageV3ViewSet(viewsets.GenericViewSet): - queryset = PackageV2.objects.all() - serializer_class = PackageV3Serializer - filter_backends = [filters.DjangoFilterBackend] - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def create(self, request, *args, **kwargs): - serializer = PackageQuerySerializer(data=request.data) - serializer.is_valid(raise_exception=True) - - purls = serializer.validated_data["purls"] - details = serializer.validated_data["details"] - approximate = serializer.validated_data["approximate"] - - if not purls: - vulnerable_purls = ( - PackageV2.objects.vulnerable() - .only("package_url") - .distinct() - .values_list("package_url", flat=True) - .order_by("package_url") - ) - page = self.paginate_queryset(vulnerable_purls) - return self.get_paginated_response(page) - - plain_purls = None - - if approximate: - plain_purls = [ - str( - PackageURL( - type=p.type, - namespace=p.namespace, - name=p.name, - version=p.version, - ) - ) - for p in map(PackageURL.from_string, purls) - ] - - if not details: - if approximate: - query = ( - PackageV2.objects.filter(plain_package_url__in=plain_purls) - .values_list("plain_package_url", flat=True) - .distinct() - .order_by("plain_package_url") - ) - else: - query = ( - PackageV2.objects.filter(package_url__in=purls) - .values_list("package_url", flat=True) - .distinct() - .order_by("package_url") - ) - - page = self.paginate_queryset(query) - return self.get_paginated_response(page) - - if approximate: - query = ( - PackageV2.objects.filter(plain_package_url__in=plain_purls) - .order_by("plain_package_url") - .distinct("plain_package_url") - ) - else: - query = ( - PackageV2.objects.filter(package_url__in=purls) - .order_by("package_url") - .distinct("package_url") - ) - - page = self.paginate_queryset(query) - serializer = self.get_serializer(page, many=True, context={"request": request}) - return self.get_paginated_response(serializer.data) - - -class AdvisoryV3ViewSet(viewsets.GenericViewSet): - queryset = AdvisoryV2.objects.all() - serializer_class = AdvisoryV3Serializer - filter_backends = [filters.DjangoFilterBackend] - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def create(self, request, *args, **kwargs): - serializer = PackageQuerySerializer(data=request.data) - serializer.is_valid(raise_exception=True) - - purls = serializer.validated_data["purls"] - - latest_advisories = AdvisoryV2.objects.latest_advisories_for_purls(purls=purls) - - page = self.paginate_queryset(latest_advisories) - serializer = self.get_serializer(page, many=True, context={"request": request}) - return self.get_paginated_response(serializer.data) - - -class PackageAdvisoriesViewSet(viewsets.ReadOnlyModelViewSet): - serializer_class = AdvisoryV3Serializer - relation = None - - def get_queryset(self): - purl = self.request.query_params.get("purl") - - if not purl: - return AdvisoryV2.objects.none() - - return AdvisoryV2.objects.filter(**{self.relation: purl}).latest_per_avid() - - -class FixingAdvisoriesViewSet(PackageAdvisoriesViewSet): - relation = "impacted_packages__fixed_by_packages__package_url" - - -class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): - relation = "impacted_packages__affecting_packages__package_url" diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py new file mode 100644 index 000000000..14a3effa7 --- /dev/null +++ b/vulnerabilities/api_v3.py @@ -0,0 +1,400 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from urllib.parse import urlencode + +from django_filters import rest_framework as filters +from packageurl import PackageURL +from rest_framework import serializers +from rest_framework import viewsets +from rest_framework.reverse import reverse +from rest_framework.throttling import AnonRateThrottle + +from vulnerabilities.models import AdvisoryReference +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import AdvisoryWeakness +from vulnerabilities.models import PackageV2 +from vulnerabilities.throttling import PermissionBasedUserRateThrottle +from vulnerabilities.utils import group_advisories_by_content + + +class PackageQuerySerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, + ) + details = serializers.BooleanField(default=False) + approximate = serializers.BooleanField(default=False) + + def validate(self, data): + if not data["purls"]: + if data["details"] or data["approximate"]: + raise serializers.ValidationError( + "details and approximate must be false when purls is empty" + ) + return data + + +class AdvisoryQuerySerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + required=False, + default=list, + ) + + def validate(self, data): + if not data["purls"]: + raise serializers.ValidationError("purls is required") + return data + + +class AdvisoryReferenceSerializer(serializers.ModelSerializer): + url = serializers.CharField() + reference_type = serializers.CharField() + reference_id = serializers.CharField() + + class Meta: + model = AdvisoryReference + fields = ["url", "reference_type", "reference_id"] + + +class AdvisorySeveritySerializer(serializers.ModelSerializer): + class Meta: + model = AdvisorySeverity + fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] + + def to_representation(self, instance): + data = super().to_representation(instance) + published_at = data.get("published_at", None) + if not published_at: + data.pop("published_at") + return data + + +class AdvisoryWeaknessSerializer(serializers.ModelSerializer): + cwe_id = serializers.CharField() + name = serializers.CharField() + description = serializers.CharField() + + class Meta: + model = AdvisoryWeakness + fields = ["cwe_id", "name", "description"] + + +class AdvisoryV3Serializer(serializers.ModelSerializer): + aliases = serializers.SerializerMethodField() + weaknesses = AdvisoryWeaknessSerializer(many=True) + references = AdvisoryReferenceSerializer(many=True) + severities = AdvisorySeveritySerializer(many=True) + advisory_id = serializers.CharField(source="avid", read_only=True) + related_ssvc_trees = serializers.SerializerMethodField() + + def get_related_ssvc_trees(self, obj): + related_ssvcs = obj.related_ssvcs.all().select_related("source_advisory") + source_ssvcs = obj.source_ssvcs.all().select_related("source_advisory") + + seen = set() + result = [] + + for ssvc in list(related_ssvcs) + list(source_ssvcs): + key = (ssvc.vector, ssvc.source_advisory_id) + if key in seen: + continue + seen.add(key) + + result.append( + { + "vector": ssvc.vector, + "decision": ssvc.decision, + "options": ssvc.options, + "source_url": ssvc.source_advisory.url, + } + ) + + return result + + class Meta: + model = AdvisoryV2 + fields = [ + "advisory_id", + "url", + "aliases", + "summary", + "severities", + "weaknesses", + "references", + "exploitability", + "weighted_severity", + "risk_score", + "related_ssvc_trees", + ] + + def get_aliases(self, obj): + return [alias.alias for alias in obj.aliases.all()] + + +class PackageV3Serializer(serializers.ModelSerializer): + purl = serializers.CharField(source="package_url") + risk_score = serializers.FloatField(read_only=True) + affected_by_vulnerabilities = serializers.SerializerMethodField() + affected_by_vulnerabilities_url = serializers.SerializerMethodField() + fixing_vulnerabilities = serializers.SerializerMethodField() + fixing_vulnerabilities_url = serializers.SerializerMethodField() + next_non_vulnerable_version = serializers.SerializerMethodField() + latest_non_vulnerable_version = serializers.SerializerMethodField() + + class Meta: + model = PackageV2 + fields = [ + "purl", + "affected_by_vulnerabilities", + "affected_by_vulnerabilities_url", + "fixing_vulnerabilities", + "fixing_vulnerabilities_url", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", + "risk_score", + ] + + def to_representation(self, instance): + data = super().to_representation(instance) + + if data.get("affected_by_vulnerabilities") is None: + data.pop("affected_by_vulnerabilities", None) + else: + data.pop("affected_by_vulnerabilities_url", None) + + if data.get("fixing_vulnerabilities") is None: + data.pop("fixing_vulnerabilities", None) + else: + data.pop("fixing_vulnerabilities_url", None) + + return data + + def get_affected_by_vulnerabilities_url(self, obj): + request = self.context.get("request") + if not request: + return None + + base = reverse("affected-by-advisories-list") + url = request.build_absolute_uri(base) + + return f"{url}?{urlencode({'purl': obj.package_url})}" + + def get_fixing_vulnerabilities_url(self, obj): + request = self.context.get("request") + if not request: + return None + + base = reverse("fixing-advisories-list") + url = request.build_absolute_uri(base) + + return f"{url}?{urlencode({'purl': obj.package_url})}" + + def get_affected_by_vulnerabilities(self, package): + """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" + advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + + advisories = list(advisories_qs[:101]) + if len(advisories) > 100: + return None + + advisory_by_avid = {adv.avid: adv for adv in advisories} + avids = advisory_by_avid.keys() + + impacts = ( + package.affected_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) + + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + + grouped = group_advisories_by_content(advisories) + + result = [] + for entry in grouped.values(): + primary = entry["primary"] + impact = impact_by_avid.get(primary.avid) + if not impact: + continue + + result.append( + { + "advisory_id": primary.avid, + "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], + } + ) + + return result + + def get_fixing_vulnerabilities(self, package): + advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) + + advisories = list(advisories_qs[:101]) + if len(advisories) > 100: + return None + + advisory_by_avid = {adv.avid: adv for adv in advisories} + avids = advisory_by_avid.keys() + + impacts = ( + package.fixed_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) + + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + + grouped = group_advisories_by_content(advisories) + + result = [] + for entry in grouped.values(): + primary = entry["primary"] + impact = impact_by_avid.get(primary.avid) + if not impact: + continue + + result.append( + { + "advisory_id": primary.avid, + "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], + } + ) + + return result + + def get_next_non_vulnerable_version(self, package): + if next_non_vulnerable := package.get_non_vulnerable_versions()[0]: + return next_non_vulnerable.version + + def get_latest_non_vulnerable_version(self, package): + if latest_non_vulnerable := package.get_non_vulnerable_versions()[-1]: + return latest_non_vulnerable.version + + +class PackageV3ViewSet(viewsets.GenericViewSet): + queryset = PackageV2.objects.all() + serializer_class = PackageV3Serializer + filter_backends = [filters.DjangoFilterBackend] + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + + def create(self, request, *args, **kwargs): + serializer = PackageQuerySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + purls = serializer.validated_data["purls"] + details = serializer.validated_data["details"] + approximate = serializer.validated_data["approximate"] + + if not purls: + vulnerable_purls = ( + PackageV2.objects.vulnerable() + .only("package_url") + .distinct() + .values_list("package_url", flat=True) + .order_by("package_url") + ) + page = self.paginate_queryset(vulnerable_purls) + return self.get_paginated_response(page) + + plain_purls = None + + if approximate: + plain_purls = [ + str( + PackageURL( + type=p.type, + namespace=p.namespace, + name=p.name, + version=p.version, + ) + ) + for p in map(PackageURL.from_string, purls) + ] + + if not details: + if approximate: + query = ( + PackageV2.objects.filter(plain_package_url__in=plain_purls) + .values_list("plain_package_url", flat=True) + .distinct() + .order_by("plain_package_url") + ) + else: + query = ( + PackageV2.objects.filter(package_url__in=purls) + .values_list("package_url", flat=True) + .distinct() + .order_by("package_url") + ) + + page = self.paginate_queryset(query) + return self.get_paginated_response(page) + + if approximate: + query = ( + PackageV2.objects.filter(plain_package_url__in=plain_purls) + .order_by("plain_package_url") + .distinct("plain_package_url") + ) + else: + query = ( + PackageV2.objects.filter(package_url__in=purls) + .order_by("package_url") + .distinct("package_url") + ) + + page = self.paginate_queryset(query) + serializer = self.get_serializer(page, many=True, context={"request": request}) + return self.get_paginated_response(serializer.data) + + +class AdvisoryV3ViewSet(viewsets.GenericViewSet): + queryset = AdvisoryV2.objects.all() + serializer_class = AdvisoryV3Serializer + filter_backends = [filters.DjangoFilterBackend] + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + + def create(self, request, *args, **kwargs): + serializer = PackageQuerySerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + purls = serializer.validated_data["purls"] + + latest_advisories = AdvisoryV2.objects.latest_advisories_for_purls(purls=purls) + + page = self.paginate_queryset(latest_advisories) + serializer = self.get_serializer(page, many=True, context={"request": request}) + return self.get_paginated_response(serializer.data) + + +class PackageAdvisoriesViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = AdvisoryV3Serializer + relation = None + throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] + + def get_queryset(self): + purl = self.request.query_params.get("purl") + + if not purl: + return AdvisoryV2.objects.none() + + return AdvisoryV2.objects.filter(**{self.relation: purl}).latest_per_avid() + + +class FixingAdvisoriesViewSet(PackageAdvisoriesViewSet): + relation = "impacted_packages__fixed_by_packages__package_url" + + +class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): + relation = "impacted_packages__affecting_packages__package_url" diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 0fcee200b..08d1371d7 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -20,13 +20,13 @@ from vulnerabilities.api import CPEViewSet from vulnerabilities.api import PackageViewSet from vulnerabilities.api import VulnerabilityViewSet -from vulnerabilities.api_v2 import AdvisoryV3ViewSet -from vulnerabilities.api_v2 import AffectedByAdvisoriesViewSet +from vulnerabilities.api_v3 import AdvisoryV3ViewSet +from vulnerabilities.api_v3 import AffectedByAdvisoriesViewSet from vulnerabilities.api_v2 import CodeFixV2ViewSet from vulnerabilities.api_v2 import CodeFixViewSet -from vulnerabilities.api_v2 import FixingAdvisoriesViewSet +from vulnerabilities.api_v3 import FixingAdvisoriesViewSet from vulnerabilities.api_v2 import PackageV2ViewSet -from vulnerabilities.api_v2 import PackageV3ViewSet +from vulnerabilities.api_v3 import PackageV3ViewSet from vulnerabilities.api_v2 import PipelineScheduleV2ViewSet from vulnerabilities.api_v2 import VulnerabilityV2ViewSet from vulnerabilities.views import AdminLoginView From 3b7f794cb5e2396e31a7f8556829cbcd9a471276 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 15:35:47 +0530 Subject: [PATCH 03/65] Fix formatting issues Signed-off-by: Tushar Goel --- vulnerablecode/urls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 08d1371d7..fa6ddf9e3 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -20,15 +20,15 @@ from vulnerabilities.api import CPEViewSet from vulnerabilities.api import PackageViewSet from vulnerabilities.api import VulnerabilityViewSet -from vulnerabilities.api_v3 import AdvisoryV3ViewSet -from vulnerabilities.api_v3 import AffectedByAdvisoriesViewSet from vulnerabilities.api_v2 import CodeFixV2ViewSet from vulnerabilities.api_v2 import CodeFixViewSet -from vulnerabilities.api_v3 import FixingAdvisoriesViewSet from vulnerabilities.api_v2 import PackageV2ViewSet -from vulnerabilities.api_v3 import PackageV3ViewSet from vulnerabilities.api_v2 import PipelineScheduleV2ViewSet from vulnerabilities.api_v2 import VulnerabilityV2ViewSet +from vulnerabilities.api_v3 import AdvisoryV3ViewSet +from vulnerabilities.api_v3 import AffectedByAdvisoriesViewSet +from vulnerabilities.api_v3 import FixingAdvisoriesViewSet +from vulnerabilities.api_v3 import PackageV3ViewSet from vulnerabilities.views import AdminLoginView from vulnerabilities.views import AdvisoryDetails from vulnerabilities.views import AdvisoryPackagesDetails From 5981a7f1765a6f9bb3396fabd9320393ccc15c28 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 19:14:35 +0530 Subject: [PATCH 04/65] Add tests Signed-off-by: Tushar Goel --- api_v3_usage.rst | 338 +++++++++++++++++++++++++++ vulnerabilities/api_v3.py | 2 +- vulnerabilities/tests/test_api_v2.py | 71 ------ vulnerabilities/tests/test_api_v3.py | 252 ++++++++++++++++++++ 4 files changed, 591 insertions(+), 72 deletions(-) create mode 100644 api_v3_usage.rst create mode 100644 vulnerabilities/tests/test_api_v3.py diff --git a/api_v3_usage.rst b/api_v3_usage.rst new file mode 100644 index 000000000..ae8c38b7f --- /dev/null +++ b/api_v3_usage.rst @@ -0,0 +1,338 @@ + +Package endpoint +------------------ + +We are moving from API v1 to API V3. + +- /api/packages earlier had "bulk_search", "bulk_lookup", "lookup" and "all" endpoints. + +- /api/v3/packages has only one endpoint, which have same capabilities as all of these endpoints. + +- Response by package endpoint, will always be paginated, with 10 results per page, and will have "next" and "previous" links for pagination. If there are more than 100 advisories for a package, then it will return "affected_by_vulnerabilities_url" and "fixing_vulnerabilities_url" instead of "affected_by_vulnerabilities" and "fixing_vulnerabilities" respectively. + +"all" + +- Instead of doing /api/packages/all, we can do /api/v3/packages with empty purls list. + +- To get all vulnerable packages: + +``` +POST /api/v3/packages +{ + "purls": [] +} +``` + +Response: + +``` + +{ + "count": 596, + "next": "http://example.com/api/v3/packages?page=2", + "previous": null, + "results": [ + "pkg:npm/626@1.1.1", + "pkg:npm/aedes@0.35.0", + "pkg:npm/airbrake@0.3.8", + "pkg:npm/angular-http-server@1.4.3", + "pkg:npm/apex-publish-static-files@2.0.0", + "pkg:npm/atob@2.0.3", + "pkg:npm/augustine@0.2.3", + "pkg:npm/backbone@0.3.3", + "pkg:npm/base64-url@1.3.3", + "pkg:npm/base64url@2.0.0" + ] +} +``` + + +"bulk_search" + +- Instead of doing /api/packages/bulk_search, we can do /api/v3/packages with purls list and "details" as false or true (by default it's false), earlier we had "purls_only" . Also, previosuly we used to have "plain_purl" as a parameter, to ignore qualifiers and subpaths in purls, now we have "approximate", if set to True will ignore qualifiers and subpaths in purls. + +Examples: + +- To get only purls of vulnerable packages: +``` +POST /api/v3/packages +{ + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], + "details": false +} +``` + +Response: + +``` +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + "pkg:npm/atob@2.0.3" + ] +} + +``` + +- To get details of vulnerable packages: +``` +POST /api/v3/packages +{ + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], + "details": true +} +``` + +Response: +``` + +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "purl": "pkg:npm/atob@2.0.3", + "affected_by_vulnerabilities": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "fixed_by_packages": [ + "pkg:npm/atob@2.1.0" + ], + "duplicate_advisory_ids": [] + } + ], + "fixing_vulnerabilities": [], + "next_non_vulnerable_version": "2.1.0", + "latest_non_vulnerable_version": "2.1.0", + "risk_score": null + } + ] +} +``` + +- To get details of vulnerable packages by ignoring qualifiers and subpaths in purls: +``` +POST /api/v3/packages +{ + "purls": ["pkg:npm/atob@2.0.3?foo=bar", "pkg:pypi/sample@2.0.0"], + "approximate": true, + "details": true +} +``` + +Response: +``` + +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "purl": "pkg:npm/atob@2.0.3", + "affected_by_vulnerabilities": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "fixed_by_packages": [ + "pkg:npm/atob@2.1.0" + ], + "duplicate_advisory_ids": [] + } + ], + "fixing_vulnerabilities": [], + "next_non_vulnerable_version": "2.1.0", + "latest_non_vulnerable_version": "2.1.0", + "risk_score": null + } + ] +} +``` + +- To get vulnerable packages by ignoring qualifiers and subpaths in purls: +``` +POST /api/v3/packages +{ + "purls": ["pkg:npm/atob@2.0.3?foo=bar"], + "approximate": true, +} +``` + +Response: + +``` +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + "pkg:npm/atob@2.0.3" + ] +} + +``` + +Advisory endpoint +------------------ + +- You can get all advisories for a purl or list of purls by using /api/v3/advisories endpoint. It will also be paginated with 10 results per page, and will have "next" and "previous" links for pagination + +``` +POST /api/v3/advisories +{ + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"] +} +``` + +Response: + +``` +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "aliases": [ + "CVE-2018-3745" + ], + "summary": "Out-of-bounds Read\n`atob` allocates uninitialized Buffers when number is passed in input on Node.js 4.x and below", + "severities": [ + { + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "value": "6.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:H" + } + ], + "weaknesses": [], + "references": [ + { + "url": "https://hackerone.com/reports/321686", + "reference_type": "", + "reference_id": "" + }, + { + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "reference_type": "", + "reference_id": "403" + } + ], + "exploitability": null, + "weighted_severity": null, + "risk_score": null, + "related_ssvc_trees": [] + } + ] +} +``` + +Affected By Advisories endpoint +-------------------------------------- + +- You can get all advisories that fix a purl by using /api/v3/affected-by-advisories?purl= endpoint + +``` +GET /api/v3/affected-by-advisories?purl=pkg:npm/atob@2.0.3 +``` + +Response: +``` +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "aliases": [ + "CVE-2018-3745" + ], + "summary": "Out-of-bounds Read\n`atob` allocates uninitialized Buffers when number is passed in input on Node.js 4.x and below", + "severities": [ + { + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "value": "6.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:H" + } + ], + "weaknesses": [], + "references": [ + { + "url": "https://hackerone.com/reports/321686", + "reference_type": "", + "reference_id": "" + }, + { + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "reference_type": "", + "reference_id": "403" + } + ], + "exploitability": null, + "weighted_severity": null, + "risk_score": null, + "related_ssvc_trees": [] + } + ] +} +``` + +Fixing Advisories endpoint +----------------------------- + +- You can get all advisories that are fixed by a purl by using /api/v3/fixing-advisories?purl= endpoint + +``` +GET /api/v3/fixing-advisories?purl=pkg:npm/atob@2.1.0 +``` + +Response: +``` +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "aliases": [ + "CVE-2018-3745" + ], + "summary": "Out-of-bounds Read\n`atob` allocates uninitialized Buffers when number is passed in input on Node.js 4.x and below", + "severities": [ + { + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "value": "6.5", + "scoring_system": "cvssv3", + "scoring_elements": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:H" + } + ], + "weaknesses": [], + "references": [ + { + "url": "https://hackerone.com/reports/321686", + "reference_type": "", + "reference_id": "" + }, + { + "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", + "reference_type": "", + "reference_id": "403" + } + ], + "exploitability": null, + "weighted_severity": null, + "risk_score": null, + "related_ssvc_trees": [] + } + ] +} +``` diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 14a3effa7..c1247e1b1 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -333,9 +333,9 @@ def create(self, request, *args, **kwargs): else: query = ( PackageV2.objects.filter(package_url__in=purls) - .values_list("package_url", flat=True) .distinct() .order_by("package_url") + .values_list("package_url", flat=True) ) page = self.paginate_queryset(query) diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py index 6968123c7..c4abe3b97 100644 --- a/vulnerabilities/tests/test_api_v2.py +++ b/vulnerabilities/tests/test_api_v2.py @@ -834,74 +834,3 @@ def test_filter_codefix_by_advisory_id_not_found(self): response = self.client.get(self.url, {"advisory_id": "nonexistent/ADVISORY-ID"}) assert response.status_code == status.HTTP_200_OK assert response.data["count"] == 0 - - -class AdvisoriesPackageV2Tests(APITestCase): - def setUp(self): - from vulnerabilities.models import ImpactedPackage - - self.advisory = AdvisoryV2.objects.create( - datasource_id="ghsa", - advisory_id="GHSA-1234", - avid="ghsa/GHSA-1234", - unique_content_id="f" * 64, - url="https://example.com/advisory", - date_collected="2025-07-01T00:00:00Z", - ) - - self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0") - self.impact = ImpactedPackage.objects.create( - advisory=self.advisory, base_purl="pkg:pypi/sample" - ) - self.impact.affecting_packages.add(self.package) - - self.client = APIClient(enforce_csrf_checks=True) - - def test_list_with_purl_filter(self): - url = reverse("package-v3-list") - with self.assertNumQueries(31): - response = self.client.get(url, {"purl": "pkg:pypi/sample@1.0.0"}) - assert response.status_code == 200 - assert "packages" in response.data["results"] - assert "advisories_by_id" in response.data["results"] - assert self.advisory.avid in response.data["results"]["advisories_by_id"] - - def test_bulk_lookup(self): - url = reverse("package-v3-bulk-lookup") - with self.assertNumQueries(30): - response = self.client.post(url, {"purls": ["pkg:pypi/sample@1.0.0"]}, format="json") - assert response.status_code == 200 - assert "packages" in response.data - assert "advisories_by_id" in response.data - assert self.advisory.avid in response.data["advisories_by_id"] - - def test_bulk_search_plain(self): - url = reverse("package-v3-bulk-search") - payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": True, "purl_only": False} - with self.assertNumQueries(30): - response = self.client.post(url, payload, format="json") - assert response.status_code == 200 - assert "packages" in response.data - assert "advisories_by_id" in response.data - - def test_bulk_search_purl_only(self): - url = reverse("package-v3-bulk-search") - payload = {"purls": ["pkg:pypi/sample@1.0.0"], "plain_purl": False, "purl_only": True} - with self.assertNumQueries(17): - response = self.client.post(url, payload, format="json") - assert response.status_code == 200 - assert "pkg:pypi/sample@1.0.0" in response.data - - def test_lookup_single_package(self): - url = reverse("package-v3-lookup") - with self.assertNumQueries(23): - response = self.client.post(url, {"purl": "pkg:pypi/sample@1.0.0"}, format="json") - assert response.status_code == 200 - assert any(pkg["purl"] == "pkg:pypi/sample@1.0.0" for pkg in response.data) - - def test_get_all_vulnerable_purls(self): - url = reverse("package-v3-all") - with self.assertNumQueries(3): - response = self.client.get(url) - assert response.status_code == 200 - assert "pkg:pypi/sample@1.0.0" in response.data diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py new file mode 100644 index 000000000..db6242d8b --- /dev/null +++ b/vulnerabilities/tests/test_api_v3.py @@ -0,0 +1,252 @@ +from django.urls import reverse +from packageurl import PackageURL +from rest_framework import status +from rest_framework.test import APIClient +from rest_framework.test import APITestCase +from univers.version_range import PypiVersionRange + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipes.advisory import insert_advisory_v2 + + +class APIV3TestCase(APITestCase): + def setUp(self): + from vulnerabilities.models import ImpactedPackage + + self.advisory = AdvisoryV2.objects.create( + datasource_id="ghsa", + advisory_id="GHSA-1234", + avid="ghsa/GHSA-1234", + unique_content_id="f" * 64, + url="https://example.com/advisory", + date_collected="2025-07-01T00:00:00Z", + ) + + self.package = PackageV2.objects.from_purl(purl="pkg:pypi/sample@1.0.0") + self.impact = ImpactedPackage.objects.create( + advisory=self.advisory, base_purl="pkg:pypi/sample" + ) + self.impact.affecting_packages.add(self.package) + + self.client = APIClient(enforce_csrf_checks=True) + + def test_packages_post_without_details(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(4): + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0"], + "details": False, + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0], "pkg:pypi/sample@1.0.0") + + def test_packages_post_with_details(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(21): + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0"], + "details": True, + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + pkg = response.data["results"][0] + self.assertEqual(pkg["purl"], "pkg:pypi/sample@1.0.0") + + def test_advisories_post(self): + url = reverse("advisory-v3-list") + + with self.assertNumQueries(10): + response = self.client.post( + url, + data={"purls": ["pkg:pypi/sample@1.0.0"]}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + advisory = response.data["results"][0] + self.assertEqual(advisory["advisory_id"], "ghsa/GHSA-1234") + + def test_affected_by_advisories_list(self): + url = reverse("affected-by-advisories-list") + + with self.assertNumQueries(10): + response = self.client.get( + url, + {"purl": "pkg:pypi/sample@1.0.0"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["advisory_id"], "ghsa/GHSA-1234") + + def test_fixing_advisories_list_empty(self): + url = reverse("fixing-advisories-list") + + with self.assertNumQueries(3): + response = self.client.get( + url, + {"purl": "pkg:pypi/sample@1.0.0"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]), 0) + + def test_packages_pagination(self): + url = reverse("package-v3-list") + + response = self.client.post( + url, + data={"purls": []}, + format="json", + ) + + self.assertEqual(response.status_code, 200) + + self.assertIn("count", response.data) + self.assertEqual(response.data["count"], 1) + self.assertIn("results", response.data) + self.assertIn("next", response.data) + + def test_packages_approximate(self): + url = reverse("package-v3-list") + + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0?foo=bar"], + "approximate": True, + "details": False, + }, + format="json", + ) + + self.assertEqual(response.status_code, 200) + self.assertGreaterEqual(len(response.data["results"]), 1) + self.assertIn("pkg:pypi/sample@1.0.0", response.data["results"]) + + +class APIV3TestCaseOnePackageMultipleAdvisories(APITestCase): + def setUp(self): + from vulnerabilities.importer import AdvisoryDataV2 + from vulnerabilities.importer import AffectedPackageV2 + + affected_packages = [] + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="pypi", name="sample"), + affected_version_range=PypiVersionRange.from_string("vers:pypi/=1.0.0"), + ) + ) + + for i in range(1, 102): + advisory = AdvisoryDataV2( + advisory_id=f"GHSA-1234{i}", + aliases=[f"CVE-2021-1234{i}"], + summary="Sample advisory", + affected_packages=affected_packages, + url="https://example.com/advisory", + original_advisory_text="Sample advisory text", + ) + + insert_advisory_v2(advisory, "ghsa_importer", print, 100) + + self.client = APIClient(enforce_csrf_checks=True) + + def test_packages_post_purl_with_many_advisories(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(11): + response = self.client.post( + url, + data={ + "purls": ["pkg:pypi/sample@1.0.0"], + "details": True, + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertIsNotNone(results[0]["affected_by_vulnerabilities_url"]) + + def test_advisories_post(self): + url = reverse("advisory-v3-list") + + with self.assertNumQueries(64): + response = self.client.post( + url, + data={"purls": ["pkg:pypi/sample@1.0.0"]}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]), 10) + advisory = response.data["results"][0] + self.assertEqual(advisory["advisory_id"], "ghsa_importer/GHSA-12341") + + +class APIV3TestCaseOneAdvisoryMultiplePackages(APITestCase): + def setUp(self): + from vulnerabilities.importer import AdvisoryDataV2 + from vulnerabilities.importer import AffectedPackageV2 + + affected_packages = [] + for i in range(1, 102): + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="pypi", name=f"sample{i}"), + affected_version_range=PypiVersionRange.from_string("vers:pypi/=1.0.0"), + ) + ) + + advisory = AdvisoryDataV2( + advisory_id=f"GHSA-1234{i}", + aliases=[f"CVE-2021-1234{i}"], + summary="Sample advisory", + affected_packages=affected_packages, + url="https://example.com/advisory", + original_advisory_text="Sample advisory text", + ) + + insert_advisory_v2(advisory, "ghsa_importer", print, 100) + + self.client = APIClient(enforce_csrf_checks=True) + + def test_get_all_vulnerable_purls(self): + url = reverse("package-v3-list") + + with self.assertNumQueries(4): + response = self.client.post( + url, + data={ + "purls": [], + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data["results"] + self.assertEqual(len(results), 10) + self.assertIn("next", response.data) From 4461016b1960fd2e568d95595c7197e8b1966d37 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 19:18:13 +0530 Subject: [PATCH 05/65] Disable Admin panel Signed-off-by: Tushar Goel --- vulnerablecode/urls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index fa6ddf9e3..112678a24 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -182,10 +182,10 @@ def __init__(self, *args, **kwargs): TemplateView.as_view(template_name="tos.html"), name="api_tos", ), - path( - "admin/", - admin.site.urls, - ), + # path( + # "admin/", + # admin.site.urls, + # ), ] if DEBUG: From e8abe8533745df9903caba958150b25818058dc6 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 21:18:38 +0530 Subject: [PATCH 06/65] Inline vulnerability data Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index c1247e1b1..c37e141d1 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -204,8 +204,8 @@ def get_affected_by_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) advisories = list(advisories_qs[:101]) - if len(advisories) > 100: - return None + # if len(advisories) > 100: + # return None advisory_by_avid = {adv.avid: adv for adv in advisories} avids = advisory_by_avid.keys() @@ -241,8 +241,8 @@ def get_fixing_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) advisories = list(advisories_qs[:101]) - if len(advisories) > 100: - return None + # if len(advisories) > 100: + # return None advisory_by_avid = {adv.avid: adv for adv in advisories} avids = advisory_by_avid.keys() @@ -359,6 +359,34 @@ def create(self, request, *args, **kwargs): return self.get_paginated_response(serializer.data) +class AffectedByAdvisoryV3Serializer(AdvisoryV3Serializer): + fixed_by_packages = serializers.SerializerMethodField() + + def get_fixed_by_packages(self, obj): + return list( + obj.impacted_packages.values_list("fixed_by_packages__package_url", flat=True) + .exclude(fixed_by_packages__package_url__isnull=True) + .distinct() + ) + + class Meta: + model = AdvisoryV2 + fields = [ + "advisory_id", + "url", + "aliases", + "summary", + "severities", + "weaknesses", + "references", + "exploitability", + "weighted_severity", + "risk_score", + "related_ssvc_trees", + "fixed_by_packages", + ] + + class AdvisoryV3ViewSet(viewsets.GenericViewSet): queryset = AdvisoryV2.objects.all() serializer_class = AdvisoryV3Serializer @@ -398,3 +426,4 @@ class FixingAdvisoriesViewSet(PackageAdvisoriesViewSet): class AffectedByAdvisoriesViewSet(PackageAdvisoriesViewSet): relation = "impacted_packages__affecting_packages__package_url" + serializer_class = AffectedByAdvisoryV3Serializer From 1adbb4aebef4e3cd1a4b2bff9120e89265aecd5a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 21:29:07 +0530 Subject: [PATCH 07/65] Revert changes Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index c37e141d1..2549aacc9 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -204,8 +204,8 @@ def get_affected_by_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) advisories = list(advisories_qs[:101]) - # if len(advisories) > 100: - # return None + if len(advisories) > 100: + return None advisory_by_avid = {adv.avid: adv for adv in advisories} avids = advisory_by_avid.keys() @@ -241,8 +241,8 @@ def get_fixing_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) advisories = list(advisories_qs[:101]) - # if len(advisories) > 100: - # return None + if len(advisories) > 100: + return None advisory_by_avid = {adv.avid: adv for adv in advisories} avids = advisory_by_avid.keys() From 6f065ef2ab12f1fdd289e97f5f11432873d8feff Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 21:30:25 +0530 Subject: [PATCH 08/65] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/tests/test_api_v3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index db6242d8b..6d8b9519b 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -86,7 +86,7 @@ def test_advisories_post(self): def test_affected_by_advisories_list(self): url = reverse("affected-by-advisories-list") - with self.assertNumQueries(10): + with self.assertNumQueries(11): response = self.client.get( url, {"purl": "pkg:pypi/sample@1.0.0"}, From a7520b30c8fb6aa93a88b79d2cc92ffaf919a21e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 22:25:10 +0530 Subject: [PATCH 09/65] Remove advisories count Signed-off-by: Tushar Goel --- .../templates/package_details_v2.html | 10 +++- vulnerabilities/views.py | 59 +++++++++---------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index 9cc9ea343..38d6c2e9f 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -192,9 +192,15 @@ {{ item.pkg.version }}
- - Subject of {{ item.affected_count }} other advisories. + {% if item.pkg.is_vulnerable %} + + Vulnerable + {% else %} + + Not vulnerable + + {% endif %} {% endfor %} {% else %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 860bde8eb..c3222fe92 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import logging +from collections import defaultdict from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -15,8 +16,8 @@ from django.contrib.auth.views import LoginView from django.core.exceptions import ValidationError from django.core.mail import send_mail -from django.db.models import Count -from django.db.models import F +from django.db.models import Exists +from django.db.models import OuterRef from django.db.models import Prefetch from django.http.response import Http404 from django.shortcuts import get_object_or_404 @@ -227,48 +228,46 @@ def get_context_data(self, **kwargs): return context def get_fixed_package_details(self, package): - affected_impacts = package.affected_in_impacts.select_related("advisory").prefetch_related( - Prefetch( - "fixed_by_packages", - queryset=( - models.PackageV2.objects.annotate(affected_count=Count("affected_in_impacts")) - ), - ) + rows = package.affected_in_impacts.values_list( + "advisory__avid", + "fixed_by_packages", ) - fixed_impacts = package.fixed_in_impacts.select_related("advisory") - - affected_avids = {impact.advisory.avid for impact in affected_impacts if impact.advisory_id} + pkg_ids = {pkg_id for _, pkg_id in rows if pkg_id} - fixed_avids = {impact.advisory.avid for impact in fixed_impacts if impact.advisory_id} - - all_avids = affected_avids | fixed_avids + pkg_map = { + p.id: p + for p in models.PackageV2.objects.filter(id__in=pkg_ids).annotate( + is_vulnerable=Exists( + models.ImpactedPackage.objects.filter(affecting_packages=OuterRef("pk")) + ) + ) + } - advisories = models.AdvisoryV2.objects.latest_for_avids(all_avids) - advisory_by_avid = {adv.avid: adv for adv in advisories} + fixed_pkg_details = defaultdict(list) - fixed_pkg_details = {} + for avid, pkg_id in rows: + if not pkg_id: + continue - for impact in affected_impacts: - advisory = advisory_by_avid.get(impact.advisory.avid) - if not advisory: + pkg = pkg_map.get(pkg_id) + if not pkg: continue - fixed_pkg_details.setdefault(impact.advisory.avid, []).extend( + fixed_pkg_details[avid].append( { "pkg": pkg, - "affected_count": pkg.affected_count, + "is_vulnerable": pkg.is_vulnerable, } - for pkg in impact.fixed_by_packages.all() ) - affected_by_advisories = { - advisory_by_avid[avid] for avid in affected_avids if avid in advisory_by_avid - } + affected_by_advisories = models.AdvisoryV2.objects.latest_affecting_advisories_for_purl( + package.package_url + ) - fixing_advisories = { - advisory_by_avid[avid] for avid in fixed_avids if avid in advisory_by_avid - } + fixing_advisories = models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + package.package_url + ) return fixed_pkg_details, affected_by_advisories, fixing_advisories From 9e72decee3af6583354efed39d1b8e544fdc921a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 16 Mar 2026 23:33:30 +0530 Subject: [PATCH 10/65] Make search more efficient Signed-off-by: Tushar Goel --- vulnerabilities/templates/packages_v2.html | 8 ++++---- vulnerabilities/views.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/templates/packages_v2.html b/vulnerabilities/templates/packages_v2.html index fe2b05abe..752e0709c 100644 --- a/vulnerabilities/templates/packages_v2.html +++ b/vulnerabilities/templates/packages_v2.html @@ -41,14 +41,14 @@ - Affected by vulnerabilities + Vulnerable - Fixing vulnerabilities + Risk Score @@ -61,8 +61,8 @@ href="{{ package.get_absolute_url }}?search={{ search }}" target="_self">{{ package.purl }} - {{ package.vulnerability_count }} - {{ package.patched_vulnerability_count }} + {{ package.is_vulnerable|yesno:"Yes,No" }} + {{ package.risk_score }} {% empty %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index c3222fe92..6d78b650e 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -100,9 +100,9 @@ def get_queryset(self, query=None): query = query or self.request.GET.get("search") or "" return ( self.model.objects.search(query) - .with_vulnerability_counts() .prefetch_related() .order_by("package_url") + .with_is_vulnerable() ) From c52e2abbe5b88c214cf5e6d35a491fbf0473bd36 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 14:49:47 +0530 Subject: [PATCH 11/65] Make improvers query correct and faster Signed-off-by: Tushar Goel --- .../migrations/0117_advisoryv2_risk_score.py | 24 +++++ vulnerabilities/models.py | 18 ++-- .../v2_improvers/collect_ssvc_trees.py | 3 +- .../compute_advisory_content_hash.py | 2 +- .../v2_improvers/compute_package_risk.py | 102 ++++++++++++------ .../v2_improvers/enhance_with_exploitdb.py | 2 +- .../v2_improvers/enhance_with_kev.py | 2 +- .../v2_improvers/enhance_with_metasploit.py | 2 +- .../v2_improvers/relate_severities.py | 23 ++-- vulnerabilities/risk.py | 18 ++-- 10 files changed, 131 insertions(+), 65 deletions(-) create mode 100644 vulnerabilities/migrations/0117_advisoryv2_risk_score.py diff --git a/vulnerabilities/migrations/0117_advisoryv2_risk_score.py b/vulnerabilities/migrations/0117_advisoryv2_risk_score.py new file mode 100644 index 000000000..47733da5e --- /dev/null +++ b/vulnerabilities/migrations/0117_advisoryv2_risk_score.py @@ -0,0 +1,24 @@ +# Generated by Django 5.2.11 on 2026-03-17 09:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0116_advisoryv2_advisory_content_hash"), + ] + + operations = [ + migrations.AddField( + model_name="advisoryv2", + name="risk_score", + field=models.DecimalField( + blank=True, + decimal_places=1, + help_text="Risk expressed as a number ranging from 0 to 10. Risk is calculated from weighted severity and exploitability values. It is the maximum value of (the weighted severity multiplied by its exploitability) or 10. Risk = min(weighted severity * exploitability, 10)", + max_digits=3, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index c4de3611a..7fa162022 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3047,17 +3047,13 @@ class AdvisoryV2(models.Model): help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.", ) - @property - def risk_score(self): - """ - Risk expressed as a number ranging from 0 to 10. - Risk is calculated from weighted severity and exploitability values. - It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 - Risk = min(weighted severity * exploitability, 10) - """ - if self.exploitability and self.weighted_severity: - risk_score = min(float(self.exploitability * self.weighted_severity), 10.0) - return round(risk_score, 1) + risk_score = models.DecimalField( + null=True, + blank=True, + max_digits=3, + decimal_places=1, + help_text="Risk expressed as a number ranging from 0 to 10. Risk is calculated from weighted severity and exploitability values. It is the maximum value of (the weighted severity multiplied by its exploitability) or 10. Risk = min(weighted severity * exploitability, 10)", + ) objects = AdvisoryV2QuerySet.as_manager() diff --git a/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py b/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py index c98b8c9a9..d96e25cd7 100644 --- a/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py +++ b/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py @@ -36,7 +36,8 @@ def steps(cls): def collect_ssvc_data(self): vulnrichment_advisories = ( - AdvisoryV2.objects.filter( + AdvisoryV2.objects.latest_per_avid() + .filter( severities__scoring_system=SCORING_SYSTEMS["ssvc"], ) .distinct() diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py index fe5a3c97e..935631419 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py @@ -27,7 +27,7 @@ def steps(cls): def compute_advisory_content_hash(self): """Compute Advisory Content Hash for Advisory.""" - advisories = AdvisoryV2.objects.filter(advisory_content_hash__isnull=True) + advisories = AdvisoryV2.objects.latest_per_avid().filter(advisory_content_hash__isnull=True) advisories_count = advisories.count() diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index 9caaaeb95..e625176bb 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -8,8 +8,9 @@ # from aboutcode.pipeline import LoopProgress from django.db.models import Prefetch -from django.db.models import Q +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 @@ -36,61 +37,92 @@ def steps(cls): ) def compute_and_store_vulnerability_risk_score(self): + affected_advisories = ( - AdvisoryV2.objects.filter(impacted_packages__affecting_packages__isnull=False) + AdvisoryV2.objects.latest_per_avid() + .filter(impacted_packages__affecting_packages__isnull=False) + .only("id") .prefetch_related( - "references", - "severities", - "exploits", + Prefetch( + "references", queryset=AdvisoryReference.objects.only("id", "reference_type") + ), + Prefetch( + "severities", + queryset=AdvisorySeverity.objects.only("id", "value", "url", "scoring_system"), + ), + Prefetch("exploits", queryset=AdvisoryExploit.objects.only("id")), Prefetch( "related_advisory_severities", - queryset=AdvisoryV2.objects.prefetch_related("severities"), + queryset=AdvisoryV2.objects.only("id").prefetch_related( + Prefetch( + "severities", + queryset=AdvisorySeverity.objects.only( + "id", "value", "url", "scoring_system" + ), + ) + ), ), ) .distinct() ) + estimated_vulnerability_count = affected_advisories.count() + self.log( - f"Calculating risk for {affected_advisories.count():,d} advisory with a affected packages records" + f"Calculating risk for {estimated_vulnerability_count:,d} advisory with a affected packages records" ) - progress = LoopProgress(total_iterations=affected_advisories.count(), logger=self.log) + progress = LoopProgress( + logger=self.log, total_iterations=estimated_vulnerability_count, progress_step=5 + ) updatables = [] updated_vulnerability_count = 0 batch_size = 5000 for advisory in progress.iter(affected_advisories.iterator(chunk_size=batch_size)): + references = advisory.references.all() exploits = advisory.exploits.all() - severities = AdvisorySeverity.objects.filter( - Q(advisories=advisory) | Q(advisories__related_to_advisory_severities=advisory) - ).distinct() + severities = list(advisory.severities.all()) + + for rel in advisory.related_advisory_severities.all(): + severities.extend(rel.severities.all()) - weighted_severity, exploitability = compute_vulnerability_risk_factors( - references=references, - severities=severities, - exploits=exploits, - ) - advisory.weighted_severity = weighted_severity - advisory.exploitability = exploitability - updatables.append(advisory) + + try: + weighted_severity, exploitability = compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + + advisory.weighted_severity = weighted_severity + advisory.exploitability = exploitability + if advisory.exploitability and advisory.weighted_severity: + risk_score = min(float(advisory.exploitability * advisory.weighted_severity), 10.0) + advisory.risk_score = round(risk_score, 1) + updatables.append(advisory) + except Exception as e: + self.log(f"Error computing risk score for advisory {advisory.advisory_id}: {e}") if len(updatables) >= batch_size: updated_vulnerability_count += bulk_update( model=AdvisoryV2, items=updatables, - fields=["weighted_severity", "exploitability"], + fields=["weighted_severity", "exploitability", "risk_score"], logger=self.log, ) - - updated_vulnerability_count += bulk_update( - model=AdvisoryV2, - items=updatables, - fields=["weighted_severity", "exploitability"], - logger=self.log, - ) + updatables.clear() + + if updatables: + updated_vulnerability_count += bulk_update( + model=AdvisoryV2, + items=updatables, + fields=["weighted_severity", "exploitability", "risk_score"], + logger=self.log, + ) self.log( f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" @@ -109,17 +141,19 @@ def compute_and_store_package_risk_score(self): updatables = [] updated_package_count = 0 - batch_size = 10000 + batch_size = 1000 for package in progress.iter(affected_packages.iterator(chunk_size=batch_size)): - risk_score = compute_package_risk_v2(package) - - if not risk_score: + try: + risk_score = compute_package_risk_v2(package) + if not risk_score: + continue + package.risk_score = risk_score + updatables.append(package) + except Exception as e: + self.log(f"Error computing risk score for package {package.purl}: {e}") continue - package.risk_score = risk_score - updatables.append(package) - if len(updatables) >= batch_size: updated_package_count += bulk_update( model=PackageV2, diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py index c306502d8..70afa4ef1 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py @@ -89,7 +89,7 @@ def add_vulnerability_exploit(row, logger): for adv in alias.advisories.all(): advisories.add(adv) else: - advs = AdvisoryV2.objects.filter(advisory_id=raw_alias) + advs = AdvisoryV2.objects.filter(advisory_id=raw_alias).latest_per_avid() for adv in advs: advisories.add(adv) except AdvisoryAlias.DoesNotExist: diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py index f08747b5e..6cccc4d82 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py @@ -78,7 +78,7 @@ def add_vulnerability_exploit(kev_vul, logger): for adv in alias.advisories.all(): advisories.add(adv) else: - advs = AdvisoryV2.objects.filter(advisory_id=cve_id) + advs = AdvisoryV2.objects.filter(advisory_id=cve_id).latest_per_avid() for adv in advs: advisories.add(adv) except AdvisoryAlias.DoesNotExist: diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py index fbfea5150..3ce1ff7c9 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py @@ -83,7 +83,7 @@ def add_advisory_exploit(record, logger): for adv in alias.advisories.all(): advisories.add(adv) else: - advs = AdvisoryV2.objects.filter(advisory_id=ref) + advs = AdvisoryV2.objects.filter(advisory_id=ref).latest_per_avid() for adv in advs: advisories.add(adv) except AdvisoryAlias.DoesNotExist: diff --git a/vulnerabilities/pipelines/v2_improvers/relate_severities.py b/vulnerabilities/pipelines/v2_improvers/relate_severities.py index 97a86404b..9ce3e0a30 100644 --- a/vulnerabilities/pipelines/v2_improvers/relate_severities.py +++ b/vulnerabilities/pipelines/v2_improvers/relate_severities.py @@ -61,8 +61,8 @@ def relate_severities(self): severity_score_advisories = ( AdvisoryV2.objects.filter(datasource_id__in=self.pipelines) .filter(severities__scoring_system__in=self.SUPPORTED_SYSTEMS) - .distinct() .latest_per_avid() + .distinct() ) total = severity_score_advisories.count() @@ -70,14 +70,21 @@ def relate_severities(self): advisory_id_map = {} - qs = AdvisoryV2.objects.filter( - advisory_id__in=severity_score_advisories.values("advisory_id") - ).values("id", "advisory_id") - - alias_qs = AdvisoryV2.objects.filter( - aliases__alias__in=severity_score_advisories.values("advisory_id") - ).values("id", "aliases__alias") + qs = ( + AdvisoryV2.objects.filter( + advisory_id__in=severity_score_advisories.values("advisory_id") + ) + .latest_per_avid() + .values("id", "advisory_id") + ) + alias_qs = ( + AdvisoryV2.objects.filter( + aliases__alias__in=severity_score_advisories.values("advisory_id") + ) + .latest_per_avid() + .values("id", "aliases__alias") + ) for row in qs: advisory_id_map.setdefault(row["advisory_id"], set()).add(row["id"]) diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 0628422bb..471828f0d 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -8,9 +8,11 @@ # from urllib.parse import urlparse -from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import AdvisoryV2, VulnerabilityReference from vulnerabilities.severity_systems import EPSS from vulnerabilities.weight_config import WEIGHT_CONFIG +from django.db.models import Max + DEFAULT_WEIGHT = 5 @@ -123,12 +125,14 @@ def compute_package_risk_v2(package): Calculate the risk for a package by iterating over all vulnerabilities that affects this package and determining the associated risk. """ - result = [] - for impact in package.affected_in_impacts.all(): - if risk := impact.advisory.risk_score: - result.append(float(risk)) - if not result: + max_risk = ( + AdvisoryV2.objects + .latest_affecting_advisories_for_purl(package.purl) + .aggregate(max_risk=Max("risk_score")) + )["max_risk"] + + if max_risk is None: return - return round(max(result), 1) + return round(float(max_risk), 1) From 34f20831b8b8311baf3edddde33dff59658f8003 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 14:50:25 +0530 Subject: [PATCH 12/65] Fix formatting issues Signed-off-by: Tushar Goel --- .../pipelines/v2_improvers/compute_package_risk.py | 5 +++-- vulnerabilities/risk.py | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index e625176bb..43ecded0d 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -90,7 +90,6 @@ def compute_and_store_vulnerability_risk_score(self): for rel in advisory.related_advisory_severities.all(): severities.extend(rel.severities.all()) - try: weighted_severity, exploitability = compute_vulnerability_risk_factors( references=references, @@ -101,7 +100,9 @@ def compute_and_store_vulnerability_risk_score(self): advisory.weighted_severity = weighted_severity advisory.exploitability = exploitability if advisory.exploitability and advisory.weighted_severity: - risk_score = min(float(advisory.exploitability * advisory.weighted_severity), 10.0) + risk_score = min( + float(advisory.exploitability * advisory.weighted_severity), 10.0 + ) advisory.risk_score = round(risk_score, 1) updatables.append(advisory) except Exception as e: diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py index 471828f0d..dd7401d80 100644 --- a/vulnerabilities/risk.py +++ b/vulnerabilities/risk.py @@ -8,11 +8,12 @@ # from urllib.parse import urlparse -from vulnerabilities.models import AdvisoryV2, VulnerabilityReference -from vulnerabilities.severity_systems import EPSS -from vulnerabilities.weight_config import WEIGHT_CONFIG from django.db.models import Max +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.weight_config import WEIGHT_CONFIG DEFAULT_WEIGHT = 5 @@ -127,9 +128,9 @@ def compute_package_risk_v2(package): """ max_risk = ( - AdvisoryV2.objects - .latest_affecting_advisories_for_purl(package.purl) - .aggregate(max_risk=Max("risk_score")) + AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.purl).aggregate( + max_risk=Max("risk_score") + ) )["max_risk"] if max_risk is None: From 22493b22dd527a71aeebd5d647e352b6c9d8a19a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 15:11:59 +0530 Subject: [PATCH 13/65] Optimize package risk score calculation Signed-off-by: Tushar Goel --- .../v2_improvers/compute_package_risk.py | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index 43ecded0d..bfd77fc23 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # from aboutcode.pipeline import LoopProgress +from django.db.models import Max from django.db.models import Prefetch from vulnerabilities.models import AdvisoryExploit @@ -15,7 +16,6 @@ from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.risk import compute_package_risk_v2 from vulnerabilities.risk import compute_vulnerability_risk_factors @@ -130,45 +130,47 @@ def compute_and_store_vulnerability_risk_score(self): ) def compute_and_store_package_risk_score(self): - affected_packages = (PackageV2.objects.filter(affected_in_impacts__isnull=False)).distinct() + qs = ( + PackageV2.objects.filter(affected_in_impacts__advisory__risk_score__isnull=False) + .annotate(computed_risk=Max("affected_in_impacts__advisory__risk_score")) + .only("id") + ) - self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") + estimated = qs.count() progress = LoopProgress( - total_iterations=affected_packages.count(), + total_iterations=estimated, logger=self.log, progress_step=5, ) - updatables = [] - updated_package_count = 0 - batch_size = 1000 + self.log(f"Computing risk for {estimated:,d} packages") - for package in progress.iter(affected_packages.iterator(chunk_size=batch_size)): - try: - risk_score = compute_package_risk_v2(package) - if not risk_score: - continue - package.risk_score = risk_score - updatables.append(package) - except Exception as e: - self.log(f"Error computing risk score for package {package.purl}: {e}") - continue + batch = [] + batch_size = 5000 + updated = 0 - if len(updatables) >= batch_size: - updated_package_count += bulk_update( + for pkg in progress.iter(qs.iterator(chunk_size=batch_size)): + + pkg.risk_score = round(float(pkg.computed_risk), 1) + batch.append(pkg) + + if len(batch) >= batch_size: + updated += bulk_update( model=PackageV2, - items=updatables, + items=batch, fields=["risk_score"], logger=self.log, ) - updated_package_count += bulk_update( + batch.clear() + + updated += bulk_update( model=PackageV2, - items=updatables, + items=batch, fields=["risk_score"], logger=self.log, ) - self.log(f"Successfully added risk score for {updated_package_count:,d} package") + self.log(f"Successfully added risk score for {updated:,d} package") def bulk_update(model, items, fields, logger): From eded0653e879071fbe6b570fe3745c2340f2c2c9 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 15:25:37 +0530 Subject: [PATCH 14/65] Use only latest per avid aadvisories to compute package risk score Signed-off-by: Tushar Goel --- .../pipelines/v2_improvers/compute_package_risk.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index bfd77fc23..dacf7e6c8 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -130,8 +130,14 @@ def compute_and_store_vulnerability_risk_score(self): ) def compute_and_store_package_risk_score(self): + + latest_advisories = AdvisoryV2.objects.latest_per_avid() + qs = ( - PackageV2.objects.filter(affected_in_impacts__advisory__risk_score__isnull=False) + PackageV2.objects.filter( + affected_in_impacts__advisory__risk_score__isnull=False, + affected_in_impacts__advisory__in=latest_advisories, + ) .annotate(computed_risk=Max("affected_in_impacts__advisory__risk_score")) .only("id") ) From 7847c348a7e1e89139d5359ce6b5bf775b5cca47 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 17:36:49 +0530 Subject: [PATCH 15/65] Handle packages which are subject of more than 100 advisories Signed-off-by: Tushar Goel --- .../templates/affected_by_advisories.html | 116 +++++++++ .../templates/fixing_advisories.html | 76 ++++++ .../templates/package_details_v2.html | 27 +- vulnerabilities/views.py | 238 ++++++++++-------- 4 files changed, 348 insertions(+), 109 deletions(-) create mode 100644 vulnerabilities/templates/affected_by_advisories.html create mode 100644 vulnerabilities/templates/fixing_advisories.html diff --git a/vulnerabilities/templates/affected_by_advisories.html b/vulnerabilities/templates/affected_by_advisories.html new file mode 100644 index 000000000..01721b84f --- /dev/null +++ b/vulnerabilities/templates/affected_by_advisories.html @@ -0,0 +1,116 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} +{% load static %} +{% load url_filters %} +{% load utils %} + +{% block content %} +
+
+
+
+ {{ page_obj.paginator.count|intcomma }} results +
+ {% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+
+
+ +
+
+ + + + + + + + + + + + + {% for advisory in page_obj %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryFixed in package version
+ + {{advisory.avid }} + +
+ {% if advisory.alias|length != 0 %} + Aliases: + {% endif %} +
+ {% for alias in advisory.alias %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} + + {% if advisory.secondary|length != 0 %} +

Supporting advisories are listed below the primary advisory.

+ {% for secondary in advisory.secondary %} + + {{secondary.avid }} + + {% endfor %} + {% endif %} + 		+ {{advisory.url}} + + {{advisory.date_published}} + + {{ advisory.summary }} + + {% with fixed=fixed_package_details|get_item:advisory.avid %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ {% if item.pkg.is_vulnerable %} + + Vulnerable + + {% else %} + + Not vulnerable + + {% endif %} +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} +
+ This package is not known to be subject of any advisories. +
+
+ +{% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} +{% endif %} +{% endblock %} +
diff --git a/vulnerabilities/templates/fixing_advisories.html b/vulnerabilities/templates/fixing_advisories.html new file mode 100644 index 000000000..64af4fc65 --- /dev/null +++ b/vulnerabilities/templates/fixing_advisories.html @@ -0,0 +1,76 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} + +{% block content %} +
+
+
+
+ {{ page_obj.paginator.count|intcomma }} results +
+ {% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} + {% endif %} +
+
+
+ +
+
+ + + + + + + + + + + + {% for advisory in page_obj %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryAliases
+ + {{advisory.avid }} + + + {{advisory.url}} + + {{advisory.date_published}} + + {{ advisory.summary }} + + {% for alias in advisory.alias %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
+ This package is not known to fix any advisories. +
+
+ +{% if is_paginated %} + {% include 'includes/pagination.html' with page_obj=page_obj %} +{% endif %} +{% endblock %} +
diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index 38d6c2e9f..f90585b9d 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -45,7 +45,7 @@
- {% if affected_by_advisories_v2|length != 0 %} + {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %}
{% else %}
@@ -82,7 +82,7 @@
- {% if affected_by_advisories_v2|length != 0 %} + {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %}
@@ -128,10 +128,10 @@ {% endif %}
+ {% if affected_by_advisories_v2|length != 0 %}
Vulnerabilities affecting this package ({{ affected_by_advisories_v2|length }})
-
@@ -218,9 +218,20 @@ {% endfor %}
+ {% elif affected_by_advisories_v2_url %} +
+ This package is subject to more than 100 advisories. Please refer to the following + URL for vulnerabilities affecting this package: Advisories +
+ {% else %} +
+ This package is not known to be subject of any advisories. +
+ {% endif %}
+ {% if fixing_advisories_v2|length != 0 %}
Vulnerabilities fixed by this package ({{ fixing_advisories_v2|length }})
@@ -285,6 +296,16 @@
+ {% elif fixing_advisories_v2_url %} +
+ This package is known to fix more than 100 advisories. Please refer to the following + URL for vulnerabilities fixed by this package: Advisories +
+ {% else %} +
+ This package is not known to fix any advisories. +
+ {% endif %}
diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 6d78b650e..ed6f2e18c 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -78,34 +78,6 @@ def get_queryset(self, query=None): ) -class PackageSearchV2(ListView): - model = models.PackageV2 - template_name = "packages_v2.html" - ordering = ["type", "namespace", "name", "version"] - paginate_by = PAGE_SIZE - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - request_query = self.request.GET - context["package_search_form"] = PackageSearchForm(request_query) - context["search"] = request_query.get("search") - return context - - def get_queryset(self, query=None): - """ - Return a Package queryset for the ``query``. - Make a best effort approach to find matching packages either based - on exact purl, partial purl or just name and namespace. - """ - query = query or self.request.GET.get("search") or "" - return ( - self.model.objects.search(query) - .prefetch_related() - .order_by("package_url") - .with_is_vulnerable() - ) - - class VulnerabilitySearch(ListView): model = models.Vulnerability template_name = "vulnerabilities.html" @@ -124,24 +96,6 @@ def get_queryset(self, query=None): return self.model.objects.search(query=query).with_package_counts() -class AdvisorySearch(ListView): - model = models.AdvisoryV2 - template_name = "vulnerabilities.html" - ordering = ["advisory_id"] - paginate_by = PAGE_SIZE - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - request_query = self.request.GET - context["advisory_search_form"] = VulnerabilitySearchForm(request_query) - context["search"] = request_query.get("search") - return context - - def get_queryset(self, query=None): - query = query or self.request.GET.get("search") or "" - return self.model.objects.search(query=query).with_package_counts() - - class PackageDetails(DetailView): model = models.Package template_name = "package_details.html" @@ -183,93 +137,128 @@ def get_object(self, queryset=None): return package -class PackageV2Details(DetailView): +class PackageSearchV2(ListView): model = models.PackageV2 - template_name = "package_details_v2.html" - slug_url_kwarg = "purl" - slug_field = "purl" + template_name = "packages_v2.html" + ordering = ["type", "namespace", "name", "version"] + paginate_by = PAGE_SIZE def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - package = self.object - next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() + request_query = self.request.GET + context["package_search_form"] = PackageSearchForm(request_query) + context["search"] = request_query.get("search") + return context - ( - fixed_pkg_details, - affected_by_advisories, - fixing_advisories, - ) = self.get_fixed_package_details(package) + def get_queryset(self, query=None): + """ + Return a Package queryset for the ``query``. + Make a best effort approach to find matching packages either based + on exact purl, partial purl or just name and namespace. + """ + query = query or self.request.GET.get("search") or "" + return ( + self.model.objects.search(query) + .prefetch_related() + .order_by("package_url") + .with_is_vulnerable() + ) - affected_avid_by_hash = {} - fixing_avid_by_hash = {} - affected_avid_by_hash = group_advisories_by_content(affected_by_advisories) - fixing_avid_by_hash = group_advisories_by_content(fixing_advisories) +class AffectedByAdvisoriesListView(ListView): + model = models.AdvisoryV2 + template_name = "affected_by_advisories.html" + paginate_by = PAGE_SIZE + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + purl = self.kwargs.get("purl") + package = models.PackageV2.objects.for_purl(purl).first() + context["fixed_package_details"] = get_fixed_package_details(package) + return context + + def get_queryset(self): + purl = self.kwargs.get("purl") + print(purl) + return models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl) - affecting_advs = [] - for hash in affected_avid_by_hash: - affecting_advs.append(affected_avid_by_hash[hash]) +class FixingAdvisoriesListView(ListView): + model = models.AdvisoryV2 + template_name = "fixing_advisories.html" + paginate_by = PAGE_SIZE - fixing_advs = [] + def get_queryset(self): + purl = self.kwargs.get("purl") + return models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(purl) - for hash in fixing_avid_by_hash: - fixing_advs.append(fixing_avid_by_hash[hash]) + +class PackageV2Details(DetailView): + model = models.PackageV2 + template_name = "package_details_v2.html" + slug_url_kwarg = "purl" + slug_field = "purl" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + package = self.object + + next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() context["package"] = package context["next_non_vulnerable"] = next_non_vulnerable context["latest_non_vulnerable"] = latest_non_vulnerable - context["affected_by_advisories_v2"] = affecting_advs - context["fixing_advisories_v2"] = fixing_advs - context["package_search_form"] = PackageSearchForm(self.request.GET) - context["fixed_package_details"] = fixed_pkg_details - return context + affected_by_advisories = models.AdvisoryV2.objects.latest_affecting_advisories_for_purl( + package.package_url + ) - def get_fixed_package_details(self, package): - rows = package.affected_in_impacts.values_list( - "advisory__avid", - "fixed_by_packages", + fixing_advisories = models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + package.package_url ) - pkg_ids = {pkg_id for _, pkg_id in rows if pkg_id} + affected_by_advisories_url = None + fixing_advisories_url = None - pkg_map = { - p.id: p - for p in models.PackageV2.objects.filter(id__in=pkg_ids).annotate( - is_vulnerable=Exists( - models.ImpactedPackage.objects.filter(affecting_packages=OuterRef("pk")) - ) + if affected_by_advisories.count() > 100: + affected_by_advisories_url = reverse_lazy( + "affected_by_advisories_v2", kwargs={"purl": package.package_url} ) - } + context["affected_by_advisories_v2_url"] = affected_by_advisories_url + context["affected_by_advisories_v2"] = [] + context["fixed_package_details"] = {} - fixed_pkg_details = defaultdict(list) - - for avid, pkg_id in rows: - if not pkg_id: - continue - - pkg = pkg_map.get(pkg_id) - if not pkg: - continue - - fixed_pkg_details[avid].append( - { - "pkg": pkg, - "is_vulnerable": pkg.is_vulnerable, - } + else: + fixed_pkg_details = get_fixed_package_details(package) + affected_avid_by_hash = {} + affected_avid_by_hash = group_advisories_by_content(affected_by_advisories) + affecting_advs = [] + + for hash in affected_avid_by_hash: + affecting_advs.append(affected_avid_by_hash[hash]) + context["affected_by_advisories_v2"] = affecting_advs + context["fixed_package_details"] = fixed_pkg_details + context["affected_by_advisories_v2_url"] = None + + if fixing_advisories.count() > 100: + fixing_advisories_url = reverse_lazy( + "fixing_advisories_v2", kwargs={"purl": package.package_url} ) + context["fixing_advisories_v2_url"] = fixing_advisories_url + context["fixing_advisories_v2"] = [] - affected_by_advisories = models.AdvisoryV2.objects.latest_affecting_advisories_for_purl( - package.package_url - ) + else: + fixing_avid_by_hash = {} + fixing_avid_by_hash = group_advisories_by_content(fixing_advisories) + fixing_advs = [] - fixing_advisories = models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - package.package_url - ) + for hash in fixing_avid_by_hash: + fixing_advs.append(fixing_avid_by_hash[hash]) + context["fixing_advisories_v2"] = fixing_advs + context["fixing_advisories_v2_url"] = None - return fixed_pkg_details, affected_by_advisories, fixing_advisories + return context def get_queryset(self): return ( @@ -307,6 +296,43 @@ def get_object(self, queryset=None): return package +def get_fixed_package_details(package): + rows = package.affected_in_impacts.values_list( + "advisory__avid", + "fixed_by_packages", + ) + + pkg_ids = {pkg_id for _, pkg_id in rows if pkg_id} + + pkg_map = { + p.id: p + for p in models.PackageV2.objects.filter(id__in=pkg_ids).annotate( + is_vulnerable=Exists( + models.ImpactedPackage.objects.filter(affecting_packages=OuterRef("pk")) + ) + ) + } + + fixed_pkg_details = defaultdict(list) + + for avid, pkg_id in rows: + if not pkg_id: + continue + + pkg = pkg_map.get(pkg_id) + if not pkg: + continue + + fixed_pkg_details[avid].append( + { + "pkg": pkg, + "is_vulnerable": pkg.is_vulnerable, + } + ) + + return fixed_pkg_details + + class VulnerabilityDetails(DetailView): model = models.Vulnerability template_name = "vulnerability_details.html" From d189bf6181f3ae03ee06a7213ca5dadd0a982834 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 17:52:10 +0530 Subject: [PATCH 16/65] Add URLs Signed-off-by: Tushar Goel --- vulnerablecode/urls.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 112678a24..eb1bc006b 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -32,7 +32,9 @@ from vulnerabilities.views import AdminLoginView from vulnerabilities.views import AdvisoryDetails from vulnerabilities.views import AdvisoryPackagesDetails +from vulnerabilities.views import AffectedByAdvisoriesListView from vulnerabilities.views import ApiUserCreateView +from vulnerabilities.views import FixingAdvisoriesListView from vulnerabilities.views import HomePage from vulnerabilities.views import HomePageV2 from vulnerabilities.views import PackageDetails @@ -142,6 +144,16 @@ def __init__(self, *args, **kwargs): PackageV2Details.as_view(), name="package_details_v2", ), + re_path( + r"^fixing-advisories/v2/(?Ppkg:.+)$", + FixingAdvisoriesListView.as_view(), + name="fixing_advisories_v2", + ), + re_path( + r"^affected-by-advisories/v2/(?Ppkg:.+)$", + AffectedByAdvisoriesListView.as_view(), + name="affected_by_advisories_v2", + ), path( "vulnerabilities/search/", VulnerabilitySearch.as_view(), From 7f80d661fd1194fac0cd658eb7f9537c72e14f84 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 18:34:02 +0530 Subject: [PATCH 17/65] Optimize views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index ed6f2e18c..cf9b4ccf0 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -210,18 +210,19 @@ def get_context_data(self, **kwargs): context["latest_non_vulnerable"] = latest_non_vulnerable context["package_search_form"] = PackageSearchForm(self.request.GET) - affected_by_advisories = models.AdvisoryV2.objects.latest_affecting_advisories_for_purl( + affected_by_advisories_qs = models.AdvisoryV2.objects.latest_affecting_advisories_for_purl( package.package_url ) - fixing_advisories = models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + fixing_advisories_qs = models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( package.package_url ) affected_by_advisories_url = None fixing_advisories_url = None - if affected_by_advisories.count() > 100: + affected_by_advisories = list(affected_by_advisories_qs[:101]) + if len(affected_by_advisories) > 100: affected_by_advisories_url = reverse_lazy( "affected_by_advisories_v2", kwargs={"purl": package.package_url} ) @@ -232,7 +233,7 @@ def get_context_data(self, **kwargs): else: fixed_pkg_details = get_fixed_package_details(package) affected_avid_by_hash = {} - affected_avid_by_hash = group_advisories_by_content(affected_by_advisories) + affected_avid_by_hash = group_advisories_by_content(affected_by_advisories_qs) affecting_advs = [] for hash in affected_avid_by_hash: @@ -241,7 +242,8 @@ def get_context_data(self, **kwargs): context["fixed_package_details"] = fixed_pkg_details context["affected_by_advisories_v2_url"] = None - if fixing_advisories.count() > 100: + fixing_advisories = list(fixing_advisories_qs[:101]) + if len(fixing_advisories) > 100: fixing_advisories_url = reverse_lazy( "fixing_advisories_v2", kwargs={"purl": package.package_url} ) @@ -250,7 +252,7 @@ def get_context_data(self, **kwargs): else: fixing_avid_by_hash = {} - fixing_avid_by_hash = group_advisories_by_content(fixing_advisories) + fixing_avid_by_hash = group_advisories_by_content(fixing_advisories_qs) fixing_advs = [] for hash in fixing_avid_by_hash: From ed19a1f3ffba690443381295024a837f1a48b1b9 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 17 Mar 2026 18:40:27 +0530 Subject: [PATCH 18/65] Fix rst file formatting Signed-off-by: Tushar Goel --- api_v3_usage.rst | 582 ++++++++++++++++++++--------------------------- 1 file changed, 245 insertions(+), 337 deletions(-) diff --git a/api_v3_usage.rst b/api_v3_usage.rst index ae8c38b7f..26ed9377f 100644 --- a/api_v3_usage.rst +++ b/api_v3_usage.rst @@ -1,338 +1,246 @@ +Package Endpoint +================ -Package endpoint ------------------- - -We are moving from API v1 to API V3. - -- /api/packages earlier had "bulk_search", "bulk_lookup", "lookup" and "all" endpoints. - -- /api/v3/packages has only one endpoint, which have same capabilities as all of these endpoints. - -- Response by package endpoint, will always be paginated, with 10 results per page, and will have "next" and "previous" links for pagination. If there are more than 100 advisories for a package, then it will return "affected_by_vulnerabilities_url" and "fixing_vulnerabilities_url" instead of "affected_by_vulnerabilities" and "fixing_vulnerabilities" respectively. - -"all" - -- Instead of doing /api/packages/all, we can do /api/v3/packages with empty purls list. - -- To get all vulnerable packages: - -``` -POST /api/v3/packages -{ - "purls": [] -} -``` - -Response: - -``` - -{ - "count": 596, - "next": "http://example.com/api/v3/packages?page=2", - "previous": null, - "results": [ - "pkg:npm/626@1.1.1", - "pkg:npm/aedes@0.35.0", - "pkg:npm/airbrake@0.3.8", - "pkg:npm/angular-http-server@1.4.3", - "pkg:npm/apex-publish-static-files@2.0.0", - "pkg:npm/atob@2.0.3", - "pkg:npm/augustine@0.2.3", - "pkg:npm/backbone@0.3.3", - "pkg:npm/base64-url@1.3.3", - "pkg:npm/base64url@2.0.0" - ] -} -``` - - -"bulk_search" - -- Instead of doing /api/packages/bulk_search, we can do /api/v3/packages with purls list and "details" as false or true (by default it's false), earlier we had "purls_only" . Also, previosuly we used to have "plain_purl" as a parameter, to ignore qualifiers and subpaths in purls, now we have "approximate", if set to True will ignore qualifiers and subpaths in purls. - -Examples: - -- To get only purls of vulnerable packages: -``` -POST /api/v3/packages -{ - "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], - "details": false -} -``` - -Response: - -``` -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - "pkg:npm/atob@2.0.3" - ] -} - -``` - -- To get details of vulnerable packages: -``` -POST /api/v3/packages -{ - "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], - "details": true -} -``` - -Response: -``` - -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - { - "purl": "pkg:npm/atob@2.0.3", - "affected_by_vulnerabilities": [ - { - "advisory_id": "nodejs_security_wg/npm-403", - "fixed_by_packages": [ - "pkg:npm/atob@2.1.0" - ], - "duplicate_advisory_ids": [] - } - ], - "fixing_vulnerabilities": [], - "next_non_vulnerable_version": "2.1.0", - "latest_non_vulnerable_version": "2.1.0", - "risk_score": null - } - ] -} -``` - -- To get details of vulnerable packages by ignoring qualifiers and subpaths in purls: -``` -POST /api/v3/packages -{ - "purls": ["pkg:npm/atob@2.0.3?foo=bar", "pkg:pypi/sample@2.0.0"], - "approximate": true, - "details": true -} -``` - -Response: -``` - -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - { - "purl": "pkg:npm/atob@2.0.3", - "affected_by_vulnerabilities": [ - { - "advisory_id": "nodejs_security_wg/npm-403", - "fixed_by_packages": [ - "pkg:npm/atob@2.1.0" - ], - "duplicate_advisory_ids": [] - } - ], - "fixing_vulnerabilities": [], - "next_non_vulnerable_version": "2.1.0", - "latest_non_vulnerable_version": "2.1.0", - "risk_score": null - } - ] -} -``` - -- To get vulnerable packages by ignoring qualifiers and subpaths in purls: -``` -POST /api/v3/packages -{ - "purls": ["pkg:npm/atob@2.0.3?foo=bar"], - "approximate": true, -} -``` - -Response: - -``` -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - "pkg:npm/atob@2.0.3" - ] -} - -``` - -Advisory endpoint ------------------- - -- You can get all advisories for a purl or list of purls by using /api/v3/advisories endpoint. It will also be paginated with 10 results per page, and will have "next" and "previous" links for pagination - -``` -POST /api/v3/advisories -{ - "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"] -} -``` - -Response: - -``` -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - { - "advisory_id": "nodejs_security_wg/npm-403", - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "aliases": [ - "CVE-2018-3745" - ], - "summary": "Out-of-bounds Read\n`atob` allocates uninitialized Buffers when number is passed in input on Node.js 4.x and below", - "severities": [ - { - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "value": "6.5", - "scoring_system": "cvssv3", - "scoring_elements": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:H" - } - ], - "weaknesses": [], - "references": [ - { - "url": "https://hackerone.com/reports/321686", - "reference_type": "", - "reference_id": "" - }, - { - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "reference_type": "", - "reference_id": "403" - } - ], - "exploitability": null, - "weighted_severity": null, - "risk_score": null, - "related_ssvc_trees": [] - } - ] -} -``` - -Affected By Advisories endpoint --------------------------------------- - -- You can get all advisories that fix a purl by using /api/v3/affected-by-advisories?purl= endpoint - -``` -GET /api/v3/affected-by-advisories?purl=pkg:npm/atob@2.0.3 -``` - -Response: -``` -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - { - "advisory_id": "nodejs_security_wg/npm-403", - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "aliases": [ - "CVE-2018-3745" - ], - "summary": "Out-of-bounds Read\n`atob` allocates uninitialized Buffers when number is passed in input on Node.js 4.x and below", - "severities": [ - { - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "value": "6.5", - "scoring_system": "cvssv3", - "scoring_elements": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:H" - } - ], - "weaknesses": [], - "references": [ - { - "url": "https://hackerone.com/reports/321686", - "reference_type": "", - "reference_id": "" - }, - { - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "reference_type": "", - "reference_id": "403" - } - ], - "exploitability": null, - "weighted_severity": null, - "risk_score": null, - "related_ssvc_trees": [] - } - ] -} -``` - -Fixing Advisories endpoint ------------------------------ - -- You can get all advisories that are fixed by a purl by using /api/v3/fixing-advisories?purl= endpoint - -``` -GET /api/v3/fixing-advisories?purl=pkg:npm/atob@2.1.0 -``` - -Response: -``` -{ - "count": 1, - "next": null, - "previous": null, - "results": [ - { - "advisory_id": "nodejs_security_wg/npm-403", - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "aliases": [ - "CVE-2018-3745" - ], - "summary": "Out-of-bounds Read\n`atob` allocates uninitialized Buffers when number is passed in input on Node.js 4.x and below", - "severities": [ - { - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "value": "6.5", - "scoring_system": "cvssv3", - "scoring_elements": "CVSS:3.0/AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:H" - } - ], - "weaknesses": [], - "references": [ - { - "url": "https://hackerone.com/reports/321686", - "reference_type": "", - "reference_id": "" - }, - { - "url": "https://github.com/nodejs/security-wg/blob/main/vuln/npm/403.json", - "reference_type": "", - "reference_id": "403" - } - ], - "exploitability": null, - "weighted_severity": null, - "risk_score": null, - "related_ssvc_trees": [] - } - ] -} -``` +We are migrating from **API v1** to **API v3**. + +Previously, the ``/api/packages`` endpoint exposed multiple routes: + +- ``bulk_search`` +- ``bulk_lookup`` +- ``lookup`` +- ``all`` + +In **API v3**, all these capabilities are consolidated into a **single endpoint**: + +:: + + POST /api/v3/packages + + +Pagination +---------- + +Responses from the package endpoint are **always paginated**, with **10 results per page**. + +Each response includes: + +- ``count`` — total number of results +- ``next`` — URL for the next page +- ``previous`` — URL for the previous page + +If a package is associated with **more than 100 advisories**, the response will include: + +- ``affected_by_vulnerabilities_url`` instead of ``affected_by_vulnerabilities`` +- ``fixing_vulnerabilities_url`` instead of ``fixing_vulnerabilities`` + + +Getting All Vulnerable Packages +------------------------------- + +Instead of calling ``/api/packages/all``, call the v3 endpoint with an empty ``purls`` list. + +:: + + POST /api/v3/packages + + { + "purls": [] + } + +Example response: + +:: + + { + "count": 596, + "next": "http://example.com/api/v3/packages?page=2", + "previous": null, + "results": [ + "pkg:npm/626@1.1.1", + "pkg:npm/aedes@0.35.0", + "pkg:npm/airbrake@0.3.8", + "pkg:npm/angular-http-server@1.4.3", + "pkg:npm/apex-publish-static-files@2.0.0", + "pkg:npm/atob@2.0.3", + "pkg:npm/augustine@0.2.3", + "pkg:npm/backbone@0.3.3", + "pkg:npm/base64-url@1.3.3", + "pkg:npm/base64url@2.0.0" + ] + } + + +Bulk Search (Replacement) +------------------------- + +Instead of calling ``/api/packages/bulk_search``, use: + +:: + + POST /api/v3/packages + +Parameters: + +- ``purls`` — list of package URLs to query +- ``details`` — boolean (default: ``false``) +- ``approximate`` — boolean (default: ``false``) + +The ``approximate`` flag replaces the previous ``plain_purl`` parameter. +When set to ``true``, qualifiers and subpaths in PURLs are ignored. + + +Get Only Vulnerable PURLs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + POST /api/v3/packages + + { + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], + "details": false + } + +Example response: + +:: + + { + "count": 1, + "next": null, + "previous": null, + "results": [ + "pkg:npm/atob@2.0.3" + ] + } + + +Get Detailed Vulnerability Information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + POST /api/v3/packages + + { + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"], + "details": true + } + +Example response: + +:: + + { + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "purl": "pkg:npm/atob@2.0.3", + "affected_by_vulnerabilities": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "fixed_by_packages": [ + "pkg:npm/atob@2.1.0" + ], + "duplicate_advisory_ids": [] + } + ], + "fixing_vulnerabilities": [], + "next_non_vulnerable_version": "2.1.0", + "latest_non_vulnerable_version": "2.1.0", + "risk_score": null + } + ] + } + + +Using Approximate Matching +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + POST /api/v3/packages + + { + "purls": ["pkg:npm/atob@2.0.3?foo=bar"], + "approximate": true, + "details": true + } + +Example response: + +:: + + { + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "purl": "pkg:npm/atob@2.0.3", + "affected_by_vulnerabilities": [ + { + "advisory_id": "nodejs_security_wg/npm-403", + "fixed_by_packages": [ + "pkg:npm/atob@2.1.0" + ], + "duplicate_advisory_ids": [] + } + ], + "fixing_vulnerabilities": [], + "next_non_vulnerable_version": "2.1.0", + "latest_non_vulnerable_version": "2.1.0", + "risk_score": null + } + ] + } + + +Advisory Endpoint +================= + +Retrieve advisories for one or more PURLs: + +:: + + POST /api/v3/advisories + + { + "purls": ["pkg:npm/atob@2.0.3", "pkg:pypi/sample@2.0.0"] + } + +Responses are paginated (10 results per page) and include ``next`` and ``previous`` links. + + +Affected-By Advisories Endpoint +=============================== + +Retrieve advisories that **affect (impact)** a given PURL: + +:: + + GET /api/v3/affected-by-advisories?purl= + +Example: + +:: + + GET /api/v3/affected-by-advisories?purl=pkg:npm/atob@2.0.3 + + +Fixing Advisories Endpoint +========================== + +Retrieve advisories that are **fixed by** a given PURL: + +:: + + GET /api/v3/fixing-advisories?purl= + +Example: + +:: + + GET /api/v3/fixing-advisories?purl=pkg:npm/atob@2.1.0 \ No newline at end of file From efe89349bb7a4346a8ecf18ec82ecbb49cc40cfd Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 15:19:43 +0530 Subject: [PATCH 19/65] Optimize latest advisories for purls Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 23 +++++++++++++++++++---- vulnerabilities/views.py | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 7fa162022..b0900627c 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2901,10 +2901,25 @@ def latest_advisories_for_purl(self, purl): ).latest_per_avid() def latest_advisories_for_purls(self, purls): - return self.filter( - Q(impacted_packages__affecting_packages__package_url__in=purls) - | Q(impacted_packages__fixed_by_packages__package_url__in=purls) - ).latest_per_avid() + + affecting = ImpactedPackageAffecting.objects.filter( + impacted_package__advisory_id=OuterRef("pk"), + package__package_url__in=purls, + ) + + fixed = ImpactedPackageFixedBy.objects.filter( + impacted_package__advisory_id=OuterRef("pk"), + package__package_url__in=purls, + ) + + return ( + self.annotate( + has_affecting=Exists(affecting), + has_fixed=Exists(fixed), + ) + .filter(Q(has_affecting=True) | Q(has_fixed=True)) + .latest_per_avid() + ) class AdvisoryV2(models.Model): diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index cf9b4ccf0..8765c30ec 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -47,7 +47,7 @@ from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env -PAGE_SIZE = 20 +PAGE_SIZE = 10 class PackageSearch(ListView): From a206788da4a462dab2e689e8f4355ef48036e98f Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 15:30:37 +0530 Subject: [PATCH 20/65] Optimize AdvisoryV2QuerySet Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 61 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index b0900627c..80bd2e13d 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2862,37 +2862,58 @@ def latest_for_avid(self, avid: str): ) def latest_per_avid(self): - latest_ids = ( - self.filter(avid=OuterRef("avid")) - .order_by( - F("date_collected").desc(nulls_last=True), - "-id", - ) - .values("id")[:1] - ) - - return self.filter(id=Subquery(latest_ids)) + return self.order_by( + "avid", + F("date_collected").desc(nulls_last=True), + "-id", + ).distinct("avid") def latest_for_avids(self, avids): return self.filter(avid__in=avids).latest_per_avid() def latest_affecting_advisories_for_purl(self, purl): - return self.filter( - impacted_packages__affecting_packages__package_url=purl - ).latest_per_avid() + affecting_exists = ImpactedPackageAffecting.objects.filter( + impacted_package__advisory_id=OuterRef("pk"), + package__package_url=purl, + ) + + return ( + self.annotate(has_affecting=Exists(affecting_exists)) + .filter(has_affecting=True) + .latest_per_avid() + ) def latest_affecting_advisories_for_purls(self, purls): - return self.filter( - impacted_packages__affecting_packages__package_url__in=purls - ).latest_per_avid() + affecting_exists = ImpactedPackageAffecting.objects.filter( + impacted_package__advisory_id=OuterRef("pk"), + package__package_url__in=purls, + ) + + return ( + self.annotate(has_affecting=Exists(affecting_exists)) + .filter(has_affecting=True) + .latest_per_avid() + ) def latest_fixed_by_advisories_for_purl(self, purl): - return self.filter(impacted_packages__fixed_by_packages__package_url=purl).latest_per_avid() + fixed_exists = ImpactedPackageFixedBy.objects.filter( + impacted_package__advisory_id=OuterRef("pk"), + package__package_url=purl, + ) + + return ( + self.annotate(has_fixed=Exists(fixed_exists)).filter(has_fixed=True).latest_per_avid() + ) def latest_fixed_by_advisories_for_purls(self, purls): - return self.filter( - impacted_packages__fixed_by_packages__package_url__in=purls - ).latest_per_avid() + fixed_exists = ImpactedPackageFixedBy.objects.filter( + impacted_package__advisory_id=OuterRef("pk"), + package__package_url__in=purls, + ) + + return ( + self.annotate(has_fixed=Exists(fixed_exists)).filter(has_fixed=True).latest_per_avid() + ) def latest_advisories_for_purl(self, purl): return self.filter( From 3cc61561b6cef529710fdda994535956a69a5362 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 15:56:19 +0530 Subject: [PATCH 21/65] Optimize queries Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 43 ++++++++++++++++++++++++---- vulnerabilities/tests/test_api_v3.py | 2 +- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 2549aacc9..cd47c9697 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -9,6 +9,7 @@ from urllib.parse import urlencode +from django.db.models import Prefetch from django_filters import rest_framework as filters from packageurl import PackageURL from rest_framework import serializers @@ -90,7 +91,11 @@ class Meta: class AdvisoryV3Serializer(serializers.ModelSerializer): - aliases = serializers.SerializerMethodField() + aliases = serializers.SlugRelatedField( + many=True, + read_only=True, + slug_field="alias", + ) weaknesses = AdvisoryWeaknessSerializer(many=True) references = AdvisoryReferenceSerializer(many=True) severities = AdvisorySeveritySerializer(many=True) @@ -98,13 +103,12 @@ class AdvisoryV3Serializer(serializers.ModelSerializer): related_ssvc_trees = serializers.SerializerMethodField() def get_related_ssvc_trees(self, obj): - related_ssvcs = obj.related_ssvcs.all().select_related("source_advisory") - source_ssvcs = obj.source_ssvcs.all().select_related("source_advisory") - seen = set() result = [] - for ssvc in list(related_ssvcs) + list(source_ssvcs): + all_ssvcs = list(obj.related_ssvcs.all()) + list(obj.source_ssvcs.all()) + + for ssvc in all_ssvcs: key = (ssvc.vector, ssvc.source_advisory_id) if key in seen: continue @@ -399,7 +403,34 @@ def create(self, request, *args, **kwargs): purls = serializer.validated_data["purls"] - latest_advisories = AdvisoryV2.objects.latest_advisories_for_purls(purls=purls) + latest_advisories = AdvisoryV2.objects.latest_advisories_for_purls( + purls=purls + ).prefetch_related( + Prefetch( + "references", + queryset=AdvisoryReference.objects.only( + "id", + "url", + "reference_type", + "reference_id", + ), + ), + Prefetch( + "severities", + queryset=AdvisorySeverity.objects.only( + "id", + "url", + "value", + "scoring_system", + "scoring_elements", + "published_at", + ), + ), + "weaknesses", + "aliases", + "related_ssvcs__source_advisory", + "source_ssvcs__source_advisory", + ) page = self.paginate_queryset(latest_advisories) serializer = self.get_serializer(page, many=True, context={"request": request}) diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index 6d8b9519b..f3b8d9373 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -193,7 +193,7 @@ def test_packages_post_purl_with_many_advisories(self): def test_advisories_post(self): url = reverse("advisory-v3-list") - with self.assertNumQueries(64): + with self.assertNumQueries(10): response = self.client.post( url, data={"purls": ["pkg:pypi/sample@1.0.0"]}, From feff85f5ac187956ea5a0368fc3aeaec79ab0e44 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 20:41:39 +0530 Subject: [PATCH 22/65] Optimise package details Signed-off-by: Tushar Goel --- vulnerabilities/templates/packages_v2.html | 4 ++-- vulnerabilities/views.py | 17 ----------------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/vulnerabilities/templates/packages_v2.html b/vulnerabilities/templates/packages_v2.html index 752e0709c..4348575da 100644 --- a/vulnerabilities/templates/packages_v2.html +++ b/vulnerabilities/templates/packages_v2.html @@ -40,14 +40,14 @@ + data-tooltip="This is the status of the package. If it is vulnerable, it means that there are known vulnerabilities associated with this package."> Vulnerable + data-tooltip="This is the risk score of the package based on its vulnerabilities."> Risk Score diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 8765c30ec..0827c2b3e 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -179,7 +179,6 @@ def get_context_data(self, **kwargs): def get_queryset(self): purl = self.kwargs.get("purl") - print(purl) return models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl) @@ -262,22 +261,6 @@ def get_context_data(self, **kwargs): return context - def get_queryset(self): - return ( - super() - .get_queryset() - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.select_related("advisory"), - ), - ) - ) - def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() From 979d5640716bcaeb5d747ffd2917825a58e35a61 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 20:46:39 +0530 Subject: [PATCH 23/65] Optimize package details view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 0827c2b3e..c14f55e87 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -220,7 +220,10 @@ def get_context_data(self, **kwargs): affected_by_advisories_url = None fixing_advisories_url = None - affected_by_advisories = list(affected_by_advisories_qs[:101]) + affected_by_advisories_qs_ids = affected_by_advisories_qs.only("id") + fixing_advisories_qs_ids = fixing_advisories_qs.only("id") + + affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) if len(affected_by_advisories) > 100: affected_by_advisories_url = reverse_lazy( "affected_by_advisories_v2", kwargs={"purl": package.package_url} @@ -241,7 +244,7 @@ def get_context_data(self, **kwargs): context["fixed_package_details"] = fixed_pkg_details context["affected_by_advisories_v2_url"] = None - fixing_advisories = list(fixing_advisories_qs[:101]) + fixing_advisories = list(fixing_advisories_qs_ids[:101]) if len(fixing_advisories) > 100: fixing_advisories_url = reverse_lazy( "fixing_advisories_v2", kwargs={"purl": package.package_url} From bc1a43465b37d8bb180d639a5a7b384788f9e1ec Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 20:59:25 +0530 Subject: [PATCH 24/65] Optimize views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index c14f55e87..a99952100 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -179,7 +179,7 @@ def get_context_data(self, **kwargs): def get_queryset(self): purl = self.kwargs.get("purl") - return models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl) + return models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl).only("advisory_id", "summary", "url", "date_published").prefetch_related("aliases") class FixingAdvisoriesListView(ListView): @@ -189,7 +189,7 @@ class FixingAdvisoriesListView(ListView): def get_queryset(self): purl = self.kwargs.get("purl") - return models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(purl) + return models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(purl).only("advisory_id", "summary", "url", "date_published").prefetch_related("aliases") class PackageV2Details(DetailView): From a5aa6718180eb502af24cc8dec87916f5ffd1529 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 18 Mar 2026 21:47:16 +0530 Subject: [PATCH 25/65] Optimize queries Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 53 ++++++++++++++-------------- vulnerabilities/models.py | 2 +- vulnerabilities/tests/test_api_v3.py | 4 +-- vulnerabilities/views.py | 12 +++++-- 4 files changed, 39 insertions(+), 32 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index cd47c9697..944a15149 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -207,11 +207,13 @@ def get_affected_by_vulnerabilities(self, package): """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) - advisories = list(advisories_qs[:101]) - if len(advisories) > 100: + advisories_ids = advisories_qs.only("id") + + advisories_ids = list(advisories_ids[:101]) + if len(advisories_ids) > 100: return None - advisory_by_avid = {adv.avid: adv for adv in advisories} + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} avids = advisory_by_avid.keys() impacts = ( @@ -222,7 +224,7 @@ def get_affected_by_vulnerabilities(self, package): impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - grouped = group_advisories_by_content(advisories) + grouped = group_advisories_by_content(advisories_qs) result = [] for entry in grouped.values(): @@ -244,30 +246,17 @@ def get_affected_by_vulnerabilities(self, package): def get_fixing_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - advisories = list(advisories_qs[:101]) - if len(advisories) > 100: - return None - - advisory_by_avid = {adv.avid: adv for adv in advisories} - avids = advisory_by_avid.keys() - - impacts = ( - package.fixed_in_impacts.filter(advisory__avid__in=avids) - .select_related("advisory") - .prefetch_related("fixed_by_packages") - ) + advisories_ids = advisories_qs.only("id") - impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + advisories_ids = list(advisories_ids[:101]) + if len(advisories_ids) > 100: + return None - grouped = group_advisories_by_content(advisories) + grouped = group_advisories_by_content(advisories_qs) result = [] for entry in grouped.values(): primary = entry["primary"] - impact = impact_by_avid.get(primary.avid) - if not impact: - continue - result.append( { "advisory_id": primary.avid, @@ -301,14 +290,24 @@ def create(self, request, *args, **kwargs): approximate = serializer.validated_data["approximate"] if not purls: - vulnerable_purls = ( - PackageV2.objects.vulnerable() - .only("package_url") - .distinct() + pkg_ids = ( + PackageV2.objects.vulnerable().values_list("id", flat=True) + # .distinct() + ) + + # vulnerable_purls = ( + # PackageV2.objects.vulnerable() + # .only("package_url") + # .values_list("package_url", flat=True) + # .distinct() + # .order_by("package_url") + # ) + query = ( + PackageV2.objects.filter(id__in=pkg_ids) .values_list("package_url", flat=True) .order_by("package_url") ) - page = self.paginate_queryset(vulnerable_purls) + page = self.paginate_queryset(query) return self.get_paginated_response(page) plain_purls = None diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 80bd2e13d..e66716b3c 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3387,7 +3387,7 @@ def vulnerable(self): """ Return only packages that are vulnerable. """ - return self.filter(affected_in_impacts__isnull=False) + return self.filter(id__in=ImpactedPackageAffecting.objects.values("package_id").distinct()) def with_is_vulnerable(self): """ diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index f3b8d9373..6b88e5ee5 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -53,7 +53,7 @@ def test_packages_post_without_details(self): def test_packages_post_with_details(self): url = reverse("package-v3-list") - with self.assertNumQueries(21): + with self.assertNumQueries(23): response = self.client.post( url, data={ @@ -174,7 +174,7 @@ def setUp(self): def test_packages_post_purl_with_many_advisories(self): url = reverse("package-v3-list") - with self.assertNumQueries(11): + with self.assertNumQueries(12): response = self.client.post( url, data={ diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index a99952100..856c10ce6 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -179,7 +179,11 @@ def get_context_data(self, **kwargs): def get_queryset(self): purl = self.kwargs.get("purl") - return models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl).only("advisory_id", "summary", "url", "date_published").prefetch_related("aliases") + return ( + models.AdvisoryV2.objects.latest_affecting_advisories_for_purl(purl) + .only("advisory_id", "summary", "url", "date_published") + .prefetch_related("aliases") + ) class FixingAdvisoriesListView(ListView): @@ -189,7 +193,11 @@ class FixingAdvisoriesListView(ListView): def get_queryset(self): purl = self.kwargs.get("purl") - return models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(purl).only("advisory_id", "summary", "url", "date_published").prefetch_related("aliases") + return ( + models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(purl) + .only("advisory_id", "summary", "url", "date_published") + .prefetch_related("aliases") + ) class PackageV2Details(DetailView): From dc1a076def520fca62e2b2aaa76431b85f73bf7e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 19 Mar 2026 13:00:23 +0530 Subject: [PATCH 26/65] Fix typo in archlinux importer Signed-off-by: Tushar Goel --- vulnerabilities/pipelines/v2_importers/archlinux_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py index b0f005592..b666e48a2 100644 --- a/vulnerabilities/pipelines/v2_importers/archlinux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/archlinux_importer.py @@ -105,7 +105,7 @@ def parse_advisory(self, record) -> AdvisoryDataV2: VulnerabilitySeverity( system=severity_systems.ARCHLINUX, value=severity, - url="https://security.archlinux.org/{avg_name}.json", + url=f"https://security.archlinux.org/{avg_name}.json", ) ] From 4d264435246f98a0018edce61f12223ead7220a2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 19 Mar 2026 13:17:59 +0530 Subject: [PATCH 27/65] All vulnerable packages API Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 944a15149..b55822e41 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -9,6 +9,8 @@ from urllib.parse import urlencode +from django.db.models import Exists +from django.db.models import OuterRef from django.db.models import Prefetch from django_filters import rest_framework as filters from packageurl import PackageURL @@ -21,6 +23,7 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness +from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle from vulnerabilities.utils import group_advisories_by_content @@ -290,20 +293,11 @@ def create(self, request, *args, **kwargs): approximate = serializer.validated_data["approximate"] if not purls: - pkg_ids = ( - PackageV2.objects.vulnerable().values_list("id", flat=True) - # .distinct() - ) + impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) - # vulnerable_purls = ( - # PackageV2.objects.vulnerable() - # .only("package_url") - # .values_list("package_url", flat=True) - # .distinct() - # .order_by("package_url") - # ) query = ( - PackageV2.objects.filter(id__in=pkg_ids) + PackageV2.objects.annotate(has_vuln=Exists(impacted)) + .filter(has_vuln=True) .values_list("package_url", flat=True) .order_by("package_url") ) From f04f625a1259f93f09b6e08dc695e3241fe60ae2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 19 Mar 2026 13:49:22 +0530 Subject: [PATCH 28/65] Optimize advisoryqueyset Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e66716b3c..bd3f1ce22 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2915,33 +2915,24 @@ def latest_fixed_by_advisories_for_purls(self, purls): self.annotate(has_fixed=Exists(fixed_exists)).filter(has_fixed=True).latest_per_avid() ) - def latest_advisories_for_purl(self, purl): - return self.filter( - Q(impacted_packages__affecting_packages__package_url=purl) - | Q(impacted_packages__fixed_by_packages__package_url=purl) - ).latest_per_avid() - def latest_advisories_for_purls(self, purls): - - affecting = ImpactedPackageAffecting.objects.filter( - impacted_package__advisory_id=OuterRef("pk"), - package__package_url__in=purls, - ) - - fixed = ImpactedPackageFixedBy.objects.filter( - impacted_package__advisory_id=OuterRef("pk"), - package__package_url__in=purls, - ) - - return ( - self.annotate( - has_affecting=Exists(affecting), - has_fixed=Exists(fixed), + adv_ids = ImpactedPackageAffecting.objects.filter( + package__package_url__in=purls + ).values_list( + "impacted_package__advisory_id", + flat=True, + ).union( + ImpactedPackageFixedBy.objects.filter( + package__package_url__in=purls + ).values_list( + "impacted_package__advisory_id", + flat=True, ) - .filter(Q(has_affecting=True) | Q(has_fixed=True)) - .latest_per_avid() ) + qs = AdvisoryV2.objects.filter(id__in=Subquery(adv_ids)) + return qs.latest_per_avid() + class AdvisoryV2(models.Model): """ From 79210552e38301f353a20541770ae0e67ae2939e Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 19 Mar 2026 14:48:41 +0530 Subject: [PATCH 29/65] Fix errors Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 69 ++++++++----------- .../compute_advisory_content_hash.py | 7 +- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index bd3f1ce22..4f73ead9d 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2872,65 +2872,52 @@ def latest_for_avids(self, avids): return self.filter(avid__in=avids).latest_per_avid() def latest_affecting_advisories_for_purl(self, purl): - affecting_exists = ImpactedPackageAffecting.objects.filter( - impacted_package__advisory_id=OuterRef("pk"), - package__package_url=purl, - ) - - return ( - self.annotate(has_affecting=Exists(affecting_exists)) - .filter(has_affecting=True) - .latest_per_avid() + adv_ids = ImpactedPackageAffecting.objects.filter(package__package_url=purl).values_list( + "impacted_package__advisory_id", + flat=True, ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_affecting_advisories_for_purls(self, purls): - affecting_exists = ImpactedPackageAffecting.objects.filter( - impacted_package__advisory_id=OuterRef("pk"), - package__package_url__in=purls, - ) - - return ( - self.annotate(has_affecting=Exists(affecting_exists)) - .filter(has_affecting=True) - .latest_per_avid() + adv_ids = ImpactedPackageAffecting.objects.filter( + package__package_url__in=purls + ).values_list( + "impacted_package__advisory_id", + flat=True, ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_fixed_by_advisories_for_purl(self, purl): - fixed_exists = ImpactedPackageFixedBy.objects.filter( - impacted_package__advisory_id=OuterRef("pk"), - package__package_url=purl, - ) - - return ( - self.annotate(has_fixed=Exists(fixed_exists)).filter(has_fixed=True).latest_per_avid() + adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( + "impacted_package__advisory_id", + flat=True, ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_fixed_by_advisories_for_purls(self, purls): - fixed_exists = ImpactedPackageFixedBy.objects.filter( - impacted_package__advisory_id=OuterRef("pk"), - package__package_url__in=purls, + adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( + "impacted_package__advisory_id", + flat=True, ) - return ( - self.annotate(has_fixed=Exists(fixed_exists)).filter(has_fixed=True).latest_per_avid() - ) + return self.filter(id__in=Subquery(adv_ids)).latest_per_avid() def latest_advisories_for_purls(self, purls): - adv_ids = ImpactedPackageAffecting.objects.filter( - package__package_url__in=purls - ).values_list( - "impacted_package__advisory_id", - flat=True, - ).union( - ImpactedPackageFixedBy.objects.filter( - package__package_url__in=purls - ).values_list( + adv_ids = ( + ImpactedPackageAffecting.objects.filter(package__package_url__in=purls) + .values_list( "impacted_package__advisory_id", flat=True, ) + .union( + ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list( + "impacted_package__advisory_id", + flat=True, + ) + ) ) - qs = AdvisoryV2.objects.filter(id__in=Subquery(adv_ids)) + qs = self.filter(id__in=Subquery(adv_ids)) return qs.latest_per_avid() diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py index 935631419..8b285d361 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py @@ -41,8 +41,11 @@ def compute_advisory_content_hash(self): batch_size = 5000 for advisory in progress.iter(advisories.iterator(chunk_size=batch_size)): - advisory.advisory_content_hash = compute_advisory_content(advisory) - to_update.append(advisory) + try: + advisory.advisory_content_hash = compute_advisory_content(advisory) + to_update.append(advisory) + except Exception as e: + self.log(f"Error computing advisory_content_hash for {advisory.avid}: {e}") if len(to_update) >= batch_size: AdvisoryV2.objects.bulk_update( From 455a0fe39978ef5a29d16c6f50268636610a429c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 19 Mar 2026 16:43:54 +0530 Subject: [PATCH 30/65] Forward to HTTPS Signed-off-by: Tushar Goel --- vulnerablecode/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index ae6638b76..435cb8953 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -54,6 +54,9 @@ # WARNING: Set this to False in production STAGING = env.bool("STAGING", default=True) +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +USE_X_FORWARDED_HOST = True + EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend" EMAIL_HOST = env.str("EMAIL_HOST", default="") EMAIL_USE_TLS = True From 584fd499cd178c378e129a547559ba73d01b9ccd Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 01:37:41 +0530 Subject: [PATCH 31/65] Group advisories with alias and affected packages Signed-off-by: Tushar Goel --- vulnerabilities/improvers/__init__.py | 2 + .../0118_advisoryset_advisorysetmember.py | 71 ++++ vulnerabilities/models.py | 50 +++ .../group_advisories_for_packages.py | 172 ++++++++ .../templates/package_details_v3.html | 367 ++++++++++++++++++ vulnerabilities/views.py | 105 +++++ vulnerablecode/urls.py | 4 +- 7 files changed, 769 insertions(+), 2 deletions(-) create mode 100644 vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py create mode 100644 vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py create mode 100644 vulnerabilities/templates/package_details_v3.html diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 982b4bbd8..61a1fc882 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -32,6 +32,7 @@ enhance_with_metasploit as enhance_with_metasploit_v2, ) from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2 +from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages from vulnerabilities.pipelines.v2_improvers import relate_severities from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 from vulnerabilities.utils import create_registry @@ -76,5 +77,6 @@ collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, compute_advisory_content_hash.ComputeAdvisoryContentHash, + group_advisories_for_packages.GroupAdvisoriesForPackages, ] ) diff --git a/vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py b/vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py new file mode 100644 index 000000000..467d7b36c --- /dev/null +++ b/vulnerabilities/migrations/0118_advisoryset_advisorysetmember.py @@ -0,0 +1,71 @@ +# Generated by Django 5.2.11 on 2026-03-25 10:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0117_advisoryv2_risk_score"), + ] + + operations = [ + migrations.CreateModel( + name="AdvisorySet", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "relation_type", + models.CharField( + choices=[("affecting", "Affecting"), ("fixing", "Fixing")], max_length=20 + ), + ), + ("identifiers", models.JSONField()), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.packagev2" + ), + ), + ( + "primary_advisory", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, to="vulnerabilities.advisoryv2" + ), + ), + ], + ), + migrations.CreateModel( + name="AdvisorySetMember", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("is_primary", models.BooleanField(default=False)), + ( + "advisory", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.advisoryv2" + ), + ), + ( + "advisory_set", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="members", + to="vulnerabilities.advisoryset", + ), + ), + ], + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 4f73ead9d..05bf86a17 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2920,6 +2920,53 @@ def latest_advisories_for_purls(self, purls): qs = self.filter(id__in=Subquery(adv_ids)) return qs.latest_per_avid() + def latest_advisories_for_purl(self, purl): + adv_ids = ( + ImpactedPackageAffecting.objects.filter(package__package_url=purl) + .values_list( + "impacted_package__advisory_id", + flat=True, + ) + .union( + ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list( + "impacted_package__advisory_id", + flat=True, + ) + ) + ) + + qs = self.filter(id__in=Subquery(adv_ids)) + return qs.latest_per_avid() + + +class AdvisorySet(models.Model): + + RELATION_TYPE_CHOICES = [ + ("affecting", "Affecting"), + ("fixing", "Fixing"), + ] + + package = models.ForeignKey("PackageV2", on_delete=models.CASCADE) + relation_type = models.CharField(max_length=20, choices=RELATION_TYPE_CHOICES) + + identifiers = models.JSONField() + + primary_advisory = models.ForeignKey("AdvisoryV2", on_delete=models.PROTECT) + + created_at = models.DateTimeField(auto_now_add=True) + + +class AdvisorySetMember(models.Model): + + advisory_set = models.ForeignKey( + AdvisorySet, + on_delete=models.CASCADE, + related_name="members", + ) + + advisory = models.ForeignKey("AdvisoryV2", on_delete=models.CASCADE) + is_primary = models.BooleanField(default=False) + class AdvisoryV2(models.Model): """ @@ -3085,6 +3132,9 @@ def save(self, *args, **kwargs): self.full_clean() return super().save(*args, **kwargs) + def __str__(self): + return self.avid + @property def get_status_label(self): label_by_status = {choice[0]: choice[1] for choice in VulnerabilityStatusType.choices} diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py new file mode 100644 index 000000000..244de770e --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -0,0 +1,172 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from collections import defaultdict + +from django.db import transaction + +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.utils import compute_advisory_content + + +class GroupAdvisoriesForPackages(VulnerableCodePipeline): + """Detect and flag packages that do not exist upstream.""" + + pipeline_id = "group_advisories_for_packages" + + @classmethod + def steps(cls): + return (cls.group_advisories_for_packages,) + + def group_advisories_for_packages(self): + group_advisoris_for_packages(logger=self.log) + + +def merge_advisories(advisories): + + advisories = list(advisories) + + content_hash_map = defaultdict(list) + result_groups = [] + + for adv in advisories: + + if adv.advisory_content_hash: + content_hash_map[adv.advisory_content_hash].append(adv) + else: + content_hash = compute_advisory_content(advisory_data=adv) + if content_hash: + content_hash_map[content_hash].append(adv) + else: + result_groups.append([adv]) + + final_groups = [] + + for group in content_hash_map.values(): + groups = get_merged_identifier_groups(group) + final_groups.extend(groups) + + return final_groups + + +def get_merged_identifier_groups(advisories): + + identifier_groups = defaultdict(set) + advisory_to_identifiers = defaultdict(set) + + advisories = list(advisories) + + for adv in advisories: + + identifier_groups[adv.advisory_id].add(adv) + advisory_to_identifiers[adv].add(adv.advisory_id) + + for alias in adv.aliases.all(): + identifier_groups[alias.alias].add(adv) + advisory_to_identifiers[adv].add(alias.alias) + + groups = [set(advs) for advs in identifier_groups.values() if len(advs) > 1] + + merged = [] + + for group in groups: + group = set(group) + + i = 0 + while i < len(merged): + if group & merged[i]: + group |= merged[i] + merged.pop(i) + else: + i += 1 + + merged.append(group) + + all_grouped = set() + for g in merged: + all_grouped |= g + + for adv in advisories: + if adv not in all_grouped: + merged.append({adv}) + + final_groups = [] + + for group in merged: + identifiers = set() + for adv in group: + for alias in adv.aliases.values_list("alias", flat=True): + identifiers.add(alias) + + primary = max(group, key=lambda a: a.precedence if a.precedence is not None else -1) + + secondary = [a for a in group if a != primary] + + final_groups.append((identifiers, primary, secondary)) + + return final_groups + + +def group_advisoris_for_packages(logger=None): + for package in PackageV2.objects.iterator(): + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related("aliases") + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related("aliases") + + try: + delete_and_save_advisory_set(package, affecting_advisories, relation="affecting") + delete_and_save_advisory_set(package, fixed_by_advisories, relation="fixing") + except Exception as e: + print(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + continue + + +@transaction.atomic +def delete_and_save_advisory_set(package, advisories, relation=None): + AdvisorySet.objects.filter(package=package, relation_type=relation).delete() + + groups = merge_advisories(advisories) + + membership_to_create = [] + + for identifiers, primary, secondary in groups: + + advisory_set = AdvisorySet.objects.create( + package=package, + relation_type=relation, + identifiers=list(identifiers), + primary_advisory=primary, + ) + + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=primary, + is_primary=True, + ) + ) + + for adv in secondary: + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=adv, + is_primary=False, + ) + ) + + AdvisorySetMember.objects.bulk_create(membership_to_create) diff --git a/vulnerabilities/templates/package_details_v3.html b/vulnerabilities/templates/package_details_v3.html new file mode 100644 index 000000000..44ec1c297 --- /dev/null +++ b/vulnerabilities/templates/package_details_v3.html @@ -0,0 +1,367 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} +{% load static %} +{% load url_filters %} +{% load utils %} + +{% block title %} +VulnerableCode Package Details - {{ package.purl }} +{% endblock %} + +{% block content %} +
+ {% include "package_search_box_v2.html"%} +
+ +{% if package %} +
+
+
+
+ Package details: + {{ package.purl }} + +
+
+ +
+ +
+ +
+
+
+ {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %} +
+ {% else %} +
+ {% endif %} + + + + + + + {% if package.is_ghost %} + + + + + {% endif %} + +
+ + purl + + + {{ package.purl }} +
+ Tags + + + Ghost + +
+
+ {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %} + +
+ + + + + + + + + + + + + + + +
+ Next non-vulnerable version + + {% if next_non_vulnerable.version %} + {{ next_non_vulnerable.version }} + {% else %} + None. + {% endif %} +
+ Latest non-vulnerable version + + {% if latest_non_vulnerable.version %} + {{ latest_non_vulnerable.version }} + {% else %} + None. + {% endif %} +
+ Risk score + + {{package.risk_score}} +
+
+ + {% endif %} + +
+ {% if affected_by_advisories_v2|length != 0 %} +
+ Vulnerabilities affecting this package ({{ affected_by_advisories_v2|length }}) +
+ + + + + + + + + + + + + {% for advisory in affected_by_advisories_v2 %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryFixed in package version
+ + {{advisory.primary_advisory.advisory_id }} + +
+ {% if advisory.identifiers|length != 0 %} + Aliases: + {% endif %} +
+ {% for alias in advisory.identifiers %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
+ {% if advisory.secondary_members|length != 0 %} +

Supporting advisories are listed below the primary advisory.

+ {% for secondary in advisory.secondary_members %} + + {{secondary.advisory.avid }}
+ + {% endfor %} + {% endif %} + 		+ {{advisory.primary_advisory.url}} + + {{advisory.primary_advisory.date_published}} + + {{ advisory.primary_advisory.summary }} + + {% with fixed=fixed_package_details|get_item:advisory.primary_advisory.avid %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ {% if item.pkg.is_vulnerable %} + + Vulnerable + + {% else %} + + Not vulnerable + + {% endif %} +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} +
+ This package is not known to be subject of any advisories. +
+ {% elif affected_by_advisories_v2_url %} +
+ This package is subject to more than 100 advisories. Please refer to the following + URL for vulnerabilities affecting this package: Advisories +
+ {% else %} +
+ This package is not known to be subject of any advisories. +
+ {% endif %} +
+ +
+ {% if fixing_advisories_v2|length != 0 %} +
+ Vulnerabilities fixed by this package ({{ fixing_advisories_v2|length }}) +
+ + + + + + + + + + + + + {% for advisory in fixing_advisories_v2 %} + + + + + + + + {% empty %} + + + + {% endfor %} + +
AdvisorySourceDate PublishedSummaryAliases
+ + {{advisory.primary_advisory.advisory_id }} + +
+ {% if advisory.secondary_members|length != 0 %} +

Supporting advisories are listed below the primary advisory.

+ {% for secondary in advisory.secondary_members %} + + {{secondary.advisory.avid }}
+ + {% endfor %} + {% endif %} + 		+ {{advisory.primary_advisory.url}} + + {{advisory.primary_advisory.date_published}} + + {{ advisory.primary_advisory.summary }} + + {% for alias in advisory.identifiers %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} +
+ This package is not known to fix any advisories. +
+ +
+ {% elif fixing_advisories_v2_url %} +
+ This package is known to fix more than 100 advisories. Please refer to the following + URL for vulnerabilities fixed by this package: Advisories +
+ {% else %} +
+ This package is not known to fix any advisories. +
+ {% endif %} +
+
+
+ + +
+
+
+
+ +{% endif %} +{% endblock %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 856c10ce6..b9d172ca1 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -37,6 +37,7 @@ from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import PipelineSchedulePackageForm from vulnerabilities.forms import VulnerabilitySearchForm +from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule @@ -292,6 +293,110 @@ def get_object(self, queryset=None): return package +class PackageV3Details(DetailView): + model = models.PackageV2 + template_name = "package_details_v3.html" + slug_url_kwarg = "purl" + slug_field = "purl" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + package = self.object + + next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() + + context["package"] = package + context["next_non_vulnerable"] = next_non_vulnerable + context["latest_non_vulnerable"] = latest_non_vulnerable + context["package_search_form"] = PackageSearchForm(self.request.GET) + + affected_by_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="affecting") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + fixing_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="fixing") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + print(affected_by_advisories_qs) + print(fixing_advisories_qs) + + affected_by_advisories_url = None + fixing_advisories_url = None + + affected_by_advisories_qs_ids = affected_by_advisories_qs.only("id") + fixing_advisories_qs_ids = fixing_advisories_qs.only("id") + + # affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) + # if len(affected_by_advisories) > 100: + # affected_by_advisories_url = reverse_lazy( + # "affected_by_advisories_v2", kwargs={"purl": package.package_url} + # ) + # context["affected_by_advisories_v2_url"] = affected_by_advisories_url + # context["affected_by_advisories_v2"] = [] + # context["fixed_package_details"] = {} + + # else: + fixed_pkg_details = get_fixed_package_details(package) + + context["affected_by_advisories_v2"] = affected_by_advisories_qs + context["fixed_package_details"] = fixed_pkg_details + context["affected_by_advisories_v2_url"] = None + + # fixing_advisories = list(fixing_advisories_qs_ids[:101]) + # if len(fixing_advisories) > 100: + # fixing_advisories_url = reverse_lazy( + # "fixing_advisories_v2", kwargs={"purl": package.package_url} + # ) + # context["fixing_advisories_v2_url"] = fixing_advisories_url + # context["fixing_advisories_v2"] = [] + + # else: + context["fixing_advisories_v2"] = fixing_advisories_qs + context["fixing_advisories_v2_url"] = None + + return context + + def get_object(self, queryset=None): + if queryset is None: + queryset = self.get_queryset() + + purl = self.kwargs.get(self.slug_url_kwarg) + if purl: + queryset = queryset.for_purl(purl) + else: + cls = self.__class__.__name__ + raise AttributeError( + f"Package details view {cls} must be called with a purl, " f"but got: {purl!r}" + ) + + try: + package = queryset.get() + except queryset.model.DoesNotExist: + raise Http404(f"No Package found for purl: {purl}") + return package + + def get_fixed_package_details(package): rows = package.affected_in_impacts.values_list( "advisory__avid", diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index eb1bc006b..efbfc9c6f 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -29,7 +29,7 @@ from vulnerabilities.api_v3 import AffectedByAdvisoriesViewSet from vulnerabilities.api_v3 import FixingAdvisoriesViewSet from vulnerabilities.api_v3 import PackageV3ViewSet -from vulnerabilities.views import AdminLoginView +from vulnerabilities.views import AdminLoginView, PackageV3Details from vulnerabilities.views import AdvisoryDetails from vulnerabilities.views import AdvisoryPackagesDetails from vulnerabilities.views import AffectedByAdvisoriesListView @@ -141,7 +141,7 @@ def __init__(self, *args, **kwargs): ), re_path( r"^packages/v2/(?Ppkg:.+)$", - PackageV2Details.as_view(), + PackageV3Details.as_view(), name="package_details_v2", ), re_path( From 59fd85ff1bc14535e206d941fb5eb804e360ebbf Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 01:47:24 +0530 Subject: [PATCH 32/65] Fix content hash logic Signed-off-by: Tushar Goel --- vulnerabilities/utils.py | 5 +---- vulnerablecode/urls.py | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index f90d42401..88adf1c41 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -611,6 +611,7 @@ def normalize_text(text): def normalize_list(lst): """Sort a list to ensure consistent ordering.""" + lst = [x for x in lst if x] return sorted(lst) if lst else [] @@ -885,13 +886,9 @@ def compute_advisory_content(advisory_data): if isinstance(advisory_data, AdvisoryV2): advisory_data = advisory_data.to_advisory_data() normalized_data = { - "summary": normalize_text(advisory_data.summary), "affected_packages": [ pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg ], - "severities": [sev.to_dict() for sev in normalize_list(advisory_data.severities) if sev], - "weaknesses": normalize_list(advisory_data.weaknesses), - "patches": [patch.to_dict() for patch in normalize_list(advisory_data.patches)], } normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index efbfc9c6f..745d2a469 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -29,7 +29,7 @@ from vulnerabilities.api_v3 import AffectedByAdvisoriesViewSet from vulnerabilities.api_v3 import FixingAdvisoriesViewSet from vulnerabilities.api_v3 import PackageV3ViewSet -from vulnerabilities.views import AdminLoginView, PackageV3Details +from vulnerabilities.views import AdminLoginView from vulnerabilities.views import AdvisoryDetails from vulnerabilities.views import AdvisoryPackagesDetails from vulnerabilities.views import AffectedByAdvisoriesListView @@ -41,6 +41,7 @@ from vulnerabilities.views import PackageSearch from vulnerabilities.views import PackageSearchV2 from vulnerabilities.views import PackageV2Details +from vulnerabilities.views import PackageV3Details from vulnerabilities.views import PipelineRunDetailView from vulnerabilities.views import PipelineRunListView from vulnerabilities.views import PipelineScheduleListView From f562fd85c8440aab8229fcec73774f83c82deb09 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 01:48:13 +0530 Subject: [PATCH 33/65] Test out small use case Signed-off-by: Tushar Goel --- .../pipelines/v2_improvers/group_advisories_for_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 244de770e..386ce63af 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -118,7 +118,7 @@ def get_merged_identifier_groups(advisories): def group_advisoris_for_packages(logger=None): - for package in PackageV2.objects.iterator(): + for package in PackageV2.objects.filter(package_url="pkg:pypi/django@1.5.2").iterator(): affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl ).prefetch_related("aliases") From 931e111e8eed66a23894c203406c31693a035708 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 02:09:16 +0530 Subject: [PATCH 34/65] Group for all packages Signed-off-by: Tushar Goel --- .../pipelines/v2_improvers/group_advisories_for_packages.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 386ce63af..f26211f10 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -36,11 +36,13 @@ def merge_advisories(advisories): advisories = list(advisories) + print(len(advisories)) + content_hash_map = defaultdict(list) result_groups = [] for adv in advisories: - + print(adv.avid) if adv.advisory_content_hash: content_hash_map[adv.advisory_content_hash].append(adv) else: @@ -118,7 +120,7 @@ def get_merged_identifier_groups(advisories): def group_advisoris_for_packages(logger=None): - for package in PackageV2.objects.filter(package_url="pkg:pypi/django@1.5.2").iterator(): + for package in PackageV2.objects.iterator(): affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl ).prefetch_related("aliases") From 3286f90cb8214544dcb7609bbf1b4e418c7a50ad Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 02:53:28 +0530 Subject: [PATCH 35/65] Change process to compute hash Signed-off-by: Tushar Goel --- .../group_advisories_for_packages.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index f26211f10..52d16c093 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -7,6 +7,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import hashlib +import json from collections import defaultdict from django.db import transaction @@ -16,7 +18,7 @@ from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.utils import compute_advisory_content +from vulnerabilities.utils import normalize_list class GroupAdvisoriesForPackages(VulnerableCodePipeline): @@ -42,15 +44,26 @@ def merge_advisories(advisories): result_groups = [] for adv in advisories: - print(adv.avid) - if adv.advisory_content_hash: - content_hash_map[adv.advisory_content_hash].append(adv) + affected = [] + fixed = [] + + for impact in adv.impacted_packages.all(): + affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) + + fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) + + normalized_data = { + "affected_packages": normalize_list(affected), + "fixed_packages": normalize_list(fixed), + } + + normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) + content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() + + if content_hash: + content_hash_map[content_hash].append(adv) else: - content_hash = compute_advisory_content(advisory_data=adv) - if content_hash: - content_hash_map[content_hash].append(adv) - else: - result_groups.append([adv]) + result_groups.append([adv]) final_groups = [] From 8266b254887d3e70e9eb391824e64bec4d6d2cd4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 03:01:05 +0530 Subject: [PATCH 36/65] Prefetch affected packages Signed-off-by: Tushar Goel --- .../group_advisories_for_packages.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 52d16c093..0d466e44a 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -38,8 +38,6 @@ def merge_advisories(advisories): advisories = list(advisories) - print(len(advisories)) - content_hash_map = defaultdict(list) result_groups = [] @@ -77,18 +75,15 @@ def merge_advisories(advisories): def get_merged_identifier_groups(advisories): identifier_groups = defaultdict(set) - advisory_to_identifiers = defaultdict(set) advisories = list(advisories) for adv in advisories: identifier_groups[adv.advisory_id].add(adv) - advisory_to_identifiers[adv].add(adv.advisory_id) - for alias in adv.aliases.all(): - identifier_groups[alias.alias].add(adv) - advisory_to_identifiers[adv].add(alias.alias) + for alias in adv.aliases.values_list("alias", flat=True): + identifier_groups[alias].add(adv) groups = [set(advs) for advs in identifier_groups.values() if len(advs) > 1] @@ -134,13 +129,26 @@ def get_merged_identifier_groups(advisories): def group_advisoris_for_packages(logger=None): for package in PackageV2.objects.iterator(): - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ).prefetch_related("aliases") + print(package) + affecting_advisories = ( + AdvisoryV2.objects + .latest_affecting_advisories_for_purl(purl=package.purl) + .prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + ) - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - purl=package.purl - ).prefetch_related("aliases") + fixed_by_advisories = ( + AdvisoryV2.objects + .latest_fixed_by_advisories_for_purl(purl=package.purl) + .prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + ) try: delete_and_save_advisory_set(package, affecting_advisories, relation="affecting") From bf29369b4c4e67b8e31648f41be78db0ca9d6463 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 03:22:05 +0530 Subject: [PATCH 37/65] Cache the advisory content hash Signed-off-by: Tushar Goel --- .../group_advisories_for_packages.py | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 0d466e44a..87b05eb5f 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -34,34 +34,23 @@ def group_advisories_for_packages(self): group_advisoris_for_packages(logger=self.log) +CONTENT_HASH_CACHE = {} + + def merge_advisories(advisories): advisories = list(advisories) content_hash_map = defaultdict(list) - result_groups = [] for adv in advisories: - affected = [] - fixed = [] - - for impact in adv.impacted_packages.all(): - affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) - - fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) - - normalized_data = { - "affected_packages": normalize_list(affected), - "fixed_packages": normalize_list(fixed), - } - - normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) - content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() - - if content_hash: - content_hash_map[content_hash].append(adv) + if adv.avid in CONTENT_HASH_CACHE: + content_hash = CONTENT_HASH_CACHE[adv.avid] else: - result_groups.append([adv]) + content_hash = compute_advisory_content_hash(adv) + CONTENT_HASH_CACHE[adv.avid] = content_hash + + content_hash_map[content_hash].append(adv) final_groups = [] @@ -72,6 +61,25 @@ def merge_advisories(advisories): return final_groups +def compute_advisory_content_hash(adv): + affected = [] + fixed = [] + + for impact in adv.impacted_packages.all(): + affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) + + fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) + + normalized_data = { + "affected_packages": normalize_list(affected), + "fixed_packages": normalize_list(fixed), + } + + normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) + content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() + return content_hash + + def get_merged_identifier_groups(advisories): identifier_groups = defaultdict(set) @@ -130,24 +138,20 @@ def get_merged_identifier_groups(advisories): def group_advisoris_for_packages(logger=None): for package in PackageV2.objects.iterator(): print(package) - affecting_advisories = ( - AdvisoryV2.objects - .latest_affecting_advisories_for_purl(purl=package.purl) - .prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", ) - fixed_by_advisories = ( - AdvisoryV2.objects - .latest_fixed_by_advisories_for_purl(purl=package.purl) - .prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", - ) + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", ) try: From 680f45e920342325f5e3040c25aea135dd524890 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 12:29:14 +0530 Subject: [PATCH 38/65] Group specific ecosystems Signed-off-by: Tushar Goel --- .../v2_improvers/group_advisories_for_packages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 87b05eb5f..99aa079f7 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -41,6 +41,9 @@ def merge_advisories(advisories): advisories = list(advisories) + if len(advisories) > 1000: + return + content_hash_map = defaultdict(list) for adv in advisories: @@ -136,7 +139,9 @@ def get_merged_identifier_groups(advisories): def group_advisoris_for_packages(logger=None): - for package in PackageV2.objects.iterator(): + for package in PackageV2.objects.filter( + type__in=["npm", "pypi", "nuget", "maven", "composer"] + ).iterator(): print(package) affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl From b9c4f185abf7a3ce295b82a10f0ccea00f1c65a7 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Fri, 27 Mar 2026 12:32:46 +0530 Subject: [PATCH 39/65] Group specific ecosystems Signed-off-by: Tushar Goel --- .../pipelines/v2_improvers/group_advisories_for_packages.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 99aa079f7..75b983e1c 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -41,9 +41,6 @@ def merge_advisories(advisories): advisories = list(advisories) - if len(advisories) > 1000: - return - content_hash_map = defaultdict(list) for adv in advisories: From 312d4444b0e98de04888d47e07c5a393960e224c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 28 Mar 2026 01:00:19 +0530 Subject: [PATCH 40/65] Use V2 views Signed-off-by: Tushar Goel --- .../templates/package_details_v2.html | 52 ++------- vulnerabilities/utils.py | 100 ++++++++++++++++++ vulnerabilities/views.py | 67 ++++++++---- vulnerablecode/urls.py | 2 +- 4 files changed, 156 insertions(+), 65 deletions(-) diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index f90585b9d..06c15f0d0 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -136,8 +136,6 @@ Advisory - Source - Date Published Summary Fixed in package version @@ -147,15 +145,15 @@ {% for advisory in affected_by_advisories_v2 %} - - {{advisory.primary.avid }} + + {{advisory.identifier }}
- {% if advisory.primary.alias|length != 0 %} + {% if advisory.aliases|length != 0 %} Aliases: {% endif %}
- {% for alias in advisory.primary.alias %} + {% for alias in advisory.aliases %} {% if alias.url %} {{ alias }} @@ -166,26 +164,12 @@ {% endif %} {% endfor %} - {% if advisory.secondary|length != 0 %} -

Supporting advisories are listed below the primary advisory.

- {% for secondary in advisory.secondary %} - - {{secondary.avid }} - - {% endfor %} - {% endif %} - {{advisory.primary.url}} - - - {{advisory.primary.date_published}} - - - {{ advisory.primary.summary }} + {{ advisory.advisory.summary|truncatewords:20 }} - {% with fixed=fixed_package_details|get_item:advisory.primary.avid %} + {% with fixed=fixed_package_details|get_item:advisory.advisory.avid %} {% if fixed %} {% for item in fixed %}
@@ -240,8 +224,6 @@ Advisory - Source - Date Published Summary Aliases @@ -250,30 +232,16 @@ {% for advisory in fixing_advisories_v2 %} - - {{advisory.primary.avid }} + + {{advisory.identifier }}
- {% if advisory.secondary|length != 0 %} -

Supporting advisories are listed below the primary advisory.

- {% for secondary in advisory.secondary %} - - {{secondary.avid }} - - {% endfor %} - {% endif %} - - - {{advisory.primary.url}} - - - {{advisory.primary.date_published}} - {{ advisory.primary.summary }} + {{ advisory.advisory.summary|truncatewords:20 }} - {% for alias in advisory.primary.alias %} + {% for alias in advisory.aliases %} {% if alias.url %} {{ alias }} diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 88adf1c41..2dd606a92 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -895,3 +895,103 @@ def compute_advisory_content(advisory_data): content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() return content_hash + + +def merge_advisories(advisories, package): + + advisories = list(advisories) + + content_hash_map = defaultdict(list) + + for adv in advisories: + content_hash = compute_advisory_content_hash(adv, package) + content_hash_map[content_hash].append(adv) + + final_groups = [] + + for group in content_hash_map.values(): + groups = get_merged_identifier_groups(group) + final_groups.extend(groups) + + return final_groups + + +def compute_advisory_content_hash(adv, package): + affected = [] + fixed = [] + + version_less_purl = PackageURL( + type=package.type, + namespace=package.namespace, + name=package.name, + qualifiers=package.qualifiers, + subpath=package.subpath, + ) + + for impact in adv.impacted_packages.filter(base_purl=str(version_less_purl)): + affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) + fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) + + normalized_data = { + "affected_packages": normalize_list(affected), + "fixed_packages": normalize_list(fixed), + } + + normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) + content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() + return content_hash + + +def get_merged_identifier_groups(advisories): + + identifier_groups = defaultdict(set) + + advisories = list(advisories) + + for adv in advisories: + + identifier_groups[adv.advisory_id].add(adv) + + for alias in adv.aliases.values_list("alias", flat=True): + identifier_groups[alias].add(adv) + + groups = [set(advs) for advs in identifier_groups.values() if len(advs) > 1] + + merged = [] + + for group in groups: + group = set(group) + + i = 0 + while i < len(merged): + if group & merged[i]: + group |= merged[i] + merged.pop(i) + else: + i += 1 + + merged.append(group) + + all_grouped = set() + for g in merged: + all_grouped |= g + + for adv in advisories: + if adv not in all_grouped: + merged.append({adv}) + + final_groups = [] + + for group in merged: + identifiers = set() + for adv in group: + for alias in adv.aliases.all(): + identifiers.add(alias) + + primary = max(group, key=lambda a: a.precedence if a.precedence is not None else -1) + + secondary = [a for a in group if a != primary] + + final_groups.append((identifiers, primary, secondary)) + + return final_groups diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index b9d172ca1..c8bfc6634 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -38,6 +38,7 @@ from vulnerabilities.forms import PipelineSchedulePackageForm from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule @@ -45,6 +46,7 @@ from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import group_advisories_by_content +from vulnerabilities.utils import merge_advisories from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env @@ -218,22 +220,30 @@ def get_context_data(self, **kwargs): context["latest_non_vulnerable"] = latest_non_vulnerable context["package_search_form"] = PackageSearchForm(self.request.GET) - affected_by_advisories_qs = models.AdvisoryV2.objects.latest_affecting_advisories_for_purl( - package.package_url + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", ) - fixing_advisories_qs = models.AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( - package.package_url + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ).prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", ) affected_by_advisories_url = None fixing_advisories_url = None - affected_by_advisories_qs_ids = affected_by_advisories_qs.only("id") - fixing_advisories_qs_ids = fixing_advisories_qs.only("id") + affected_by_advisories_qs_ids = affecting_advisories.only("id") + fixing_advisories_qs_ids = fixed_by_advisories.only("id") - affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) - if len(affected_by_advisories) > 100: + affected_by_advisories = list(affected_by_advisories_qs_ids[:1001]) + if len(affected_by_advisories) > 1001: affected_by_advisories_url = reverse_lazy( "affected_by_advisories_v2", kwargs={"purl": package.package_url} ) @@ -242,19 +252,25 @@ def get_context_data(self, **kwargs): context["fixed_package_details"] = {} else: + advisories = [] + fixed_pkg_details = get_fixed_package_details(package) - affected_avid_by_hash = {} - affected_avid_by_hash = group_advisories_by_content(affected_by_advisories_qs) - affecting_advs = [] + groups = merge_advisories(affecting_advisories, package) + for aliases, primary, _ in groups: + identifier = primary.advisory_id.split("/")[-1] + + filtered_aliases = [alias for alias in aliases if alias.alias != identifier] + + advisories.append( + {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} + ) - for hash in affected_avid_by_hash: - affecting_advs.append(affected_avid_by_hash[hash]) - context["affected_by_advisories_v2"] = affecting_advs + context["affected_by_advisories_v2"] = advisories context["fixed_package_details"] = fixed_pkg_details context["affected_by_advisories_v2_url"] = None - fixing_advisories = list(fixing_advisories_qs_ids[:101]) - if len(fixing_advisories) > 100: + fixing_advisories = list(fixing_advisories_qs_ids[:1001]) + if len(fixing_advisories) > 1001: fixing_advisories_url = reverse_lazy( "fixing_advisories_v2", kwargs={"purl": package.package_url} ) @@ -262,13 +278,20 @@ def get_context_data(self, **kwargs): context["fixing_advisories_v2"] = [] else: - fixing_avid_by_hash = {} - fixing_avid_by_hash = group_advisories_by_content(fixing_advisories_qs) - fixing_advs = [] + advisories = [] + + fixed_pkg_details = get_fixed_package_details(package) + groups = merge_advisories(fixing_advisories, package) + for aliases, primary, _ in groups: + identifier = primary.advisory_id.split("/")[-1] + + filtered_aliases = [alias for alias in aliases if alias.alias != identifier] + + advisories.append( + {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} + ) - for hash in fixing_avid_by_hash: - fixing_advs.append(fixing_avid_by_hash[hash]) - context["fixing_advisories_v2"] = fixing_advs + context["fixing_advisories_v2"] = advisories context["fixing_advisories_v2_url"] = None return context diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 745d2a469..44cacd9b0 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -142,7 +142,7 @@ def __init__(self, *args, **kwargs): ), re_path( r"^packages/v2/(?Ppkg:.+)$", - PackageV3Details.as_view(), + PackageV2Details.as_view(), name="package_details_v2", ), re_path( From 0b753c9cf35bab2450a8502c1ffc89d710b5f047 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 30 Mar 2026 16:32:04 +0530 Subject: [PATCH 41/65] Adjust API and UI for new grouping Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 115 +++++++++-- vulnerabilities/improvers/__init__.py | 2 - ...remove_advisoryset_identifiers_and_more.py | 30 +++ vulnerabilities/models.py | 15 +- .../v2_importers/github_osv_importer.py | 2 +- .../pipelines/v2_importers/pypa_importer.py | 2 +- .../pipelines/v2_importers/pysec_importer.py | 2 +- .../compute_advisory_content_hash.py | 65 ------ .../group_advisories_for_packages.py | 162 +-------------- vulnerabilities/pipes/advisory.py | 3 - vulnerabilities/pipes/group_advisories.py | 50 +++++ .../templates/package_details_v2.html | 102 +++++++++- .../test_compute_advisory_content_hash.py | 88 -------- vulnerabilities/tests/test_advisory_merge.py | 192 ++++++++++++++++++ vulnerabilities/tests/test_api_v3.py | 21 +- vulnerabilities/utils.py | 104 +++++----- vulnerabilities/views.py | 126 +++++++----- 17 files changed, 620 insertions(+), 461 deletions(-) create mode 100644 vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py delete mode 100644 vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py create mode 100644 vulnerabilities/pipes/group_advisories.py delete mode 100644 vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py create mode 100644 vulnerabilities/tests/test_advisory_merge.py diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index b55822e41..ea82dcce3 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -20,13 +20,16 @@ from rest_framework.throttling import AnonRateThrottle from vulnerabilities.models import AdvisoryReference +from vulnerabilities.models import AdvisorySet from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle -from vulnerabilities.utils import group_advisories_by_content +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS +from vulnerabilities.utils import get_advisories_from_groups +from vulnerabilities.utils import merge_and_save_grouped_advisories class PackageQuerySerializer(serializers.Serializer): @@ -210,6 +213,32 @@ def get_affected_by_vulnerabilities(self, package): """Return a dictionary with advisory as keys and their details, including fixed_by_packages.""" advisories_qs = AdvisoryV2.objects.latest_affecting_advisories_for_purl(package.package_url) + advisories = [] + + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() + + if is_grouped: + affected_by_advisories_qs = AdvisorySet.objects.filter( + package=package, relation_type="affecting" + ).select_related("primary_advisory") + + affected_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") + for adv in affected_by_advisories_qs + ] + + advisories = get_advisories_from_groups(affected_groups) + return self.return_advisories_data(package, advisories_qs, advisories) + + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = advisories_qs.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + advisories = merge_and_save_grouped_advisories(package, advisories_qs, "affecting") + return self.return_advisories_data(package, advisories_qs, advisories) + advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) @@ -227,20 +256,19 @@ def get_affected_by_vulnerabilities(self, package): impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - grouped = group_advisories_by_content(advisories_qs) - result = [] - for entry in grouped.values(): - primary = entry["primary"] - impact = impact_by_avid.get(primary.avid) + + for advisory in advisories_qs: + impact = impact_by_avid.get(advisory.avid) if not impact: continue result.append( { - "advisory_id": primary.avid, + "advisory_id": advisory.advisory_id.split("/")[-1], + "aliases": [alias.alias for alias in advisory.aliases.all()], + "summary": advisory.summary, "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], - "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], } ) @@ -249,21 +277,82 @@ def get_affected_by_vulnerabilities(self, package): def get_fixing_vulnerabilities(self, package): advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) + advisories = [] + + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() + + if is_grouped: + fixing_advisories_qs = AdvisorySet.objects.filter( + package=package, relation_type="fixing" + ).select_related("primary_advisory") + + fixing_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") for adv in fixing_advisories_qs + ] + + advisories = get_advisories_from_groups(fixing_groups) + return self.return_fixing_advisories_data(advisories) + + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_qs = advisories_qs.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + advisories = merge_and_save_grouped_advisories(package, advisories_qs, "fixing") + return self.return_fixing_advisories_data(advisories) + advisories_ids = advisories_qs.only("id") advisories_ids = list(advisories_ids[:101]) if len(advisories_ids) > 100: return None - grouped = group_advisories_by_content(advisories_qs) + results = [] + for advisory in advisories_qs: + results.append( + { + "advisory_id": advisory.advisory_id.split("/")[-1], + } + ) + return results + + def return_fixing_advisories_data(self, advisories): result = [] - for entry in grouped.values(): - primary = entry["primary"] + for advisory in advisories: result.append( { - "advisory_id": primary.avid, - "duplicate_advisory_ids": [a.avid for a in entry["secondary"]], + "advisory_id": advisory["identifier"], + } + ) + + return result + + def return_advisories_data(self, package, advisories_qs, advisories): + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} + avids = advisory_by_avid.keys() + + impacts = ( + package.affected_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) + + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + + result = [] + for advisory in advisories: + impact = impact_by_avid.get(advisory["advisory"].avid) + if not impact: + continue + + result.append( + { + "advisory_id": advisory["identifier"], + "aliases": [alias.alias for alias in advisory["aliases"]], + "summary": advisory["advisory"].summary, + "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], } ) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 61a1fc882..3e991d658 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -20,7 +20,6 @@ from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees -from vulnerabilities.pipelines.v2_improvers import compute_advisory_content_hash from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( @@ -76,7 +75,6 @@ compute_advisory_todo.ComputeToDo, collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, - compute_advisory_content_hash.ComputeAdvisoryContentHash, group_advisories_for_packages.GroupAdvisoriesForPackages, ] ) diff --git a/vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py b/vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py new file mode 100644 index 000000000..503e14f8d --- /dev/null +++ b/vulnerabilities/migrations/0119_remove_advisoryset_identifiers_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.11 on 2026-03-30 08:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0118_advisoryset_advisorysetmember"), + ] + + operations = [ + migrations.RemoveField( + model_name="advisoryset", + name="identifiers", + ), + migrations.RemoveField( + model_name="advisoryv2", + name="advisory_content_hash", + ), + migrations.AddField( + model_name="advisoryset", + name="aliases", + field=models.ManyToManyField( + help_text="A list of serializable Alias objects", + related_name="advisory_sets", + to="vulnerabilities.advisoryalias", + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 05bf86a17..f51a92dbd 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2949,7 +2949,11 @@ class AdvisorySet(models.Model): package = models.ForeignKey("PackageV2", on_delete=models.CASCADE) relation_type = models.CharField(max_length=20, choices=RELATION_TYPE_CHOICES) - identifiers = models.JSONField() + aliases = models.ManyToManyField( + AdvisoryAlias, + related_name="advisory_sets", + help_text="A list of serializable Alias objects", + ) primary_advisory = models.ForeignKey("AdvisoryV2", on_delete=models.PROTECT) @@ -3101,13 +3105,6 @@ class AdvisoryV2(models.Model): help_text="Related advisories that are used to calculate the severity of this advisory.", ) - advisory_content_hash = models.CharField( - max_length=64, - blank=True, - null=True, - help_text="A unique hash computed from the content of the advisory used to identify advisories with the same content.", - ) - risk_score = models.DecimalField( null=True, blank=True, @@ -3311,7 +3308,7 @@ def search(self, query: str = None): except ValueError: # otherwise use query as a plain string qs = qs.filter(package_url__icontains=query) - return qs.order_by("package_url") + return qs.order_by("package_url").order_by("-version_rank") def with_vulnerability_counts(self): return self.annotate( diff --git a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py index cfe92d93f..33acaf7f8 100644 --- a/vulnerabilities/pipelines/v2_importers/github_osv_importer.py +++ b/vulnerabilities/pipelines/v2_importers/github_osv_importer.py @@ -31,7 +31,7 @@ class GithubOSVImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" repo_url = "git+https://github.com/github/advisory-database/" - precedence = 100 + precedence = 200 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/pypa_importer.py b/vulnerabilities/pipelines/v2_importers/pypa_importer.py index 90599e99d..7a80ed70f 100644 --- a/vulnerabilities/pipelines/v2_importers/pypa_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pypa_importer.py @@ -29,7 +29,7 @@ class PyPaImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "CC-BY-4.0" license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" repo_url = "git+https://github.com/pypa/advisory-database" - precedence = 200 + precedence = 500 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/pysec_importer.py b/vulnerabilities/pipelines/v2_importers/pysec_importer.py index 05614b961..e9225a4f5 100644 --- a/vulnerabilities/pipelines/v2_importers/pysec_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pysec_importer.py @@ -29,7 +29,7 @@ class PyPIImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" spdx_license_expression = "CC-BY-4.0" - precedence = 100 + precedence = 300 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py b/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py deleted file mode 100644 index 8b285d361..000000000 --- a/vulnerabilities/pipelines/v2_improvers/compute_advisory_content_hash.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -from aboutcode.pipeline import LoopProgress - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.utils import compute_advisory_content - - -class ComputeAdvisoryContentHash(VulnerableCodePipeline): - """Compute Advisory Content Hash for Advisory.""" - - pipeline_id = "compute_advisory_content_hash_v2" - - @classmethod - def steps(cls): - return (cls.compute_advisory_content_hash,) - - def compute_advisory_content_hash(self): - """Compute Advisory Content Hash for Advisory.""" - - advisories = AdvisoryV2.objects.latest_per_avid().filter(advisory_content_hash__isnull=True) - - advisories_count = advisories.count() - - progress = LoopProgress( - total_iterations=advisories_count, - logger=self.log, - progress_step=1, - ) - - to_update = [] - batch_size = 5000 - - for advisory in progress.iter(advisories.iterator(chunk_size=batch_size)): - try: - advisory.advisory_content_hash = compute_advisory_content(advisory) - to_update.append(advisory) - except Exception as e: - self.log(f"Error computing advisory_content_hash for {advisory.avid}: {e}") - - if len(to_update) >= batch_size: - AdvisoryV2.objects.bulk_update( - to_update, - ["advisory_content_hash"], - batch_size=batch_size, - ) - to_update.clear() - - if to_update: - AdvisoryV2.objects.bulk_update( - to_update, - ["advisory_content_hash"], - batch_size=batch_size, - ) - - self.log("Finished computing advisory_content_hash") diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index 75b983e1c..d2c8f6296 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -7,18 +7,12 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import hashlib -import json -from collections import defaultdict - -from django.db import transaction - -from vulnerabilities.models import AdvisorySet -from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.utils import normalize_list +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS +from vulnerabilities.utils import merge_advisories class GroupAdvisoriesForPackages(VulnerableCodePipeline): @@ -34,112 +28,9 @@ def group_advisories_for_packages(self): group_advisoris_for_packages(logger=self.log) -CONTENT_HASH_CACHE = {} - - -def merge_advisories(advisories): - - advisories = list(advisories) - - content_hash_map = defaultdict(list) - - for adv in advisories: - if adv.avid in CONTENT_HASH_CACHE: - content_hash = CONTENT_HASH_CACHE[adv.avid] - else: - content_hash = compute_advisory_content_hash(adv) - CONTENT_HASH_CACHE[adv.avid] = content_hash - - content_hash_map[content_hash].append(adv) - - final_groups = [] - - for group in content_hash_map.values(): - groups = get_merged_identifier_groups(group) - final_groups.extend(groups) - - return final_groups - - -def compute_advisory_content_hash(adv): - affected = [] - fixed = [] - - for impact in adv.impacted_packages.all(): - affected.extend([pkg.package_url for pkg in impact.affecting_packages.all()]) - - fixed.extend([pkg.package_url for pkg in impact.fixed_by_packages.all()]) - - normalized_data = { - "affected_packages": normalize_list(affected), - "fixed_packages": normalize_list(fixed), - } - - normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) - content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() - return content_hash - - -def get_merged_identifier_groups(advisories): - - identifier_groups = defaultdict(set) - - advisories = list(advisories) - - for adv in advisories: - - identifier_groups[adv.advisory_id].add(adv) - - for alias in adv.aliases.values_list("alias", flat=True): - identifier_groups[alias].add(adv) - - groups = [set(advs) for advs in identifier_groups.values() if len(advs) > 1] - - merged = [] - - for group in groups: - group = set(group) - - i = 0 - while i < len(merged): - if group & merged[i]: - group |= merged[i] - merged.pop(i) - else: - i += 1 - - merged.append(group) - - all_grouped = set() - for g in merged: - all_grouped |= g - - for adv in advisories: - if adv not in all_grouped: - merged.append({adv}) - - final_groups = [] - - for group in merged: - identifiers = set() - for adv in group: - for alias in adv.aliases.values_list("alias", flat=True): - identifiers.add(alias) - - primary = max(group, key=lambda a: a.precedence if a.precedence is not None else -1) - - secondary = [a for a in group if a != primary] - - final_groups.append((identifiers, primary, secondary)) - - return final_groups - - def group_advisoris_for_packages(logger=None): - for package in PackageV2.objects.filter( - type__in=["npm", "pypi", "nuget", "maven", "composer"] - ).iterator(): - print(package) + for package in PackageV2.objects.filter(type__in=TYPES_WITH_MULTIPLE_IMPORTERS).iterator(): + print(f"Grouping advisories for package {package.purl}") affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl ).prefetch_related( @@ -157,45 +48,10 @@ def group_advisoris_for_packages(logger=None): ) try: - delete_and_save_advisory_set(package, affecting_advisories, relation="affecting") - delete_and_save_advisory_set(package, fixed_by_advisories, relation="fixing") + affected_groups = merge_advisories(affecting_advisories, package) + fixed_by_groups = merge_advisories(fixed_by_advisories, package) + delete_and_save_advisory_set(affected_groups, package, relation="affecting") + delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") except Exception as e: print(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") continue - - -@transaction.atomic -def delete_and_save_advisory_set(package, advisories, relation=None): - AdvisorySet.objects.filter(package=package, relation_type=relation).delete() - - groups = merge_advisories(advisories) - - membership_to_create = [] - - for identifiers, primary, secondary in groups: - - advisory_set = AdvisorySet.objects.create( - package=package, - relation_type=relation, - identifiers=list(identifiers), - primary_advisory=primary, - ) - - membership_to_create.append( - AdvisorySetMember( - advisory_set=advisory_set, - advisory=primary, - is_primary=True, - ) - ) - - for adv in secondary: - membership_to_create.append( - AdvisorySetMember( - advisory_set=advisory_set, - advisory=adv, - is_primary=False, - ) - ) - - AdvisorySetMember.objects.bulk_create(membership_to_create) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index a7f67153f..bcdd95075 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -48,7 +48,6 @@ from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 -from vulnerabilities.utils import compute_advisory_content def get_or_create_aliases(aliases: List) -> QuerySet: @@ -302,7 +301,6 @@ def insert_advisory_v2( advisory_obj = None created = False content_id = compute_content_id_v2(advisory_data=advisory) - advisory_content_hash = compute_advisory_content(advisory_data=advisory) try: default_data = { "datasource_id": pipeline_id, @@ -313,7 +311,6 @@ def insert_advisory_v2( "original_advisory_text": advisory.original_advisory_text, "url": advisory.url, "precedence": precedence, - "advisory_content_hash": advisory_content_hash, } advisory_obj, created = AdvisoryV2.objects.get_or_create( diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py new file mode 100644 index 000000000..d66365706 --- /dev/null +++ b/vulnerabilities/pipes/group_advisories.py @@ -0,0 +1,50 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.db import transaction + + +@transaction.atomic +def delete_and_save_advisory_set(groups, package, relation=None): + from vulnerabilities.models import AdvisorySet + from vulnerabilities.models import AdvisorySetMember + + AdvisorySet.objects.filter(package=package, relation_type=relation).delete() + + membership_to_create = [] + + for identifiers, primary, secondary in groups: + + advisory_set = AdvisorySet.objects.create( + package=package, + relation_type=relation, + primary_advisory=primary, + ) + + advisory_set.aliases.add(*identifiers) + advisory_set.save() + + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=primary, + is_primary=True, + ) + ) + + for adv in secondary: + membership_to_create.append( + AdvisorySetMember( + advisory_set=advisory_set, + advisory=adv, + is_primary=False, + ) + ) + + AdvisorySetMember.objects.bulk_create(membership_to_create) diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index 06c15f0d0..8c3f62756 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -141,6 +141,7 @@ + {% if grouped %} {% for advisory in affected_by_advisories_v2 %} @@ -201,6 +202,68 @@ {% endfor %} + {% else %} + + {% for advisory in affected_by_advisories_v2 %} + + + + {{advisory.advisory_id }} + +
+ {% if advisory.aliases.all|length != 0 %} + Aliases: + {% endif %} +
+ {% for alias in advisory.aliases.all %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} + + + + {{ advisory.summary|truncatewords:20 }} + + + {% with fixed=fixed_package_details|get_item:advisory.avid %} + {% if fixed %} + {% for item in fixed %} +
+ {{ item.pkg.version }} +
+ {% if item.pkg.is_vulnerable %} + + Vulnerable + + {% else %} + + Not vulnerable + + {% endif %} +
+ {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endwith %} + + + {% empty %} + + + This package is not known to be subject of any advisories. + + + {% endfor %} + + {% endif %} {% elif affected_by_advisories_v2_url %}
@@ -228,6 +291,8 @@ Aliases + + {% if grouped %} {% for advisory in fixing_advisories_v2 %} @@ -261,8 +326,43 @@ {% endfor %} - + {% else %} + + {% for advisory in fixing_advisories_v2 %} + + + + {{advisory.advisory_id }} + +
+ + + {{ advisory.summary|truncatewords:20 }} + + + {% for alias in advisory.aliases.all %} + {% if alias.url %} + {{ alias }} +
+ {% else %} + {{ alias }} +
+ {% endif %} + {% endfor %} + + + {% empty %} + + + This package is not known to fix any advisories. + + + {% endfor %} + + + {% endif %}
{% elif fixing_advisories_v2_url %}
diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py b/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py deleted file mode 100644 index 5b7f0c186..000000000 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_compute_advisory_content_hash.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from unittest.mock import patch - -import pytest - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash import ( - ComputeAdvisoryContentHash, -) - -pytestmark = pytest.mark.django_db - - -@pytest.fixture -def advisory_factory(): - def _create(count, with_hash=False, start=0): - objs = [] - for i in range(start, start + count): - objs.append( - AdvisoryV2( - summary=f"summary {i}", - advisory_content_hash="existing_hash" if with_hash else None, - unique_content_id=f"unique_id_{i}", - advisory_id=f"ADV-{i}", - datasource_id="ds", - avid=f"ds/ADV-{i}", - url=f"https://example.com/ADV-{i}", - ) - ) - return AdvisoryV2.objects.bulk_create(objs) - - return _create - - -def run_pipeline(): - pipeline = ComputeAdvisoryContentHash() - pipeline.compute_advisory_content_hash() - - -@patch( - "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" -) -def test_pipeline_updates_only_missing_hash(mock_compute, advisory_factory): - advisory_factory(3, with_hash=False, start=0) - advisory_factory(2, with_hash=True, start=100) - - mock_compute.return_value = "new_hash" - - run_pipeline() - - updated = AdvisoryV2.objects.filter(advisory_content_hash="new_hash").count() - untouched = AdvisoryV2.objects.filter(advisory_content_hash="existing_hash").count() - - assert updated == 3 - assert untouched == 2 - assert mock_compute.call_count == 3 - - -@patch( - "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" -) -def test_pipeline_bulk_update_batches(mock_compute, advisory_factory): - advisory_factory(6000, with_hash=False) - - mock_compute.return_value = "batch_hash" - - run_pipeline() - - assert AdvisoryV2.objects.filter(advisory_content_hash="batch_hash").count() == 6000 - - assert mock_compute.call_count == 6000 - - -@patch( - "vulnerabilities.pipelines.v2_improvers.compute_advisory_content_hash.compute_advisory_content" -) -def test_pipeline_no_advisories(mock_compute): - run_pipeline() - - assert mock_compute.call_count == 0 diff --git a/vulnerabilities/tests/test_advisory_merge.py b/vulnerabilities/tests/test_advisory_merge.py new file mode 100644 index 000000000..ddcc3cadb --- /dev/null +++ b/vulnerabilities/tests/test_advisory_merge.py @@ -0,0 +1,192 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import hashlib + +import pytest + +from vulnerabilities.models import AdvisoryAlias +from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import PackageV2 +from vulnerabilities.utils import compute_advisory_content_hash +from vulnerabilities.utils import delete_and_save_advisory_set +from vulnerabilities.utils import get_advisories_from_groups +from vulnerabilities.utils import get_merged_identifier_groups +from vulnerabilities.utils import merge_advisories +from vulnerabilities.utils import merge_and_save_grouped_advisories + + +@pytest.mark.django_db +class TestAdvisoryMerge: + def create_advisory(self, advisory_id, affected_versions, fixed_versions=None, precedence=None): + unique_content_id = hashlib.sha256(advisory_id.encode()).hexdigest() + + adv = AdvisoryV2.objects.create( + datasource_id="ghsa", + advisory_id=advisory_id, + avid=f"ghsa/{advisory_id}", + unique_content_id=unique_content_id, + url="https://example.com/advisory", + date_collected="2025-07-01T00:00:00Z", + precedence=precedence, + ) + + pkg = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + impact = ImpactedPackage.objects.create( + advisory=adv, + base_purl="pkg:pypi/sample", + ) + + # affected + for v in affected_versions: + p = PackageV2.objects.from_purl(f"pkg:pypi/sample@{v}") + impact.affecting_packages.add(p) + + # fixed + if fixed_versions: + for v in fixed_versions: + p = PackageV2.objects.from_purl(f"pkg:pypi/sample@{v}") + impact.fixed_by_packages.add(p) + + return adv + + def test_content_hash_same(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["2.0"]) + + h1 = compute_advisory_content_hash(adv1, package) + h2 = compute_advisory_content_hash(adv2, package) + + assert h1 == h2 + + def test_content_hash_different(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["3.0"]) + + assert compute_advisory_content_hash(adv1, package) != compute_advisory_content_hash( + adv2, package + ) + + def test_identifier_merging(self): + adv1 = self.create_advisory("A1", ["1.0"]) + adv2 = self.create_advisory("A2", ["1.0"]) + + alias = AdvisoryAlias.objects.create(alias="CVE-123") + + adv1.aliases.add(alias) + adv2.aliases.add(alias) + + groups = get_merged_identifier_groups([adv1, adv2]) + + assert len(groups) == 1 + identifiers, primary, secondary = groups[0] + + assert len(secondary) == 1 + assert primary in [adv1, adv2] + + def test_transitive_merge(self): + a1 = self.create_advisory("A1", ["1.0"]) + a2 = self.create_advisory("A2", ["1.0"]) + a3 = self.create_advisory("A3", ["1.0"]) + + alias_1 = AdvisoryAlias.objects.create(alias="CVE-1") + alias_2 = AdvisoryAlias.objects.create(alias="CVE-2") + + a1.aliases.add(alias_1) + a2.aliases.add(alias_1) + a2.aliases.add(alias_2) + a3.aliases.add(alias_2) + + groups = get_merged_identifier_groups([a1, a2, a3]) + + assert len(groups) == 1 + + def test_primary_selection_by_precedence(self): + a1 = self.create_advisory("A1", ["1.0"], precedence=1) + a2 = self.create_advisory("A2", ["1.0"], precedence=5) + + alias_1 = AdvisoryAlias.objects.create(alias="CVE-1") + + a1.aliases.add(alias_1) + a2.aliases.add(alias_1) + + groups = get_merged_identifier_groups([a1, a2]) + _, primary, _ = groups[0] + + assert primary == a2 + + def test_get_advisories_from_groups(self): + adv = self.create_advisory("GHSA-ABC-123", ["1.0"]) + adv.aliases.create(alias="CVE-999") + + groups = get_merged_identifier_groups([adv]) + result = get_advisories_from_groups(groups) + + assert result[0]["identifier"] == "GHSA-ABC-123" + assert len(result[0]["aliases"]) == 1 + + def test_delete_and_save_advisory_set(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"]) + adv2 = self.create_advisory("A2", ["1.0"]) + + adv1.aliases.create(alias="CVE-1") + + groups = [(set(adv1.aliases.all()), adv1, [adv2])] + + delete_and_save_advisory_set(groups, package, relation="affecting") + + assert AdvisorySet.objects.count() == 1 + assert AdvisorySetMember.objects.count() == 2 + + advisory_set = AdvisorySet.objects.first() + members = AdvisorySetMember.objects.filter(advisory_set=advisory_set) + + assert any(m.is_primary for m in members) + assert any(not m.is_primary for m in members) + + def test_merge_and_save_integration(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["2.0"]) + + alias = AdvisoryAlias.objects.create(alias="CVE-1") + + adv1.aliases.add(alias) + adv2.aliases.add(alias) + + result = merge_and_save_grouped_advisories( + package, + [adv1, adv2], + relation="test", + ) + + assert len(result) == 1 + assert AdvisorySet.objects.count() == 1 + assert AdvisorySetMember.objects.count() == 2 + + def test_merge_advisories_separates_different_content(self): + package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") + + adv1 = self.create_advisory("A1", ["1.0"], ["2.0"]) + adv2 = self.create_advisory("A2", ["1.0"], ["3.0"]) + + groups = merge_advisories([adv1, adv2], package) + + assert len(groups) == 2 diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index 6b88e5ee5..fa8a08b33 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -53,7 +53,7 @@ def test_packages_post_without_details(self): def test_packages_post_with_details(self): url = reverse("package-v3-list") - with self.assertNumQueries(23): + with self.assertNumQueries(33): response = self.client.post( url, data={ @@ -171,25 +171,6 @@ def setUp(self): self.client = APIClient(enforce_csrf_checks=True) - def test_packages_post_purl_with_many_advisories(self): - url = reverse("package-v3-list") - - with self.assertNumQueries(12): - response = self.client.post( - url, - data={ - "purls": ["pkg:pypi/sample@1.0.0"], - "details": True, - }, - format="json", - ) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - - results = response.data["results"] - self.assertEqual(len(results), 1) - self.assertIsNotNone(results[0]["affected_by_vulnerabilities_url"]) - def test_advisories_post(self): url = reverse("advisory-v3-list") diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 2dd606a92..5f791d30b 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -41,6 +41,7 @@ from univers.version_range import VersionRange from aboutcode.hashid import build_vcid +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set logger = logging.getLogger(__name__) @@ -845,59 +846,10 @@ def compute_patch_checksum(patch_text: str): return hashlib.sha512(patch_text.encode("utf-8")).hexdigest() -def group_advisories_by_content(advisories): - grouped = {} - - for advisory in advisories: - content_hash = ( - advisory.advisory_content_hash - if advisory.advisory_content_hash - else compute_advisory_content(advisory) - ) - - entry = grouped.setdefault( - content_hash, - {"primary": advisory, "secondary": set()}, - ) - - primary = entry["primary"] - - if advisory is primary: - continue - - if advisory.precedence > primary.precedence: - entry["primary"] = advisory - entry["secondary"].add(primary) - else: - entry["secondary"].add(advisory) - - return grouped - - -def compute_advisory_content(advisory_data): +def merge_advisories(advisories, package): """ - Compute a unique content hash for an advisory by normalizing its data and hashing it. - - :param advisory_data: An AdvisoryData object - :return: SHA-256 hash digest as content hash + Merge advisories based on their content hash and identifiers. """ - from vulnerabilities.models import AdvisoryV2 - - if isinstance(advisory_data, AdvisoryV2): - advisory_data = advisory_data.to_advisory_data() - normalized_data = { - "affected_packages": [ - pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg - ], - } - - normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) - content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() - - return content_hash - - -def merge_advisories(advisories, package): advisories = list(advisories) @@ -917,6 +869,8 @@ def merge_advisories(advisories, package): def compute_advisory_content_hash(adv, package): + """Compute a content hash for an advisory based on its affected and fixed packages for a given package. + This is used to determine if two advisories are the same based on their content.""" affected = [] fixed = [] @@ -943,6 +897,10 @@ def compute_advisory_content_hash(adv, package): def get_merged_identifier_groups(advisories): + """ + Merge advisories based on their identifiers (advisory_id and aliases). + Example: If two advisories share ``advisory_id`` or share an alias, they will be merged together. + """ identifier_groups = defaultdict(set) @@ -985,7 +943,7 @@ def get_merged_identifier_groups(advisories): for group in merged: identifiers = set() for adv in group: - for alias in adv.aliases.all(): + for alias in adv.aliases.all().order_by("alias"): identifiers.add(alias) primary = max(group, key=lambda a: a.precedence if a.precedence is not None else -1) @@ -995,3 +953,45 @@ def get_merged_identifier_groups(advisories): final_groups.append((identifiers, primary, secondary)) return final_groups + + +def get_advisories_from_groups(groups): + """ + Return a list of advisories from the merged groups of advisories. + """ + advisories = [] + for aliases, primary, _ in groups: + identifier = primary.advisory_id.split("/")[-1] + + filtered_aliases = [alias for alias in aliases if alias.alias != identifier] + + advisories.append( + {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} + ) + + return advisories + + +def merge_and_save_grouped_advisories(package, advisories, relation): + """ + Merge advisories based on their content and identifiers and save the merged advisories to the database. + """ + groups = merge_advisories(advisories, package) + delete_and_save_advisory_set(groups, package, relation) + advisories = get_advisories_from_groups(groups) + + return advisories + + +TYPES_WITH_MULTIPLE_IMPORTERS = [ + "pypi", + "maven", + "nuget", + "golang", + "npm", + "composer", + "hex", + "cargo", + "gem", + "conan", +] diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index c8bfc6634..8051dfb35 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -39,14 +39,15 @@ from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline +from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import group_advisories_by_content -from vulnerabilities.utils import merge_advisories +from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS +from vulnerabilities.utils import get_advisories_from_groups +from vulnerabilities.utils import merge_and_save_grouped_advisories from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env @@ -160,12 +161,7 @@ def get_queryset(self, query=None): on exact purl, partial purl or just name and namespace. """ query = query or self.request.GET.get("search") or "" - return ( - self.model.objects.search(query) - .prefetch_related() - .order_by("package_url") - .with_is_vulnerable() - ) + return self.model.objects.search(query).prefetch_related().with_is_vulnerable() class AffectedByAdvisoriesListView(ListView): @@ -220,57 +216,97 @@ def get_context_data(self, **kwargs): context["latest_non_vulnerable"] = latest_non_vulnerable context["package_search_form"] = PackageSearchForm(self.request.GET) + is_grouped = models.AdvisorySet.objects.filter(package=package).exists() + + if is_grouped: + context["grouped"] = True + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details + + affected_by_advisories_qs = models.AdvisorySet.objects.filter( + package=package, relation_type="affecting" + ).select_related("primary_advisory") + + fixing_advisories_qs = models.AdvisorySet.objects.filter( + package=package, relation_type="fixing" + ).select_related("primary_advisory") + + affected_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") + for adv in affected_by_advisories_qs + ] + fixing_groups = [ + (list(adv.aliases.all()), adv.primary_advisory, "") for adv in fixing_advisories_qs + ] + + affected_advisories = get_advisories_from_groups(affected_groups) + fixing_advisories = get_advisories_from_groups(fixing_groups) + + context["affected_by_advisories_v2"] = affected_advisories + context["fixing_advisories_v2"] = fixing_advisories + + return context + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", ) fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( purl=package.purl - ).prefetch_related( - "aliases", - "impacted_packages__affecting_packages", - "impacted_packages__fixed_by_packages", ) + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details + context["grouped"] = True + + affecting_advisories = affecting_advisories.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + affected_by_advisories = merge_and_save_grouped_advisories( + package, affecting_advisories, "affecting" + ) + + fixed_by_advisories = fixed_by_advisories.prefetch_related( + "aliases", + "impacted_packages__affecting_packages", + "impacted_packages__fixed_by_packages", + ) + + fixing_advisories = merge_and_save_grouped_advisories( + package, fixed_by_advisories, "fixing" + ) + + context["affected_by_advisories_v2"] = affected_by_advisories + context["fixing_advisories_v2"] = fixing_advisories + return context + + context["grouped"] = False + affected_by_advisories_url = None fixing_advisories_url = None affected_by_advisories_qs_ids = affecting_advisories.only("id") fixing_advisories_qs_ids = fixed_by_advisories.only("id") - affected_by_advisories = list(affected_by_advisories_qs_ids[:1001]) - if len(affected_by_advisories) > 1001: + affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) + if len(affected_by_advisories) > 101: affected_by_advisories_url = reverse_lazy( "affected_by_advisories_v2", kwargs={"purl": package.package_url} ) context["affected_by_advisories_v2_url"] = affected_by_advisories_url - context["affected_by_advisories_v2"] = [] - context["fixed_package_details"] = {} else: - advisories = [] - fixed_pkg_details = get_fixed_package_details(package) - groups = merge_advisories(affecting_advisories, package) - for aliases, primary, _ in groups: - identifier = primary.advisory_id.split("/")[-1] - - filtered_aliases = [alias for alias in aliases if alias.alias != identifier] - - advisories.append( - {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} - ) - - context["affected_by_advisories_v2"] = advisories context["fixed_package_details"] = fixed_pkg_details + context["affected_by_advisories_v2"] = affecting_advisories context["affected_by_advisories_v2_url"] = None - fixing_advisories = list(fixing_advisories_qs_ids[:1001]) - if len(fixing_advisories) > 1001: + fixing_advisories = list(fixing_advisories_qs_ids[:101]) + if len(fixing_advisories) > 101: fixing_advisories_url = reverse_lazy( "fixing_advisories_v2", kwargs={"purl": package.package_url} ) @@ -278,21 +314,7 @@ def get_context_data(self, **kwargs): context["fixing_advisories_v2"] = [] else: - advisories = [] - - fixed_pkg_details = get_fixed_package_details(package) - groups = merge_advisories(fixing_advisories, package) - for aliases, primary, _ in groups: - identifier = primary.advisory_id.split("/")[-1] - - filtered_aliases = [alias for alias in aliases if alias.alias != identifier] - - advisories.append( - {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} - ) - - context["fixing_advisories_v2"] = advisories - context["fixing_advisories_v2_url"] = None + context["fixing_advisories_v2"] = fixed_by_advisories return context @@ -430,7 +452,7 @@ def get_fixed_package_details(package): pkg_map = { p.id: p - for p in models.PackageV2.objects.filter(id__in=pkg_ids).annotate( + for p in models.PackageV2.objects.filter(id__in=pkg_ids, is_ghost=False).annotate( is_vulnerable=Exists( models.ImpactedPackage.objects.filter(affecting_packages=OuterRef("pk")) ) From 8dac89edfe352eb6ff146a2949bc665c9a700375 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 30 Mar 2026 17:36:32 +0530 Subject: [PATCH 42/65] Handle None in UI Signed-off-by: Tushar Goel --- .../templates/advisory_detail.html | 21 ++++++++++++++++--- .../templates/package_details_v2.html | 4 ++++ vulnerabilities/templates/packages_v2.html | 8 ++++++- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/templates/advisory_detail.html b/vulnerabilities/templates/advisory_detail.html index 595412df4..5e0e61584 100644 --- a/vulnerabilities/templates/advisory_detail.html +++ b/vulnerabilities/templates/advisory_detail.html @@ -214,9 +214,20 @@ {% for severity in severities %} {{ severity.scoring_system }} - {{ severity.value }} - + + {% if severity.value is not None %} + {{ severity.value }} + {% else %} + {{ "" }} + {% endif %} + + + {% if severity.url is not None %} + {{ severity.url }} + {% else %} + {{ "" }} + {% endif %} {% empty %} @@ -483,7 +494,11 @@
{% for severity_vector in severity_vectors %} {% if severity_vector.vector.version == '2.0' %} - Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} + Vector: {{ severity_vector.vector.vectorString }} + {% if severity_vector.origin %} + Found at + {{ severity_vector.origin }} + {% endif %} diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index 8c3f62756..a6c07c352 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -118,7 +118,11 @@ Risk score diff --git a/vulnerabilities/templates/packages_v2.html b/vulnerabilities/templates/packages_v2.html index 4348575da..f114a7159 100644 --- a/vulnerabilities/templates/packages_v2.html +++ b/vulnerabilities/templates/packages_v2.html @@ -62,7 +62,13 @@ target="_self">{{ package.purl }} - + {% empty %} From b20dc39b71f64d49288ae45ebbe4cbfd7ef79250 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 30 Mar 2026 18:41:55 +0530 Subject: [PATCH 43/65] Handle large number of advisories case Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 103 ++++++++++++++++++++------------------ vulnerabilities/views.py | 69 ++++++++++++------------- 2 files changed, 89 insertions(+), 83 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index ea82dcce3..fb9847a1b 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -215,6 +215,45 @@ def get_affected_by_vulnerabilities(self, package): advisories = [] + if package.type not in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_ids = advisories_qs.only("id") + + advisories_ids = list(advisories_ids[:101]) + if len(advisories_ids) > 100: + return None + + advisory_by_avid = {adv.avid: adv for adv in advisories_qs} + avids = advisory_by_avid.keys() + + impacts = ( + package.affected_in_impacts.filter(advisory__avid__in=avids) + .select_related("advisory") + .prefetch_related("fixed_by_packages") + ) + + impact_by_avid = {impact.advisory.avid: impact for impact in impacts} + + result = [] + + for advisory in advisories_qs: + impact = impact_by_avid.get(advisory.avid) + if not impact: + continue + + result.append( + { + "advisory_id": advisory.advisory_id.split("/")[-1], + "aliases": [alias.alias for alias in advisory.aliases.all()], + "summary": advisory.summary, + "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "severity": advisory.weighted_severity, + "exploitability": advisory.exploitability, + "risk_score": advisory.risk_score, + } + ) + + return result + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() if is_grouped: @@ -239,43 +278,25 @@ def get_affected_by_vulnerabilities(self, package): advisories = merge_and_save_grouped_advisories(package, advisories_qs, "affecting") return self.return_advisories_data(package, advisories_qs, advisories) - advisories_ids = advisories_qs.only("id") - - advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: - return None - - advisory_by_avid = {adv.avid: adv for adv in advisories_qs} - avids = advisory_by_avid.keys() - - impacts = ( - package.affected_in_impacts.filter(advisory__avid__in=avids) - .select_related("advisory") - .prefetch_related("fixed_by_packages") - ) - - impact_by_avid = {impact.advisory.avid: impact for impact in impacts} - - result = [] + def get_fixing_vulnerabilities(self, package): + advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) - for advisory in advisories_qs: - impact = impact_by_avid.get(advisory.avid) - if not impact: - continue + if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + advisories_ids = advisories_qs.only("id") - result.append( - { - "advisory_id": advisory.advisory_id.split("/")[-1], - "aliases": [alias.alias for alias in advisory.aliases.all()], - "summary": advisory.summary, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], - } - ) + advisories_ids = list(advisories_ids[:101]) + if len(advisories_ids) > 100: + return None - return result + results = [] - def get_fixing_vulnerabilities(self, package): - advisories_qs = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl(package.package_url) + for advisory in advisories_qs: + results.append( + { + "advisory_id": advisory.advisory_id.split("/")[-1], + } + ) + return results advisories = [] @@ -302,22 +323,6 @@ def get_fixing_vulnerabilities(self, package): advisories = merge_and_save_grouped_advisories(package, advisories_qs, "fixing") return self.return_fixing_advisories_data(advisories) - advisories_ids = advisories_qs.only("id") - - advisories_ids = list(advisories_ids[:101]) - if len(advisories_ids) > 100: - return None - - results = [] - - for advisory in advisories_qs: - results.append( - { - "advisory_id": advisory.advisory_id.split("/")[-1], - } - ) - return results - def return_fixing_advisories_data(self, advisories): result = [] for advisory in advisories: diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 8051dfb35..829ff22a7 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -216,6 +216,41 @@ def get_context_data(self, **kwargs): context["latest_non_vulnerable"] = latest_non_vulnerable context["package_search_form"] = PackageSearchForm(self.request.GET) + if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + context["grouped"] = False + + affected_by_advisories_url = None + fixing_advisories_url = None + + affected_by_advisories_qs_ids = affecting_advisories.only("id") + fixing_advisories_qs_ids = fixed_by_advisories.only("id") + + affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) + if len(affected_by_advisories) > 101: + affected_by_advisories_url = reverse_lazy( + "affected_by_advisories_v2", kwargs={"purl": package.package_url} + ) + context["affected_by_advisories_v2_url"] = affected_by_advisories_url + + else: + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details + context["affected_by_advisories_v2"] = affecting_advisories + context["affected_by_advisories_v2_url"] = None + + fixing_advisories = list(fixing_advisories_qs_ids[:101]) + if len(fixing_advisories) > 101: + fixing_advisories_url = reverse_lazy( + "fixing_advisories_v2", kwargs={"purl": package.package_url} + ) + context["fixing_advisories_v2_url"] = fixing_advisories_url + context["fixing_advisories_v2"] = [] + + else: + context["fixing_advisories_v2"] = fixed_by_advisories + + return context + is_grouped = models.AdvisorySet.objects.filter(package=package).exists() if is_grouped: @@ -284,40 +319,6 @@ def get_context_data(self, **kwargs): context["fixing_advisories_v2"] = fixing_advisories return context - context["grouped"] = False - - affected_by_advisories_url = None - fixing_advisories_url = None - - affected_by_advisories_qs_ids = affecting_advisories.only("id") - fixing_advisories_qs_ids = fixed_by_advisories.only("id") - - affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) - if len(affected_by_advisories) > 101: - affected_by_advisories_url = reverse_lazy( - "affected_by_advisories_v2", kwargs={"purl": package.package_url} - ) - context["affected_by_advisories_v2_url"] = affected_by_advisories_url - - else: - fixed_pkg_details = get_fixed_package_details(package) - context["fixed_package_details"] = fixed_pkg_details - context["affected_by_advisories_v2"] = affecting_advisories - context["affected_by_advisories_v2_url"] = None - - fixing_advisories = list(fixing_advisories_qs_ids[:101]) - if len(fixing_advisories) > 101: - fixing_advisories_url = reverse_lazy( - "fixing_advisories_v2", kwargs={"purl": package.package_url} - ) - context["fixing_advisories_v2_url"] = fixing_advisories_url - context["fixing_advisories_v2"] = [] - - else: - context["fixing_advisories_v2"] = fixed_by_advisories - - return context - def get_object(self, queryset=None): if queryset is None: queryset = self.get_queryset() From 4f97321140c3b5d16d5ffe2a32d66b57fae7d9d4 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 30 Mar 2026 19:06:49 +0530 Subject: [PATCH 44/65] Fix views Signed-off-by: Tushar Goel --- .../templates/advisory_detail.html | 18 +++++++++++++--- vulnerabilities/views.py | 21 ++++++++++++------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/templates/advisory_detail.html b/vulnerabilities/templates/advisory_detail.html index 5e0e61584..90f1d6d8b 100644 --- a/vulnerabilities/templates/advisory_detail.html +++ b/vulnerabilities/templates/advisory_detail.html @@ -137,7 +137,11 @@ applications, or networks. This metric is determined automatically based on the discovery of known exploits."> Exploitability @@ -146,7 +150,11 @@ data-tooltip="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10." >Weighted Severity @@ -157,7 +165,11 @@ " >Risk diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 829ff22a7..87e0c71d6 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -217,6 +217,14 @@ def get_context_data(self, **kwargs): context["package_search_form"] = PackageSearchForm(self.request.GET) if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( + purl=package.purl + ) + + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ) + context["grouped"] = False affected_by_advisories_url = None @@ -282,15 +290,14 @@ def get_context_data(self, **kwargs): return context - affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl - ) - - fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl - ) + ) - if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: + fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( + purl=package.purl + ) fixed_pkg_details = get_fixed_package_details(package) context["fixed_package_details"] = fixed_pkg_details context["grouped"] = True From 4f2d1495c1fa40283d4e180febbed52430500cd9 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 30 Mar 2026 19:50:05 +0530 Subject: [PATCH 45/65] Fix views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 87e0c71d6..63d02c5b1 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -234,7 +234,7 @@ def get_context_data(self, **kwargs): fixing_advisories_qs_ids = fixed_by_advisories.only("id") affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) - if len(affected_by_advisories) > 101: + if len(affected_by_advisories) > 100: affected_by_advisories_url = reverse_lazy( "affected_by_advisories_v2", kwargs={"purl": package.package_url} ) @@ -247,7 +247,7 @@ def get_context_data(self, **kwargs): context["affected_by_advisories_v2_url"] = None fixing_advisories = list(fixing_advisories_qs_ids[:101]) - if len(fixing_advisories) > 101: + if len(fixing_advisories) > 100: fixing_advisories_url = reverse_lazy( "fixing_advisories_v2", kwargs={"purl": package.package_url} ) From 610c205482060462014170855864108832826dc8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 00:26:56 +0530 Subject: [PATCH 46/65] Add risk, severity and exploits Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 3 ++ .../templates/package_details_v2.html | 15 ++++++++++ vulnerabilities/utils.py | 29 ++++++++++++++++--- vulnerabilities/views.py | 4 +-- 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index fb9847a1b..986096165 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -356,6 +356,9 @@ def return_advisories_data(self, package, advisories_qs, advisories): { "advisory_id": advisory["identifier"], "aliases": [alias.alias for alias in advisory["aliases"]], + "weighted_severity": advisory["weighted_severity"], + "exploitability": advisory["exploitability"], + "risk_score": advisory["risk_score"], "summary": advisory["advisory"].summary, "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], } diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index a6c07c352..8511348ec 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -142,6 +142,7 @@ + @@ -197,6 +198,13 @@ {% endif %} {% endwith %} + {% empty %} @@ -258,6 +266,13 @@ {% endif %} {% endwith %} + {% empty %} diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 5f791d30b..ecf2f6878 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -960,13 +960,34 @@ def get_advisories_from_groups(groups): Return a list of advisories from the merged groups of advisories. """ advisories = [] - for aliases, primary, _ in groups: + weighted_severity = None + exploitability = None + risk_score = None + for aliases, primary, secondaries in groups: + severity_scores = [] + exploitability_scores = [] identifier = primary.advisory_id.split("/")[-1] - filtered_aliases = [alias for alias in aliases if alias.alias != identifier] - + severity_scores.extend([adv.weighted_severity for adv in secondaries]) + exploitability_scores.extend([adv.exploitability for adv in secondaries]) + severity_scores.append(primary.weighted_severity) + exploitability_scores.append(primary.exploitability) + if severity_scores: + weighted_severity = round(max(severity_scores), 1) + if exploitability_scores: + exploitability = max(exploitability_scores) + if exploitability and weighted_severity: + risk_score = min(float(exploitability * weighted_severity), 10.0) + risk_score = round(risk_score, 1) advisories.append( - {"aliases": filtered_aliases, "advisory": primary, "identifier": identifier} + { + "aliases": filtered_aliases, + "advisory": primary, + "identifier": identifier, + "weighted_severity": weighted_severity, + "exploitability": exploitability, + "risk_score": risk_score, + } ) return advisories diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 63d02c5b1..11852aa59 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -218,7 +218,7 @@ def get_context_data(self, **kwargs): if not package.type in TYPES_WITH_MULTIPLE_IMPORTERS: affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl + purl=package.purl ) fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( @@ -292,7 +292,7 @@ def get_context_data(self, **kwargs): if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( - purl=package.purl + purl=package.purl ) fixed_by_advisories = AdvisoryV2.objects.latest_fixed_by_advisories_for_purl( From af98f071e6408a6c37a62dade593364db740bb67 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 00:30:31 +0530 Subject: [PATCH 47/65] Dedupe fixed_by_packages Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 986096165..2803ac9b8 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -360,7 +360,7 @@ def return_advisories_data(self, package, advisories_qs, advisories): "exploitability": advisory["exploitability"], "risk_score": advisory["risk_score"], "summary": advisory["advisory"].summary, - "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()], + "fixed_by_packages": list(set([pkg.purl for pkg in impact.fixed_by_packages.all()])), } ) From 97da322e5edeb3af477d65909d7e925a7a4b9e70 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 00:53:30 +0530 Subject: [PATCH 48/65] Fix severity and exploit calculation Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 50 ++++++++++++++++++++++++++++++++------- vulnerabilities/views.py | 46 ++++++++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 2803ac9b8..cf8f1c3ec 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -21,6 +21,7 @@ from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySet +from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness @@ -257,12 +258,26 @@ def get_affected_by_vulnerabilities(self, package): is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() if is_grouped: - affected_by_advisories_qs = AdvisorySet.objects.filter( - package=package, relation_type="affecting" - ).select_related("primary_advisory") + affected_by_advisories_qs = ( + AdvisorySet.objects.filter(package=package, relation_type="affecting") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) affected_groups = [ - (list(adv.aliases.all()), adv.primary_advisory, "") + ( + list(adv.aliases.all()), + adv.primary_advisory, + [member.advisory for member in adv.secondary_members], + ) for adv in affected_by_advisories_qs ] @@ -303,12 +318,27 @@ def get_fixing_vulnerabilities(self, package): is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() if is_grouped: - fixing_advisories_qs = AdvisorySet.objects.filter( - package=package, relation_type="fixing" - ).select_related("primary_advisory") + fixing_advisories_qs = ( + AdvisorySet.objects.filter(package=package, relation_type="fixing") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) fixing_groups = [ - (list(adv.aliases.all()), adv.primary_advisory, "") for adv in fixing_advisories_qs + ( + list(adv.aliases.all()), + adv.primary_advisory, + [member.advisory for member in adv.secondary_members], + ) + for adv in fixing_advisories_qs ] advisories = get_advisories_from_groups(fixing_groups) @@ -360,7 +390,9 @@ def return_advisories_data(self, package, advisories_qs, advisories): "exploitability": advisory["exploitability"], "risk_score": advisory["risk_score"], "summary": advisory["advisory"].summary, - "fixed_by_packages": list(set([pkg.purl for pkg in impact.fixed_by_packages.all()])), + "fixed_by_packages": list( + set([pkg.purl for pkg in impact.fixed_by_packages.all()]) + ), } ) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 11852aa59..c88c437b5 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -42,7 +42,6 @@ from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline -from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS @@ -266,20 +265,49 @@ def get_context_data(self, **kwargs): fixed_pkg_details = get_fixed_package_details(package) context["fixed_package_details"] = fixed_pkg_details - affected_by_advisories_qs = models.AdvisorySet.objects.filter( - package=package, relation_type="affecting" - ).select_related("primary_advisory") + affected_by_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="affecting") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) - fixing_advisories_qs = models.AdvisorySet.objects.filter( - package=package, relation_type="fixing" - ).select_related("primary_advisory") + fixing_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="fixing") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) affected_groups = [ - (list(adv.aliases.all()), adv.primary_advisory, "") + ( + list(adv.aliases.all()), + adv.primary_advisory, + [a.advisory for a in adv.secondary_members], + ) for adv in affected_by_advisories_qs ] fixing_groups = [ - (list(adv.aliases.all()), adv.primary_advisory, "") for adv in fixing_advisories_qs + ( + list(adv.aliases.all()), + adv.primary_advisory, + [a.advisory for a in adv.secondary_members], + ) + for adv in fixing_advisories_qs ] affected_advisories = get_advisories_from_groups(affected_groups) From 07433dc7d65552ff9d6cb50c10a2f68c7e5e77fb Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 16:13:12 +0530 Subject: [PATCH 49/65] Fix grouping Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 49 +++++++++------ vulnerabilities/models.py | 26 ++++++++ .../group_advisories_for_packages.py | 7 ++- vulnerabilities/pipes/group_advisories.py | 12 ++-- vulnerabilities/tests/test_advisory_merge.py | 7 ++- vulnerabilities/utils.py | 61 ++++++++++++------- vulnerabilities/views.py | 27 +++++--- 7 files changed, 127 insertions(+), 62 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index cf8f1c3ec..ea1586394 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from typing import List from urllib.parse import urlencode from django.db.models import Exists @@ -25,6 +26,8 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness +from vulnerabilities.models import Group +from vulnerabilities.models import GroupedAdvisory from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.throttling import PermissionBasedUserRateThrottle @@ -273,15 +276,15 @@ def get_affected_by_vulnerabilities(self, package): ) affected_groups = [ - ( - list(adv.aliases.all()), - adv.primary_advisory, - [member.advisory for member in adv.secondary_members], + Group( + aliases=list(adv.aliases.all()), + primary_advisory=adv.primary_advisory, + secondaries=[member.advisory for member in adv.secondary_members], ) for adv in affected_by_advisories_qs ] - advisories = get_advisories_from_groups(affected_groups) + advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) return self.return_advisories_data(package, advisories_qs, advisories) if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: @@ -290,7 +293,9 @@ def get_affected_by_vulnerabilities(self, package): "impacted_packages__affecting_packages", "impacted_packages__fixed_by_packages", ) - advisories = merge_and_save_grouped_advisories(package, advisories_qs, "affecting") + advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + package, advisories_qs, "affecting" + ) return self.return_advisories_data(package, advisories_qs, advisories) def get_fixing_vulnerabilities(self, package): @@ -333,15 +338,15 @@ def get_fixing_vulnerabilities(self, package): ) fixing_groups = [ - ( - list(adv.aliases.all()), - adv.primary_advisory, - [member.advisory for member in adv.secondary_members], + Group( + aliases=list(adv.aliases.all()), + primary_advisory=adv.primary_advisory, + secondaries=[member.advisory for member in adv.secondary_members], ) for adv in fixing_advisories_qs ] - advisories = get_advisories_from_groups(fixing_groups) + advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) return self.return_fixing_advisories_data(advisories) if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: @@ -350,15 +355,18 @@ def get_fixing_vulnerabilities(self, package): "impacted_packages__affecting_packages", "impacted_packages__fixed_by_packages", ) - advisories = merge_and_save_grouped_advisories(package, advisories_qs, "fixing") + advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( + package, advisories_qs, "fixing" + ) return self.return_fixing_advisories_data(advisories) def return_fixing_advisories_data(self, advisories): result = [] for advisory in advisories: + assert isinstance(advisory, GroupedAdvisory) result.append( { - "advisory_id": advisory["identifier"], + "advisory_id": advisory.identifier, } ) @@ -378,18 +386,19 @@ def return_advisories_data(self, package, advisories_qs, advisories): result = [] for advisory in advisories: - impact = impact_by_avid.get(advisory["advisory"].avid) + assert isinstance(advisory, GroupedAdvisory) + impact = impact_by_avid.get(advisory.advisory.avid) if not impact: continue result.append( { - "advisory_id": advisory["identifier"], - "aliases": [alias.alias for alias in advisory["aliases"]], - "weighted_severity": advisory["weighted_severity"], - "exploitability": advisory["exploitability"], - "risk_score": advisory["risk_score"], - "summary": advisory["advisory"].summary, + "advisory_id": advisory.identifier, + "aliases": [alias.alias for alias in advisory.aliases], + "weighted_severity": advisory.weighted_severity, + "exploitability": advisory.exploitability, + "risk_score": advisory.risk_score, + "summary": advisory.advisory.summary, "fixed_by_packages": list( set([pkg.purl for pkg in impact.fixed_by_packages.all()]) ), diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index f51a92dbd..45d8acf55 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -20,6 +20,9 @@ from operator import attrgetter from traceback import format_exc as traceback_format_exc from typing import List +from typing import NamedTuple +from typing import Optional +from typing import Set from typing import Union from urllib.parse import urljoin @@ -3714,3 +3717,26 @@ def __str__(self): class Meta: unique_together = ("vector", "source_advisory") + + +class Group(NamedTuple): + """ + A Group of advisories that have been merged together based on their content and identifiers. + """ + + aliases: Set[AdvisoryAlias] + primary: AdvisoryV2 + secondaries: List[AdvisoryV2] + + +class GroupedAdvisory(NamedTuple): + """ + A GroupedAdvisory represents a single advisory that has been grouped with its aliases and related advisories. + """ + + aliases: Set[AdvisoryAlias] + advisory: AdvisoryV2 + identifier: str + weighted_severity: Optional[float] + exploitability: Optional[float] + risk_score: Optional[float] diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index d2c8f6296..db49447ff 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -7,7 +7,10 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from typing import List + from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import Group from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.group_advisories import delete_and_save_advisory_set @@ -48,8 +51,8 @@ def group_advisoris_for_packages(logger=None): ) try: - affected_groups = merge_advisories(affecting_advisories, package) - fixed_by_groups = merge_advisories(fixed_by_advisories, package) + affected_groups: List[Group] = merge_advisories(affecting_advisories, package) + fixed_by_groups: List[Group] = merge_advisories(fixed_by_advisories, package) delete_and_save_advisory_set(affected_groups, package, relation="affecting") delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") except Exception as e: diff --git a/vulnerabilities/pipes/group_advisories.py b/vulnerabilities/pipes/group_advisories.py index d66365706..983ac3386 100644 --- a/vulnerabilities/pipes/group_advisories.py +++ b/vulnerabilities/pipes/group_advisories.py @@ -14,31 +14,33 @@ def delete_and_save_advisory_set(groups, package, relation=None): from vulnerabilities.models import AdvisorySet from vulnerabilities.models import AdvisorySetMember + from vulnerabilities.models import Group AdvisorySet.objects.filter(package=package, relation_type=relation).delete() membership_to_create = [] - for identifiers, primary, secondary in groups: + for group in groups: + assert isinstance(group, Group) advisory_set = AdvisorySet.objects.create( package=package, relation_type=relation, - primary_advisory=primary, + primary_advisory=group.primary, ) - advisory_set.aliases.add(*identifiers) + advisory_set.aliases.add(*group.aliases) advisory_set.save() membership_to_create.append( AdvisorySetMember( advisory_set=advisory_set, - advisory=primary, + advisory=group.primary, is_primary=True, ) ) - for adv in secondary: + for adv in group.secondaries: membership_to_create.append( AdvisorySetMember( advisory_set=advisory_set, diff --git a/vulnerabilities/tests/test_advisory_merge.py b/vulnerabilities/tests/test_advisory_merge.py index ddcc3cadb..08b586ff3 100644 --- a/vulnerabilities/tests/test_advisory_merge.py +++ b/vulnerabilities/tests/test_advisory_merge.py @@ -15,6 +15,7 @@ from vulnerabilities.models import AdvisorySet from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import Group from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 from vulnerabilities.utils import compute_advisory_content_hash @@ -136,8 +137,8 @@ def test_get_advisories_from_groups(self): groups = get_merged_identifier_groups([adv]) result = get_advisories_from_groups(groups) - assert result[0]["identifier"] == "GHSA-ABC-123" - assert len(result[0]["aliases"]) == 1 + assert result[0].identifier == "GHSA-ABC-123" + assert len(result[0].aliases) == 1 def test_delete_and_save_advisory_set(self): package = PackageV2.objects.from_purl("pkg:pypi/sample@1.0.0") @@ -147,7 +148,7 @@ def test_delete_and_save_advisory_set(self): adv1.aliases.create(alias="CVE-1") - groups = [(set(adv1.aliases.all()), adv1, [adv2])] + groups = [Group(aliases=set(adv1.aliases.all()), primary=adv1, secondaries=[adv2])] delete_and_save_advisory_set(groups, package, relation="affecting") diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index ecf2f6878..e8a13821e 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -20,7 +20,9 @@ from functools import total_ordering from http import HTTPStatus from typing import List +from typing import NamedTuple from typing import Optional +from typing import Set from typing import Tuple from typing import Union from unittest.mock import MagicMock @@ -850,6 +852,7 @@ def merge_advisories(advisories, package): """ Merge advisories based on their content hash and identifiers. """ + from vulnerabilities.models import Group advisories = list(advisories) @@ -859,7 +862,7 @@ def merge_advisories(advisories, package): content_hash = compute_advisory_content_hash(adv, package) content_hash_map[content_hash].append(adv) - final_groups = [] + final_groups: List[Group] = [] for group in content_hash_map.values(): groups = get_merged_identifier_groups(group) @@ -901,6 +904,7 @@ def get_merged_identifier_groups(advisories): Merge advisories based on their identifiers (advisory_id and aliases). Example: If two advisories share ``advisory_id`` or share an alias, they will be merged together. """ + from vulnerabilities.models import Group identifier_groups = defaultdict(set) @@ -938,7 +942,7 @@ def get_merged_identifier_groups(advisories): if adv not in all_grouped: merged.append({adv}) - final_groups = [] + final_groups: List[Group] = [] for group in merged: identifiers = set() @@ -950,7 +954,7 @@ def get_merged_identifier_groups(advisories): secondary = [a for a in group if a != primary] - final_groups.append((identifiers, primary, secondary)) + final_groups.append(Group(aliases=identifiers, primary=primary, secondaries=secondary)) return final_groups @@ -959,35 +963,48 @@ def get_advisories_from_groups(groups): """ Return a list of advisories from the merged groups of advisories. """ + from vulnerabilities.models import Group + from vulnerabilities.models import GroupedAdvisory + advisories = [] - weighted_severity = None - exploitability = None - risk_score = None - for aliases, primary, secondaries in groups: + + for group in groups: + + assert isinstance(group, Group) + weighted_severity = None + exploitability = None + risk_score = None + severity_scores = [] - exploitability_scores = [] - identifier = primary.advisory_id.split("/")[-1] - filtered_aliases = [alias for alias in aliases if alias.alias != identifier] - severity_scores.extend([adv.weighted_severity for adv in secondaries]) - exploitability_scores.extend([adv.exploitability for adv in secondaries]) - severity_scores.append(primary.weighted_severity) - exploitability_scores.append(primary.exploitability) + severity_scores.append(group.primary.weighted_severity or 0.0) + severity_scores.extend([adv.weighted_severity or 0.0 for adv in group.secondaries]) + if severity_scores: weighted_severity = round(max(severity_scores), 1) + + exploitability_scores = [] + exploitability_scores.append(group.primary.exploitability or 0.0) + exploitability_scores.extend([adv.exploitability or 0.0 for adv in group.secondaries]) + if exploitability_scores: exploitability = max(exploitability_scores) + if exploitability and weighted_severity: risk_score = min(float(exploitability * weighted_severity), 10.0) risk_score = round(risk_score, 1) + + identifier = group.primary.advisory_id.split("/")[-1] + filtered_aliases = [alias for alias in group.aliases if alias.alias != identifier] + advisories.append( - { - "aliases": filtered_aliases, - "advisory": primary, - "identifier": identifier, - "weighted_severity": weighted_severity, - "exploitability": exploitability, - "risk_score": risk_score, - } + GroupedAdvisory( + aliases=filtered_aliases, + advisory=group.primary, + identifier=identifier, + weighted_severity=weighted_severity, + exploitability=exploitability, + risk_score=risk_score, + ) ) return advisories diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index c88c437b5..f9274a18d 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -8,6 +8,7 @@ # import logging from collections import defaultdict +from typing import List from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -39,6 +40,8 @@ from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import Group +from vulnerabilities.models import GroupedAdvisory from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline @@ -295,23 +298,27 @@ def get_context_data(self, **kwargs): affected_groups = [ ( - list(adv.aliases.all()), - adv.primary_advisory, - [a.advisory for a in adv.secondary_members], + Group( + aliases=list(adv.aliases.all()), + primary=adv.primary_advisory, + secondaries=[a.advisory for a in adv.secondary_members], + ) ) for adv in affected_by_advisories_qs ] fixing_groups = [ ( - list(adv.aliases.all()), - adv.primary_advisory, - [a.advisory for a in adv.secondary_members], + Group( + aliases=list(adv.aliases.all()), + primary=adv.primary_advisory, + secondaries=[a.advisory for a in adv.secondary_members], + ) ) for adv in fixing_advisories_qs ] - affected_advisories = get_advisories_from_groups(affected_groups) - fixing_advisories = get_advisories_from_groups(fixing_groups) + affected_advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) + fixing_advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) context["affected_by_advisories_v2"] = affected_advisories context["fixing_advisories_v2"] = fixing_advisories @@ -336,7 +343,7 @@ def get_context_data(self, **kwargs): "impacted_packages__fixed_by_packages", ) - affected_by_advisories = merge_and_save_grouped_advisories( + affected_by_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( package, affecting_advisories, "affecting" ) @@ -346,7 +353,7 @@ def get_context_data(self, **kwargs): "impacted_packages__fixed_by_packages", ) - fixing_advisories = merge_and_save_grouped_advisories( + fixing_advisories: List[GroupedAdvisory] = merge_and_save_grouped_advisories( package, fixed_by_advisories, "fixing" ) From cfb2d7d00290656eec944d4e1a3aacef9257f038 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 16:20:58 +0530 Subject: [PATCH 50/65] Fix API Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index ea1586394..a15d5a0cd 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -278,7 +278,7 @@ def get_affected_by_vulnerabilities(self, package): affected_groups = [ Group( aliases=list(adv.aliases.all()), - primary_advisory=adv.primary_advisory, + primary=adv.primary_advisory, secondaries=[member.advisory for member in adv.secondary_members], ) for adv in affected_by_advisories_qs @@ -340,7 +340,7 @@ def get_fixing_vulnerabilities(self, package): fixing_groups = [ Group( aliases=list(adv.aliases.all()), - primary_advisory=adv.primary_advisory, + primary=adv.primary_advisory, secondaries=[member.advisory for member in adv.secondary_members], ) for adv in fixing_advisories_qs From 28c5c638c1d7eef23ee0bb6d73d08eb3b16ae4b8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 16:37:50 +0530 Subject: [PATCH 51/65] Ignore goruped case Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 112 +++++++++++++++++----------------- vulnerabilities/views.py | 122 +++++++++++++++++++------------------- 2 files changed, 117 insertions(+), 117 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index a15d5a0cd..0d1df5418 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -258,34 +258,34 @@ def get_affected_by_vulnerabilities(self, package): return result - is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() - - if is_grouped: - affected_by_advisories_qs = ( - AdvisorySet.objects.filter(package=package, relation_type="affecting") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - affected_groups = [ - Group( - aliases=list(adv.aliases.all()), - primary=adv.primary_advisory, - secondaries=[member.advisory for member in adv.secondary_members], - ) - for adv in affected_by_advisories_qs - ] - - advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) - return self.return_advisories_data(package, advisories_qs, advisories) + # is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() + + # if is_grouped: + # affected_by_advisories_qs = ( + # AdvisorySet.objects.filter(package=package, relation_type="affecting") + # .select_related("primary_advisory") + # .prefetch_related( + # Prefetch( + # "members", + # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + # "advisory" + # ), + # to_attr="secondary_members", + # ) + # ) + # ) + + # affected_groups = [ + # Group( + # aliases=list(adv.aliases.all()), + # primary=adv.primary_advisory, + # secondaries=[member.advisory for member in adv.secondary_members], + # ) + # for adv in affected_by_advisories_qs + # ] + + # advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) + # return self.return_advisories_data(package, advisories_qs, advisories) if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_qs = advisories_qs.prefetch_related( @@ -320,34 +320,34 @@ def get_fixing_vulnerabilities(self, package): advisories = [] - is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() - - if is_grouped: - fixing_advisories_qs = ( - AdvisorySet.objects.filter(package=package, relation_type="fixing") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - fixing_groups = [ - Group( - aliases=list(adv.aliases.all()), - primary=adv.primary_advisory, - secondaries=[member.advisory for member in adv.secondary_members], - ) - for adv in fixing_advisories_qs - ] - - advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) - return self.return_fixing_advisories_data(advisories) + # is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() + + # if is_grouped: + # fixing_advisories_qs = ( + # AdvisorySet.objects.filter(package=package, relation_type="fixing") + # .select_related("primary_advisory") + # .prefetch_related( + # Prefetch( + # "members", + # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + # "advisory" + # ), + # to_attr="secondary_members", + # ) + # ) + # ) + + # fixing_groups = [ + # Group( + # aliases=list(adv.aliases.all()), + # primary=adv.primary_advisory, + # secondaries=[member.advisory for member in adv.secondary_members], + # ) + # for adv in fixing_advisories_qs + # ] + + # advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) + # return self.return_fixing_advisories_data(advisories) if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_qs = advisories_qs.prefetch_related( diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index f9274a18d..a9d599b49 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -261,69 +261,69 @@ def get_context_data(self, **kwargs): return context - is_grouped = models.AdvisorySet.objects.filter(package=package).exists() - - if is_grouped: - context["grouped"] = True - fixed_pkg_details = get_fixed_package_details(package) - context["fixed_package_details"] = fixed_pkg_details - - affected_by_advisories_qs = ( - models.AdvisorySet.objects.filter(package=package, relation_type="affecting") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - fixing_advisories_qs = ( - models.AdvisorySet.objects.filter(package=package, relation_type="fixing") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - affected_groups = [ - ( - Group( - aliases=list(adv.aliases.all()), - primary=adv.primary_advisory, - secondaries=[a.advisory for a in adv.secondary_members], - ) - ) - for adv in affected_by_advisories_qs - ] - fixing_groups = [ - ( - Group( - aliases=list(adv.aliases.all()), - primary=adv.primary_advisory, - secondaries=[a.advisory for a in adv.secondary_members], - ) - ) - for adv in fixing_advisories_qs - ] - - affected_advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) - fixing_advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) + # is_grouped = models.AdvisorySet.objects.filter(package=package).exists() + + # if is_grouped: + # context["grouped"] = True + # fixed_pkg_details = get_fixed_package_details(package) + # context["fixed_package_details"] = fixed_pkg_details + + # affected_by_advisories_qs = ( + # models.AdvisorySet.objects.filter(package=package, relation_type="affecting") + # .select_related("primary_advisory") + # .prefetch_related( + # Prefetch( + # "members", + # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + # "advisory" + # ), + # to_attr="secondary_members", + # ) + # ) + # ) - context["affected_by_advisories_v2"] = affected_advisories - context["fixing_advisories_v2"] = fixing_advisories + # fixing_advisories_qs = ( + # models.AdvisorySet.objects.filter(package=package, relation_type="fixing") + # .select_related("primary_advisory") + # .prefetch_related( + # Prefetch( + # "members", + # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + # "advisory" + # ), + # to_attr="secondary_members", + # ) + # ) + # ) - return context + # affected_groups = [ + # ( + # Group( + # aliases=list(adv.aliases.all()), + # primary=adv.primary_advisory, + # secondaries=[a.advisory for a in adv.secondary_members], + # ) + # ) + # for adv in affected_by_advisories_qs + # ] + # fixing_groups = [ + # ( + # Group( + # aliases=list(adv.aliases.all()), + # primary=adv.primary_advisory, + # secondaries=[a.advisory for a in adv.secondary_members], + # ) + # ) + # for adv in fixing_advisories_qs + # ] + + # affected_advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) + # fixing_advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) + + # context["affected_by_advisories_v2"] = affected_advisories + # context["fixing_advisories_v2"] = fixing_advisories + + # return context if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( From 7c4859e732b95b55f2df46ee2e023796cce2731d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 16:54:06 +0530 Subject: [PATCH 52/65] Revert grouping Signed-off-by: Tushar Goel --- vulnerabilities/api_v3.py | 112 +++++++++++++++++----------------- vulnerabilities/views.py | 122 +++++++++++++++++++------------------- 2 files changed, 117 insertions(+), 117 deletions(-) diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index 0d1df5418..a15d5a0cd 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -258,34 +258,34 @@ def get_affected_by_vulnerabilities(self, package): return result - # is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() - - # if is_grouped: - # affected_by_advisories_qs = ( - # AdvisorySet.objects.filter(package=package, relation_type="affecting") - # .select_related("primary_advisory") - # .prefetch_related( - # Prefetch( - # "members", - # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - # "advisory" - # ), - # to_attr="secondary_members", - # ) - # ) - # ) - - # affected_groups = [ - # Group( - # aliases=list(adv.aliases.all()), - # primary=adv.primary_advisory, - # secondaries=[member.advisory for member in adv.secondary_members], - # ) - # for adv in affected_by_advisories_qs - # ] - - # advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) - # return self.return_advisories_data(package, advisories_qs, advisories) + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="affecting").exists() + + if is_grouped: + affected_by_advisories_qs = ( + AdvisorySet.objects.filter(package=package, relation_type="affecting") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + affected_groups = [ + Group( + aliases=list(adv.aliases.all()), + primary=adv.primary_advisory, + secondaries=[member.advisory for member in adv.secondary_members], + ) + for adv in affected_by_advisories_qs + ] + + advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) + return self.return_advisories_data(package, advisories_qs, advisories) if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_qs = advisories_qs.prefetch_related( @@ -320,34 +320,34 @@ def get_fixing_vulnerabilities(self, package): advisories = [] - # is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() - - # if is_grouped: - # fixing_advisories_qs = ( - # AdvisorySet.objects.filter(package=package, relation_type="fixing") - # .select_related("primary_advisory") - # .prefetch_related( - # Prefetch( - # "members", - # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - # "advisory" - # ), - # to_attr="secondary_members", - # ) - # ) - # ) - - # fixing_groups = [ - # Group( - # aliases=list(adv.aliases.all()), - # primary=adv.primary_advisory, - # secondaries=[member.advisory for member in adv.secondary_members], - # ) - # for adv in fixing_advisories_qs - # ] - - # advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) - # return self.return_fixing_advisories_data(advisories) + is_grouped = AdvisorySet.objects.filter(package=package, relation_type="fixing").exists() + + if is_grouped: + fixing_advisories_qs = ( + AdvisorySet.objects.filter(package=package, relation_type="fixing") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + fixing_groups = [ + Group( + aliases=list(adv.aliases.all()), + primary=adv.primary_advisory, + secondaries=[member.advisory for member in adv.secondary_members], + ) + for adv in fixing_advisories_qs + ] + + advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) + return self.return_fixing_advisories_data(advisories) if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: advisories_qs = advisories_qs.prefetch_related( diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index a9d599b49..f9274a18d 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -261,69 +261,69 @@ def get_context_data(self, **kwargs): return context - # is_grouped = models.AdvisorySet.objects.filter(package=package).exists() - - # if is_grouped: - # context["grouped"] = True - # fixed_pkg_details = get_fixed_package_details(package) - # context["fixed_package_details"] = fixed_pkg_details - - # affected_by_advisories_qs = ( - # models.AdvisorySet.objects.filter(package=package, relation_type="affecting") - # .select_related("primary_advisory") - # .prefetch_related( - # Prefetch( - # "members", - # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - # "advisory" - # ), - # to_attr="secondary_members", - # ) - # ) - # ) + is_grouped = models.AdvisorySet.objects.filter(package=package).exists() - # fixing_advisories_qs = ( - # models.AdvisorySet.objects.filter(package=package, relation_type="fixing") - # .select_related("primary_advisory") - # .prefetch_related( - # Prefetch( - # "members", - # queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - # "advisory" - # ), - # to_attr="secondary_members", - # ) - # ) - # ) + if is_grouped: + context["grouped"] = True + fixed_pkg_details = get_fixed_package_details(package) + context["fixed_package_details"] = fixed_pkg_details + + affected_by_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="affecting") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + fixing_advisories_qs = ( + models.AdvisorySet.objects.filter(package=package, relation_type="fixing") + .select_related("primary_advisory") + .prefetch_related( + Prefetch( + "members", + queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( + "advisory" + ), + to_attr="secondary_members", + ) + ) + ) + + affected_groups = [ + ( + Group( + aliases=list(adv.aliases.all()), + primary=adv.primary_advisory, + secondaries=[a.advisory for a in adv.secondary_members], + ) + ) + for adv in affected_by_advisories_qs + ] + fixing_groups = [ + ( + Group( + aliases=list(adv.aliases.all()), + primary=adv.primary_advisory, + secondaries=[a.advisory for a in adv.secondary_members], + ) + ) + for adv in fixing_advisories_qs + ] + + affected_advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) + fixing_advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) - # affected_groups = [ - # ( - # Group( - # aliases=list(adv.aliases.all()), - # primary=adv.primary_advisory, - # secondaries=[a.advisory for a in adv.secondary_members], - # ) - # ) - # for adv in affected_by_advisories_qs - # ] - # fixing_groups = [ - # ( - # Group( - # aliases=list(adv.aliases.all()), - # primary=adv.primary_advisory, - # secondaries=[a.advisory for a in adv.secondary_members], - # ) - # ) - # for adv in fixing_advisories_qs - # ] - - # affected_advisories: List[GroupedAdvisory] = get_advisories_from_groups(affected_groups) - # fixing_advisories: List[GroupedAdvisory] = get_advisories_from_groups(fixing_groups) - - # context["affected_by_advisories_v2"] = affected_advisories - # context["fixing_advisories_v2"] = fixing_advisories - - # return context + context["affected_by_advisories_v2"] = affected_advisories + context["fixing_advisories_v2"] = fixing_advisories + + return context if package.type in TYPES_WITH_MULTIPLE_IMPORTERS: affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( From 8f17b7ea3cc66f451657fbf17dc827de4ae0020d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 18:20:45 +0530 Subject: [PATCH 53/65] Change advisory ID for pypa importer Signed-off-by: Tushar Goel --- vulnerabilities/pipelines/v2_importers/pypa_importer.py | 3 +++ vulnerabilities/pipes/osv_v2.py | 9 +++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/pypa_importer.py b/vulnerabilities/pipelines/v2_importers/pypa_importer.py index 7a80ed70f..142c8a385 100644 --- a/vulnerabilities/pipelines/v2_importers/pypa_importer.py +++ b/vulnerabilities/pipelines/v2_importers/pypa_importer.py @@ -59,11 +59,14 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]: ) advisory_text = advisory.read_text() advisory_dict = saneyaml.load(advisory_text) + advisory_path = advisory.relative_to(base_directory) + advisory_id = advisory_path.parent.stem + "/" + advisory_path.stem yield parse_advisory_data_v3( raw_data=advisory_dict, supported_ecosystems=["pypi"], advisory_url=advisory_url, advisory_text=advisory_text, + advisory_id=advisory_id, ) def clean_downloads(self): diff --git a/vulnerabilities/pipes/osv_v2.py b/vulnerabilities/pipes/osv_v2.py index e70ba4a4a..65b5a5904 100644 --- a/vulnerabilities/pipes/osv_v2.py +++ b/vulnerabilities/pipes/osv_v2.py @@ -59,13 +59,18 @@ def parse_advisory_data_v3( - raw_data: dict, supported_ecosystems, advisory_url: str, advisory_text: str + raw_data: dict, + supported_ecosystems, + advisory_url: str, + advisory_text: str, + advisory_id: Optional[str] = None, ) -> Optional[AdvisoryDataV2]: """ Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and a ``supported_ecosystem`` string. """ - advisory_id = raw_data.get("id") or "" + if not advisory_id: + advisory_id = raw_data.get("id") or "" if not advisory_id: logger.error(f"Missing advisory id in OSV data: {raw_data}") return None From 5951dfd91ea28498bd4a3d314698804fbbaaf6d8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 18:24:02 +0530 Subject: [PATCH 54/65] Change documentation Signed-off-by: Tushar Goel --- PIPELINES-AVID.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PIPELINES-AVID.rst b/PIPELINES-AVID.rst index 43de21e19..3d82400f8 100644 --- a/PIPELINES-AVID.rst +++ b/PIPELINES-AVID.rst @@ -55,7 +55,7 @@ * - project-kb-statements_v2 - Vulnerability ID of the record * - pypa_importer_v2 - - ID of the OSV record + - {package_name}/{ID of the OSV record} * - pysec_importer_v2 - ID of the OSV record * - redhat_importer_v2 From 63f3416e12421da18f979459636b717e8b10dc4f Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 31 Mar 2026 19:18:19 +0530 Subject: [PATCH 55/65] Increase page_size for pagination Signed-off-by: Tushar Goel --- etc/nginx/conf.d/default.conf | 1 + vulnerablecode/settings.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/nginx/conf.d/default.conf b/etc/nginx/conf.d/default.conf index ce8081c06..131882479 100644 --- a/etc/nginx/conf.d/default.conf +++ b/etc/nginx/conf.d/default.conf @@ -12,6 +12,7 @@ server { proxy_redirect off; client_max_body_size 10G; proxy_read_timeout 600s; + proxy_set_header X-Forwarded-Proto $scheme; } location /static/ { diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 435cb8953..eaf2c1276 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -251,7 +251,7 @@ "EXCEPTION_HANDLER": "vulnerabilities.throttling.throttled_exception_handler", "DEFAULT_PAGINATION_CLASS": "vulnerabilities.pagination.SmallResultSetPagination", # Limit the load on the Database returning a small number of records by default. https://github.com/nexB/vulnerablecode/issues/819 - "PAGE_SIZE": 10, + "PAGE_SIZE": 100, # for API docs "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", "DATETIME_FORMAT": "%Y-%m-%dT%H:%M:%SZ", From e168ba9e671709aef868431dd5347ac9f9e026e2 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 1 Apr 2026 16:35:43 +0530 Subject: [PATCH 56/65] Remove risk score from UI Signed-off-by: Tushar Goel --- vulnerabilities/templates/package_details_v2.html | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index 8511348ec..a6c07c352 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -142,7 +142,6 @@ - @@ -198,13 +197,6 @@ {% endif %} {% endwith %} - {% empty %} @@ -266,13 +258,6 @@ {% endif %} {% endwith %} - {% empty %} From 012c3ac9f0756271bb7bc574c3643dc15519da3c Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 1 Apr 2026 16:39:42 +0530 Subject: [PATCH 57/65] Update API V3 usage Signed-off-by: Tushar Goel --- api_v3_usage.rst | 2 +- vulnerabilities/api_v3.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api_v3_usage.rst b/api_v3_usage.rst index 26ed9377f..0da3fe1af 100644 --- a/api_v3_usage.rst +++ b/api_v3_usage.rst @@ -83,7 +83,7 @@ Parameters: - ``purls`` — list of package URLs to query - ``details`` — boolean (default: ``false``) -- ``approximate`` — boolean (default: ``false``) +- ``ignore_qualifiers_subpath`` — boolean (default: ``false``) The ``approximate`` flag replaces the previous ``plain_purl`` parameter. When set to ``true``, qualifiers and subpaths in PURLs are ignored. diff --git a/vulnerabilities/api_v3.py b/vulnerabilities/api_v3.py index a15d5a0cd..ffa5bd941 100644 --- a/vulnerabilities/api_v3.py +++ b/vulnerabilities/api_v3.py @@ -43,13 +43,13 @@ class PackageQuerySerializer(serializers.Serializer): default=list, ) details = serializers.BooleanField(default=False) - approximate = serializers.BooleanField(default=False) + ignore_qualifiers_subpath = serializers.BooleanField(default=False) def validate(self, data): if not data["purls"]: - if data["details"] or data["approximate"]: + if data["details"] or data["ignore_qualifiers_subpath"]: raise serializers.ValidationError( - "details and approximate must be false when purls is empty" + "``details`` and ``ignore_qualifiers_subpath`` must be false when purls is empty" ) return data @@ -428,7 +428,7 @@ def create(self, request, *args, **kwargs): purls = serializer.validated_data["purls"] details = serializer.validated_data["details"] - approximate = serializer.validated_data["approximate"] + ignore_qualifiers_subpath = serializer.validated_data["ignore_qualifiers_subpath"] if not purls: impacted = ImpactedPackageAffecting.objects.filter(package_id=OuterRef("id")) @@ -444,7 +444,7 @@ def create(self, request, *args, **kwargs): plain_purls = None - if approximate: + if ignore_qualifiers_subpath: plain_purls = [ str( PackageURL( @@ -458,7 +458,7 @@ def create(self, request, *args, **kwargs): ] if not details: - if approximate: + if ignore_qualifiers_subpath: query = ( PackageV2.objects.filter(plain_package_url__in=plain_purls) .values_list("plain_package_url", flat=True) @@ -476,7 +476,7 @@ def create(self, request, *args, **kwargs): page = self.paginate_queryset(query) return self.get_paginated_response(page) - if approximate: + if ignore_qualifiers_subpath: query = ( PackageV2.objects.filter(plain_package_url__in=plain_purls) .order_by("plain_package_url") From ae1b71b89f657363ad34454ec37e4aacf911f1c0 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 1 Apr 2026 16:45:21 +0530 Subject: [PATCH 58/65] Change tests Signed-off-by: Tushar Goel --- api_v3_usage.rst | 38 +++++++++++++++++----------- vulnerabilities/tests/test_api_v3.py | 4 +-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/api_v3_usage.rst b/api_v3_usage.rst index 0da3fe1af..23732703c 100644 --- a/api_v3_usage.rst +++ b/api_v3_usage.rst @@ -85,7 +85,7 @@ Parameters: - ``details`` — boolean (default: ``false``) - ``ignore_qualifiers_subpath`` — boolean (default: ``false``) -The ``approximate`` flag replaces the previous ``plain_purl`` parameter. +The ``ignore_qualifiers_subpath`` flag replaces the previous ``plain_purl`` parameter. When set to ``true``, qualifiers and subpaths in PURLs are ignored. @@ -140,12 +140,16 @@ Example response: "purl": "pkg:npm/atob@2.0.3", "affected_by_vulnerabilities": [ { - "advisory_id": "nodejs_security_wg/npm-403", - "fixed_by_packages": [ - "pkg:npm/atob@2.1.0" - ], - "duplicate_advisory_ids": [] - } + "advisory_id": "GHSA-g5vw-3h65-2q3v", + "aliases": [], + "weighted_severity": null, + "exploitability_score": null, + "risk_score": null, + "summary": "Access control vulnerable to user data", + "fixed_by_packages": [ + "pkg:pypi/accesscontrol@7.2" + ], + }, ], "fixing_vulnerabilities": [], "next_non_vulnerable_version": "2.1.0", @@ -165,7 +169,7 @@ Using Approximate Matching { "purls": ["pkg:npm/atob@2.0.3?foo=bar"], - "approximate": true, + "ignore_qualifiers_subpath": true, "details": true } @@ -181,13 +185,17 @@ Example response: { "purl": "pkg:npm/atob@2.0.3", "affected_by_vulnerabilities": [ - { - "advisory_id": "nodejs_security_wg/npm-403", - "fixed_by_packages": [ - "pkg:npm/atob@2.1.0" - ], - "duplicate_advisory_ids": [] - } + { + "advisory_id": "GHSA-g5vw-3h65-2q3v", + "aliases": [], + "weighted_severity": null, + "exploitability_score": null, + "risk_score": null, + "summary": "Access control vulnerable to user data", + "fixed_by_packages": [ + "pkg:pypi/accesscontrol@7.2" + ], + } ], "fixing_vulnerabilities": [], "next_non_vulnerable_version": "2.1.0", diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index fa8a08b33..c7201af6a 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -126,14 +126,14 @@ def test_packages_pagination(self): self.assertIn("results", response.data) self.assertIn("next", response.data) - def test_packages_approximate(self): + def test_packages_ignore_qualifiers_subpath(self): url = reverse("package-v3-list") response = self.client.post( url, data={ "purls": ["pkg:pypi/sample@1.0.0?foo=bar"], - "approximate": True, + "ignore_qualifiers_subpath": True, "details": False, }, format="json", From 0eb2acd49158ce1e2f30ee5bb7a51da394881598 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 1 Apr 2026 16:47:57 +0530 Subject: [PATCH 59/65] Update changelog and prep for release Signed-off-by: Tushar Goel --- CHANGELOG.rst | 6 ++++++ setup.cfg | 2 +- vulnerablecode/__init__.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 294004e08..4c3d9efb4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Release notes ============= +Version v38.0.0 +--------------------- + +- This is a major version, we have changed our V3 API, refer to ``api_v3_usage.rst`` for details. +- We have started grouping advisories which have aliases or identifiers in common and also affect same set of packages together. + Version v37.0.0 --------------------- diff --git a/setup.cfg b/setup.cfg index 7e11ae621..5c8efc7dd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vulnerablecode -version = 37.0.0 +version = 38.0.0 license = Apache-2.0 AND CC-BY-SA-4.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index 13c70b495..80b725801 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -14,7 +14,7 @@ import git -__version__ = "37.0.0" +__version__ = "38.0.0" PROJECT_DIR = Path(__file__).resolve().parent From 67cf3645dd37f9c69ac6e0fa96232bd2dae4e347 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 1 Apr 2026 23:27:31 +0530 Subject: [PATCH 60/65] Fix tests Signed-off-by: Tushar Goel --- etc/nginx/conf.d/default.conf | 1 + vulnerabilities/pipes/osv_v2.py | 12 +++++++----- vulnerabilities/tests/test_api_v2.py | 8 ++++---- vulnerabilities/tests/test_api_v3.py | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/etc/nginx/conf.d/default.conf b/etc/nginx/conf.d/default.conf index 131882479..754f65b76 100644 --- a/etc/nginx/conf.d/default.conf +++ b/etc/nginx/conf.d/default.conf @@ -13,6 +13,7 @@ server { client_max_body_size 10G; proxy_read_timeout 600s; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; } location /static/ { diff --git a/vulnerabilities/pipes/osv_v2.py b/vulnerabilities/pipes/osv_v2.py index 65b5a5904..0f8a29e78 100644 --- a/vulnerabilities/pipes/osv_v2.py +++ b/vulnerabilities/pipes/osv_v2.py @@ -69,17 +69,19 @@ def parse_advisory_data_v3( Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and a ``supported_ecosystem`` string. """ - if not advisory_id: - advisory_id = raw_data.get("id") or "" - if not advisory_id: + adv_id = raw_data.get("id") + if not adv_id: logger.error(f"Missing advisory id in OSV data: {raw_data}") return None + aliases = raw_data.get("aliases") or [] + if not advisory_id: + advisory_id = adv_id + else: + aliases.append(adv_id) summary = raw_data.get("summary") or "" details = raw_data.get("details") or "" summary = build_description(summary=summary, description=details) - aliases = raw_data.get("aliases") or [] aliases.extend(raw_data.get("upstream", [])) - date_published = get_published_date(raw_data=raw_data) severities = list(get_severities(raw_data=raw_data, url=advisory_url)) references = get_references_v2(raw_data=raw_data) diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py index c4abe3b97..be447ab0b 100644 --- a/vulnerabilities/tests/test_api_v2.py +++ b/vulnerabilities/tests/test_api_v2.py @@ -185,8 +185,8 @@ def test_list_vulnerabilities_pagination(self): self.assertIn("previous", response.data) # The 'vulnerabilities' dictionary should contain vulnerabilities up to the page limit self.assertEqual( - len(response.data["results"]["vulnerabilities"]), 10 - ) # Assuming default page size is 10 + len(response.data["results"]["vulnerabilities"]), 14 + ) # Assuming default page size is 100 class PackageV2ViewSetTest(APITestCase): @@ -346,8 +346,8 @@ def test_list_packages_pagination(self): self.assertIn("next", response.data) self.assertIn("previous", response.data) self.assertEqual( - len(response.data["results"]["packages"]), 10 - ) # Assuming default page size is 10 + len(response.data["results"]["packages"]), 14 + ) # Assuming default page size is 100 def test_invalid_vulnerability_filter(self): """ diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index c7201af6a..280662f2c 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -182,7 +182,7 @@ def test_advisories_post(self): ) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(len(response.data["results"]), 10) + self.assertEqual(len(response.data["results"]), 100) advisory = response.data["results"][0] self.assertEqual(advisory["advisory_id"], "ghsa_importer/GHSA-12341") @@ -229,5 +229,5 @@ def test_get_all_vulnerable_purls(self): self.assertEqual(response.status_code, status.HTTP_200_OK) results = response.data["results"] - self.assertEqual(len(results), 10) + self.assertEqual(len(results), 100) self.assertIn("next", response.data) From fd5250976e754f7b3f507cb8291c2a21a45e7daa Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 1 Apr 2026 23:41:10 +0530 Subject: [PATCH 61/65] Fix views for ungrouped advisories Signed-off-by: Tushar Goel --- .../templates/package_details_v2.html | 16 +++++++------- vulnerabilities/views.py | 22 +++++++++++++++++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index a6c07c352..b6aa84009 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -211,15 +211,15 @@ {% for advisory in affected_by_advisories_v2 %}
Exploitability (E) + {% if package.risk_score is not None %} {{package.risk_score}} + {% else %} + {{""}} + {% endif %}
{{ package.is_vulnerable|yesno:"Yes,No" }}{{ package.risk_score }} + {% if package.risk_score is not None %} + {{ package.risk_score }} + {% else %} + {{ "" }} + {% endif %} +
- {{ advisory.exploitability }} + {% if advisory.exploitability is not None %} + {{ advisory.exploitability }} + {% else %} + {{ "" }} + {% endif %}
- {{ advisory.weighted_severity }} + {% if advisory.weighted_severity is not None %} + {{ advisory.weighted_severity }} + {% else %} + {{ "" }} + {% endif %}
- {{ advisory.risk_score }} + {% if advisory.risk_score is not None %} + {{ advisory.risk_score }} + {% else %} + {{ "" }} + {% endif %}
Advisory Summary Fixed in package versionRisk score
+ {% if advisory.risk_score is not None %} + {{ advisory.risk_score }} + {% else %} + {{ "" }} + {% endif %} +
+ {% if advisory.risk_score is not None %} + {{ advisory.risk_score }} + {% else %} + {{ "" }} + {% endif %} +
Advisory Summary Fixed in package versionRisk score
- {% if advisory.risk_score is not None %} - {{ advisory.risk_score }} - {% else %} - {{ "" }} - {% endif %} -
- {% if advisory.risk_score is not None %} - {{ advisory.risk_score }} - {% else %} - {{ "" }} - {% endif %} -
- + {{advisory.advisory_id }}
- {% if advisory.aliases.all|length != 0 %} + {% if advisory.advisory.aliases.all|length != 0 %} Aliases: {% endif %}
- {% for alias in advisory.aliases.all %} + {% for alias in advisory.advisory.aliases.all %} {% if alias.url %} {{ alias }} @@ -232,10 +232,10 @@ 		- {{ advisory.summary|truncatewords:20 }} + {{ advisory.advisory.summary|truncatewords:20 }} - {% with fixed=fixed_package_details|get_item:advisory.avid %} + {% with fixed=fixed_package_details|get_item:advisory.advisory.avid %} {% if fixed %} {% for item in fixed %}
@@ -336,16 +336,16 @@ {% for advisory in fixing_advisories_v2 %}
- + {{advisory.advisory_id }}
- {{ advisory.summary|truncatewords:20 }} + {{ advisory.advisory.summary|truncatewords:20 }} - {% for alias in advisory.aliases.all %} + {% for alias in advisory.advisory.aliases.all %} {% if alias.url %} {{ alias }} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index f9274a18d..2a3d737a4 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -245,7 +245,15 @@ def get_context_data(self, **kwargs): else: fixed_pkg_details = get_fixed_package_details(package) context["fixed_package_details"] = fixed_pkg_details - context["affected_by_advisories_v2"] = affecting_advisories + affecting_advs = [] + for adv in affecting_advisories: + affecting_advs.append( + { + "advisory_id": adv.advisory_id.split("/")[-1], + "advisory": adv, + } + ) + context["affected_by_advisories_v2"] = affecting_advs context["affected_by_advisories_v2_url"] = None fixing_advisories = list(fixing_advisories_qs_ids[:101]) @@ -257,7 +265,17 @@ def get_context_data(self, **kwargs): context["fixing_advisories_v2"] = [] else: - context["fixing_advisories_v2"] = fixed_by_advisories + fixed_by_advisories = fixed_by_advisories.prefetch_related( + "aliases", + ) + fixed_by_advisories = list(fixed_by_advisories) + fix_advs = [] + for fixed_by_adv in fixed_by_advisories: + fix_advs.append( + {"advisory_id": fixed_by_adv.advisory_id.split("/")[-1], "advisory": fixed_by_adv} + ) + + context["fixing_advisories_v2"] = fix_advs return context From 959709cfe7426eaf81c4508192ad79bb4458c4f7 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 2 Apr 2026 11:29:19 +0530 Subject: [PATCH 62/65] Minor fixes Signed-off-by: Tushar Goel --- vulnerabilities/improvers/__init__.py | 2 -- .../v2_improvers/group_advisories_for_packages.py | 2 +- vulnerabilities/tests/test_api_v3.py | 9 +++++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 3e991d658..d55ecafdb 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -20,7 +20,6 @@ from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees -from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( computer_package_version_rank as compute_version_rank_v2, @@ -70,7 +69,6 @@ enhance_with_metasploit_v2.MetasploitImproverPipeline, compute_package_risk_v2.ComputePackageRiskPipeline, compute_version_rank_v2.ComputeVersionRankPipeline, - compute_advisory_todo_v2.ComputeToDo, unfurl_version_range_v2.UnfurlVersionRangePipeline, compute_advisory_todo.ComputeToDo, collect_ssvc_trees.CollectSSVCPipeline, diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index db49447ff..b34727078 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -19,7 +19,7 @@ class GroupAdvisoriesForPackages(VulnerableCodePipeline): - """Detect and flag packages that do not exist upstream.""" + """Group advisories for packages that have multiple importers""" pipeline_id = "group_advisories_for_packages" diff --git a/vulnerabilities/tests/test_api_v3.py b/vulnerabilities/tests/test_api_v3.py index 280662f2c..36dd7fba1 100644 --- a/vulnerabilities/tests/test_api_v3.py +++ b/vulnerabilities/tests/test_api_v3.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + from django.urls import reverse from packageurl import PackageURL from rest_framework import status From 78ca5283336a94044c70736f8fd26f226b309e11 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 2 Apr 2026 11:50:42 +0530 Subject: [PATCH 63/65] Compute rank while unfurling Signed-off-by: Tushar Goel --- .../group_advisories_for_packages.py | 4 +- .../v2_improvers/unfurl_version_range.py | 2 + .../templates/package_details_v3.html | 367 ------------------ vulnerabilities/views.py | 109 +----- vulnerablecode/urls.py | 1 - 5 files changed, 8 insertions(+), 475 deletions(-) delete mode 100644 vulnerabilities/templates/package_details_v3.html diff --git a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py index b34727078..ea6fc9185 100644 --- a/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/group_advisories_for_packages.py @@ -33,7 +33,7 @@ def group_advisories_for_packages(self): def group_advisoris_for_packages(logger=None): for package in PackageV2.objects.filter(type__in=TYPES_WITH_MULTIPLE_IMPORTERS).iterator(): - print(f"Grouping advisories for package {package.purl}") + logger(f"Grouping advisories for package {package.purl}") affecting_advisories = AdvisoryV2.objects.latest_affecting_advisories_for_purl( purl=package.purl ).prefetch_related( @@ -56,5 +56,5 @@ def group_advisoris_for_packages(logger=None): delete_and_save_advisory_set(affected_groups, package, relation="affecting") delete_and_save_advisory_set(fixed_by_groups, package, relation="fixing") except Exception as e: - print(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") + logger(f"Failed rebuilding advisory sets for package {package.purl}: {e!r}") continue diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 48d40e439..1d603b88a 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -118,6 +118,8 @@ def bulk_create_with_m2m(purls, impact, relation, logger): affected_packages_v2 = PackageV2.objects.bulk_get_or_create_from_purls(purls=purls) + affected_packages_v2[-1].calculate_version_rank + relations = [ relation(impacted_package=impact, package=package) for package in affected_packages_v2 ] diff --git a/vulnerabilities/templates/package_details_v3.html b/vulnerabilities/templates/package_details_v3.html deleted file mode 100644 index 44ec1c297..000000000 --- a/vulnerabilities/templates/package_details_v3.html +++ /dev/null @@ -1,367 +0,0 @@ -{% extends "base.html" %} -{% load humanize %} -{% load widget_tweaks %} -{% load static %} -{% load url_filters %} -{% load utils %} - -{% block title %} -VulnerableCode Package Details - {{ package.purl }} -{% endblock %} - -{% block content %} -
- {% include "package_search_box_v2.html"%} -
- -{% if package %} -
-
-
-
- Package details: - {{ package.purl }} - -
-
- -
- -
- -
-
-
- {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %} -
- {% else %} -
- {% endif %} - - - - - - - {% if package.is_ghost %} - - - - - {% endif %} - -
- - purl - - - {{ package.purl }} -
- Tags - - - Ghost - -
-
- {% if affected_by_advisories_v2|length != 0 or affected_by_advisories_v2_url %} - -
- - - - - - - - - - - - - - - -
- Next non-vulnerable version - - {% if next_non_vulnerable.version %} - {{ next_non_vulnerable.version }} - {% else %} - None. - {% endif %} -
- Latest non-vulnerable version - - {% if latest_non_vulnerable.version %} - {{ latest_non_vulnerable.version }} - {% else %} - None. - {% endif %} -
- Risk score - - {{package.risk_score}} -
-
- - {% endif %} - -
- {% if affected_by_advisories_v2|length != 0 %} -
- Vulnerabilities affecting this package ({{ affected_by_advisories_v2|length }}) -
- - - - - - - - - - - - - {% for advisory in affected_by_advisories_v2 %} - - - - - - - - {% empty %} - - - - {% endfor %} - -
AdvisorySourceDate PublishedSummaryFixed in package version
- - {{advisory.primary_advisory.advisory_id }} - -
- {% if advisory.identifiers|length != 0 %} - Aliases: - {% endif %} -
- {% for alias in advisory.identifiers %} - {% if alias.url %} - {{ alias }} -
- {% else %} - {{ alias }} -
- {% endif %} - {% endfor %} -
- {% if advisory.secondary_members|length != 0 %} -

Supporting advisories are listed below the primary advisory.

- {% for secondary in advisory.secondary_members %} - - {{secondary.advisory.avid }}
- - {% endfor %} - {% endif %} - 		- {{advisory.primary_advisory.url}} - - {{advisory.primary_advisory.date_published}} - - {{ advisory.primary_advisory.summary }} - - {% with fixed=fixed_package_details|get_item:advisory.primary_advisory.avid %} - {% if fixed %} - {% for item in fixed %} -
- {{ item.pkg.version }} -
- {% if item.pkg.is_vulnerable %} - - Vulnerable - - {% else %} - - Not vulnerable - - {% endif %} -
- {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endwith %} -
- This package is not known to be subject of any advisories. -
- {% elif affected_by_advisories_v2_url %} -
- This package is subject to more than 100 advisories. Please refer to the following - URL for vulnerabilities affecting this package: Advisories -
- {% else %} -
- This package is not known to be subject of any advisories. -
- {% endif %} -
- -
- {% if fixing_advisories_v2|length != 0 %} -
- Vulnerabilities fixed by this package ({{ fixing_advisories_v2|length }}) -
- - - - - - - - - - - - - {% for advisory in fixing_advisories_v2 %} - - - - - - - - {% empty %} - - - - {% endfor %} - -
AdvisorySourceDate PublishedSummaryAliases
- - {{advisory.primary_advisory.advisory_id }} - -
- {% if advisory.secondary_members|length != 0 %} -

Supporting advisories are listed below the primary advisory.

- {% for secondary in advisory.secondary_members %} - - {{secondary.advisory.avid }}
- - {% endfor %} - {% endif %} - 		- {{advisory.primary_advisory.url}} - - {{advisory.primary_advisory.date_published}} - - {{ advisory.primary_advisory.summary }} - - {% for alias in advisory.identifiers %} - {% if alias.url %} - {{ alias }} -
- {% else %} - {{ alias }} -
- {% endif %} - {% endfor %} -
- This package is not known to fix any advisories. -
- -
- {% elif fixing_advisories_v2_url %} -
- This package is known to fix more than 100 advisories. Please refer to the following - URL for vulnerabilities fixed by this package: Advisories -
- {% else %} -
- This package is not known to fix any advisories. -
- {% endif %} -
-
-
- - -
-
-
-
- -{% endif %} -{% endblock %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 2a3d737a4..4f9f396ea 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -272,7 +272,10 @@ def get_context_data(self, **kwargs): fix_advs = [] for fixed_by_adv in fixed_by_advisories: fix_advs.append( - {"advisory_id": fixed_by_adv.advisory_id.split("/")[-1], "advisory": fixed_by_adv} + { + "advisory_id": fixed_by_adv.advisory_id.split("/")[-1], + "advisory": fixed_by_adv, + } ) context["fixing_advisories_v2"] = fix_advs @@ -399,110 +402,6 @@ def get_object(self, queryset=None): return package -class PackageV3Details(DetailView): - model = models.PackageV2 - template_name = "package_details_v3.html" - slug_url_kwarg = "purl" - slug_field = "purl" - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - package = self.object - - next_non_vulnerable, latest_non_vulnerable = package.get_non_vulnerable_versions() - - context["package"] = package - context["next_non_vulnerable"] = next_non_vulnerable - context["latest_non_vulnerable"] = latest_non_vulnerable - context["package_search_form"] = PackageSearchForm(self.request.GET) - - affected_by_advisories_qs = ( - models.AdvisorySet.objects.filter(package=package, relation_type="affecting") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - fixing_advisories_qs = ( - models.AdvisorySet.objects.filter(package=package, relation_type="fixing") - .select_related("primary_advisory") - .prefetch_related( - Prefetch( - "members", - queryset=AdvisorySetMember.objects.filter(is_primary=False).select_related( - "advisory" - ), - to_attr="secondary_members", - ) - ) - ) - - print(affected_by_advisories_qs) - print(fixing_advisories_qs) - - affected_by_advisories_url = None - fixing_advisories_url = None - - affected_by_advisories_qs_ids = affected_by_advisories_qs.only("id") - fixing_advisories_qs_ids = fixing_advisories_qs.only("id") - - # affected_by_advisories = list(affected_by_advisories_qs_ids[:101]) - # if len(affected_by_advisories) > 100: - # affected_by_advisories_url = reverse_lazy( - # "affected_by_advisories_v2", kwargs={"purl": package.package_url} - # ) - # context["affected_by_advisories_v2_url"] = affected_by_advisories_url - # context["affected_by_advisories_v2"] = [] - # context["fixed_package_details"] = {} - - # else: - fixed_pkg_details = get_fixed_package_details(package) - - context["affected_by_advisories_v2"] = affected_by_advisories_qs - context["fixed_package_details"] = fixed_pkg_details - context["affected_by_advisories_v2_url"] = None - - # fixing_advisories = list(fixing_advisories_qs_ids[:101]) - # if len(fixing_advisories) > 100: - # fixing_advisories_url = reverse_lazy( - # "fixing_advisories_v2", kwargs={"purl": package.package_url} - # ) - # context["fixing_advisories_v2_url"] = fixing_advisories_url - # context["fixing_advisories_v2"] = [] - - # else: - context["fixing_advisories_v2"] = fixing_advisories_qs - context["fixing_advisories_v2_url"] = None - - return context - - def get_object(self, queryset=None): - if queryset is None: - queryset = self.get_queryset() - - purl = self.kwargs.get(self.slug_url_kwarg) - if purl: - queryset = queryset.for_purl(purl) - else: - cls = self.__class__.__name__ - raise AttributeError( - f"Package details view {cls} must be called with a purl, " f"but got: {purl!r}" - ) - - try: - package = queryset.get() - except queryset.model.DoesNotExist: - raise Http404(f"No Package found for purl: {purl}") - return package - - def get_fixed_package_details(package): rows = package.affected_in_impacts.values_list( "advisory__avid", diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 44cacd9b0..eb1bc006b 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -41,7 +41,6 @@ from vulnerabilities.views import PackageSearch from vulnerabilities.views import PackageSearchV2 from vulnerabilities.views import PackageV2Details -from vulnerabilities.views import PackageV3Details from vulnerabilities.views import PipelineRunDetailView from vulnerabilities.views import PipelineRunListView from vulnerabilities.views import PipelineScheduleListView From c341e6b43b43abdfa722d9727ec4ff90211a5e57 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 2 Apr 2026 11:55:43 +0530 Subject: [PATCH 64/65] Adjust precedence of importers Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 6 +++--- .../pipelines/v2_importers/elixir_security_importer.py | 2 +- vulnerabilities/pipelines/v2_importers/npm_importer.py | 2 +- .../pipelines/v2_importers/retiredotnet_importer.py | 2 +- vulnerabilities/pipelines/v2_importers/ruby_importer.py | 2 +- vulnerabilities/pipes/openssl.py | 4 +++- .../tests/pipelines/v2_importers/test_collect_fix_commit.py | 4 +++- vulnerabilities/tests/test_api.py | 6 +++--- 8 files changed, 16 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 45d8acf55..90e7b0287 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1139,9 +1139,9 @@ def get_affecting_vulnerabilities(self): next_fixed_package_vulns = list(fixed_by_pkg.affected_by) fixed_by_package_details["fixed_by_purl"] = fixed_by_purl - fixed_by_package_details["fixed_by_purl_vulnerabilities"] = ( - next_fixed_package_vulns - ) + fixed_by_package_details[ + "fixed_by_purl_vulnerabilities" + ] = next_fixed_package_vulns fixed_by_pkgs.append(fixed_by_package_details) vuln_details["fixed_by_package_details"] = fixed_by_pkgs diff --git a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py index 3b9f86d8e..2269d0fbc 100644 --- a/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py +++ b/vulnerabilities/pipelines/v2_importers/elixir_security_importer.py @@ -37,7 +37,7 @@ class ElixirSecurityImporterPipeline(VulnerableCodeBaseImporterPipelineV2): repo_url = "git+https://github.com/dependabot/elixir-security-advisories" run_once = True - precedence = 200 + precedence = 400 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/npm_importer.py b/vulnerabilities/pipelines/v2_importers/npm_importer.py index 32eec2051..9ec4c16dc 100644 --- a/vulnerabilities/pipelines/v2_importers/npm_importer.py +++ b/vulnerabilities/pipelines/v2_importers/npm_importer.py @@ -41,7 +41,7 @@ class NpmImporterPipeline(VulnerableCodeBaseImporterPipelineV2): license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md" repo_url = "git+https://github.com/nodejs/security-wg" - precedence = 200 + precedence = 500 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index cb87183e3..de9f131ee 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -30,7 +30,7 @@ class RetireDotnetImporterPipeline(VulnerableCodeBaseImporterPipelineV2): pipeline_id = "retiredotnet_importer_v2" run_once = True - precedence = 200 + precedence = 400 @classmethod def steps(cls): diff --git a/vulnerabilities/pipelines/v2_importers/ruby_importer.py b/vulnerabilities/pipelines/v2_importers/ruby_importer.py index fad09a1b5..210f73566 100644 --- a/vulnerabilities/pipelines/v2_importers/ruby_importer.py +++ b/vulnerabilities/pipelines/v2_importers/ruby_importer.py @@ -58,7 +58,7 @@ class RubyImporterPipeline(VulnerableCodeBaseImporterPipelineV2): SOFTWARE. """ - precedence = 200 + precedence = 500 @classmethod def steps(cls): diff --git a/vulnerabilities/pipes/openssl.py b/vulnerabilities/pipes/openssl.py index 1dffdedc1..b240f416c 100644 --- a/vulnerabilities/pipes/openssl.py +++ b/vulnerabilities/pipes/openssl.py @@ -89,7 +89,9 @@ def get_reference(reference_name, tag, reference_url): ref_type = ( AdvisoryReference.COMMIT if "commit" in name or tag == "patch" - else AdvisoryReference.ADVISORY if "advisory" in name else AdvisoryReference.OTHER + else AdvisoryReference.ADVISORY + if "advisory" in name + else AdvisoryReference.OTHER ) return ReferenceV2( diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py b/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py index 9a687a3b7..dac2c7781 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py @@ -52,7 +52,9 @@ def test_collect_fix_commits_groups_by_vuln(mock_repo, pipeline): side_effect=lambda c: ( ["CVE-2021-0001"] if "CVE" in c.message - else ["GHSA-dead-beef-baad"] if "GHSA" in c.message else [] + else ["GHSA-dead-beef-baad"] + if "GHSA" in c.message + else [] ) ) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 31f2b7774..9ed647099 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -75,9 +75,9 @@ def cleaned_response(response): reference["scores"] = sorted( reference["scores"], key=lambda x: (x["value"], x["scoring_system"]) ) - package_data["resolved_vulnerabilities"][index]["references"][index2]["scores"] = ( - reference["scores"] - ) + package_data["resolved_vulnerabilities"][index]["references"][index2][ + "scores" + ] = reference["scores"] cleaned_response.append(package_data) From 54b0fc9773a6f19017b0129493a5cd7353818e1a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 2 Apr 2026 12:01:50 +0530 Subject: [PATCH 65/65] Upgrade black Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 6 +++--- vulnerabilities/pipes/openssl.py | 4 +--- .../tests/pipelines/v2_importers/test_collect_fix_commit.py | 4 +--- vulnerabilities/tests/test_api.py | 6 +++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 90e7b0287..45d8acf55 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -1139,9 +1139,9 @@ def get_affecting_vulnerabilities(self): next_fixed_package_vulns = list(fixed_by_pkg.affected_by) fixed_by_package_details["fixed_by_purl"] = fixed_by_purl - fixed_by_package_details[ - "fixed_by_purl_vulnerabilities" - ] = next_fixed_package_vulns + fixed_by_package_details["fixed_by_purl_vulnerabilities"] = ( + next_fixed_package_vulns + ) fixed_by_pkgs.append(fixed_by_package_details) vuln_details["fixed_by_package_details"] = fixed_by_pkgs diff --git a/vulnerabilities/pipes/openssl.py b/vulnerabilities/pipes/openssl.py index b240f416c..1dffdedc1 100644 --- a/vulnerabilities/pipes/openssl.py +++ b/vulnerabilities/pipes/openssl.py @@ -89,9 +89,7 @@ def get_reference(reference_name, tag, reference_url): ref_type = ( AdvisoryReference.COMMIT if "commit" in name or tag == "patch" - else AdvisoryReference.ADVISORY - if "advisory" in name - else AdvisoryReference.OTHER + else AdvisoryReference.ADVISORY if "advisory" in name else AdvisoryReference.OTHER ) return ReferenceV2( diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py b/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py index dac2c7781..9a687a3b7 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_collect_fix_commit.py @@ -52,9 +52,7 @@ def test_collect_fix_commits_groups_by_vuln(mock_repo, pipeline): side_effect=lambda c: ( ["CVE-2021-0001"] if "CVE" in c.message - else ["GHSA-dead-beef-baad"] - if "GHSA" in c.message - else [] + else ["GHSA-dead-beef-baad"] if "GHSA" in c.message else [] ) ) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 9ed647099..31f2b7774 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -75,9 +75,9 @@ def cleaned_response(response): reference["scores"] = sorted( reference["scores"], key=lambda x: (x["value"], x["scoring_system"]) ) - package_data["resolved_vulnerabilities"][index]["references"][index2][ - "scores" - ] = reference["scores"] + package_data["resolved_vulnerabilities"][index]["references"][index2]["scores"] = ( + reference["scores"] + ) cleaned_response.append(package_data)