Improve d2d mapping npm map

scanpipe/pipelines/deploy_to_develop.py

@@ -109,6 +109,7 @@ def steps(cls):
             cls.map_javascript_post_purldb_match,
             cls.map_javascript_path,
             cls.map_javascript_colocation,
+            cls.map_javascript_source_map_sources,
             cls.map_thirdparty_npm_packages,
             cls.map_path,
             cls.flag_mapped_resources_archives_and_ignored_directories,
@@ -449,6 +450,11 @@ def map_javascript_colocation(self):
         """Map JavaScript files based on neighborhood file mapping."""
         d2d.map_javascript_colocation(project=self.project, logger=self.log)
 
+    @optional_step("JavaScript")
+    def map_javascript_source_map_sources(self):
+        """Map .map files by resolving listed sources against the from/ codebase."""
+        d2d.map_javascript_source_map_sources(project=self.project, logger=self.log)
+
     @optional_step("JavaScript")
     def map_thirdparty_npm_packages(self):
         """Map thirdparty package using package.json metadata."""

scanpipe/pipes/d2d.py

@@ -1297,6 +1297,79 @@ def _map_javascript_colocation_resource(
     )
 
 
+def _map_javascript_source_map_resource(to_map, from_resources, from_resources_index):
+    """Map a `.map` file by resolving its `sources` against the `from/` codebase."""
+    sources = js.get_map_sources(to_map)
+    if not sources:
+        return 0
+
+    matched_from_resources = []
+    for source_path in sources:
+        match = pathmap.find_paths(source_path, from_resources_index)
+        if not match:
+            to_map.update(status=flag.REQUIRES_REVIEW)
+            return 0
+
+        # Reject ambiguous matches where there are more candidate resources
+        # than the number of path segments actually matched.
+        if len(match.resource_ids) > match.matched_path_length:
+            to_map.update(status=flag.REQUIRES_REVIEW)
+            return 0
+
+        from_resource = from_resources.get(id=match.resource_ids[0])
+        matched_from_resources.append(from_resource)
+
+    # All sources resolved – create relations and mark the .map file as mapped.
+    for from_resource in matched_from_resources:
+        pipes.make_relation(
+            from_resource=from_resource,
+            to_resource=to_map,
+            map_type="js_source_map",
+            extra_data={"source_count": len(sources)},
+        )
+
+    to_map.update(status=flag.MAPPED)
+    return 1
+
+
+def map_javascript_source_map_sources(project, logger=None):
+    """Map .map files by resolving their sources against the from/ codebase."""
+    project_files = project.codebaseresources.files()
+
+    to_resources_dot_map = (
+        project_files.to_codebase()
+        .no_status()
+        .filter(extension=".map")
+        .exclude(name__startswith=".")
+        .exclude(path__contains="/node_modules/")
+    )
+
+    from_resources = project_files.from_codebase().exclude(path__contains="/test/")
+    resource_count = to_resources_dot_map.count()
+
+    if logger:
+        logger(
+            f"Mapping {resource_count:,d} .map source-map files by resolving their "
+            f"sources against the from/ codebase."
+        )
+
+    from_resources_index = pathmap.build_index(
+        from_resources.values_list("id", "path"), with_subpaths=True
+    )
+
+    resource_iterator = to_resources_dot_map.iterator(chunk_size=2000)
+    progress = LoopProgress(resource_count, logger)
+    map_count = 0
+
+    for to_map in progress.iter(resource_iterator):
+        map_count += _map_javascript_source_map_resource(
+            to_map, from_resources, from_resources_index
+        )
+
+    if logger:
+        logger(f"{map_count:,d} .map source-map files mapped")
+
+
 def flag_processed_archives(project):
     """
     Flag package archives as processed if they meet the following criteria:

scanpipe/pipes/d2d_config.py

@@ -164,7 +164,16 @@ class EcosystemConfig:
     ),
     "MacOS": EcosystemConfig(
         ecosystem_option="MacOS",
-        source_symbol_extensions=[".c", ".cpp", ".h", ".m", ".swift"],
+        source_symbol_extensions=[
+            ".c",
+            ".cpp",
+            ".h",
+            ".m",
+            ".swift",
+            ".go",
+            ".ts",
+            ".tsx",
+        ],
     ),
     "Windows": EcosystemConfig(
         ecosystem_option="Windows",
@@ -174,6 +183,18 @@ class EcosystemConfig:
         ecosystem_option="Python",
         source_symbol_extensions=[".pyx", ".pxd", ".py", ".pyi"],
         matchable_resource_extensions=[".py", ".pyi"],
+        deployed_resource_path_exclusions=[
+            "*.cmake",
+            "*.mo",
+            "*.toml",
+            "*.txt",
+            "*.db",
+            "*.conf",
+            "*.pth",
+            "*.dist-info/*",
+            "*.pc",
+            "*.la",
+        ],
     ),
 }
 

scanpipe/tests/data/d2d-javascript/from/error.ts

@@ -0,0 +1 @@
+export function error() { }

scanpipe/tests/data/d2d-javascript/from/queue.ts

@@ -0,0 +1 @@
+export function queue() { }

scanpipe/tests/pipes/test_d2d.py

@@ -2500,3 +2500,89 @@ def test_scanpipe_pipes_d2d_map_python_protobuf_files_no_py_files(self):
         d2d.map_python_protobuf_files(self.project1)
         relations = self.project1.codebaserelations.filter(map_type="protobuf_mapping")
         self.assertEqual(0, relations.count())
+
+    def test_scanpipe_pipes_d2d_map_javascript_source_map_all_found(self):
+        """
+        Test .map file with all sources present.
+
+        It creates relations and is flagged MAPPED.
+        """
+        to_dir = self.project1.codebase_path / "to/dist"
+        to_dir.mkdir(parents=True)
+        copy_input(
+            self.data / "d2d-javascript" / "to" / "bundle.js.map",
+            to_dir,
+        )
+
+        from_dir = self.project1.codebase_path / "from/src"
+        from_dir.mkdir(parents=True)
+        copy_inputs(
+            [
+                self.data / "d2d-javascript" / "from" / "queue.ts",
+                self.data / "d2d-javascript" / "from" / "error.ts",
+            ],
+            from_dir,
+        )
+
+        pipes.collect_and_create_codebase_resources(self.project1)
+
+        to_map = self.project1.codebaseresources.get(path="to/dist/bundle.js.map")
+        from_queue = self.project1.codebaseresources.get(path="from/src/queue.ts")
+        from_error = self.project1.codebaseresources.get(path="from/src/error.ts")
+
+        buffer = io.StringIO()
+        d2d.map_javascript_source_map_sources(self.project1, logger=buffer.write)
+
+        self.assertIn(
+            "Mapping 1 .map source-map files by resolving their sources",
+            buffer.getvalue(),
+        )
+        self.assertIn("1 .map source-map files mapped", buffer.getvalue())
+
+        to_map.refresh_from_db()
+        self.assertEqual(flag.MAPPED, to_map.status)
+
+        self.assertEqual(2, self.project1.codebaserelations.count())
+        relation_types = set(
+            self.project1.codebaserelations.values_list("map_type", flat=True)
+        )
+        self.assertEqual({"js_source_map"}, relation_types)
+
+        related_from_ids = set(
+            self.project1.codebaserelations.values_list("from_resource_id", flat=True)
+        )
+        self.assertIn(from_queue.id, related_from_ids)
+        self.assertIn(from_error.id, related_from_ids)
+
+    def test_scanpipe_pipes_d2d_map_javascript_source_map_partial(self):
+        """
+        Test .map file with missing source.
+
+        It is flagged REQUIRES_REVIEW with no relations.
+        """
+        to_dir = self.project1.codebase_path / "to/dist"
+        to_dir.mkdir(parents=True)
+        copy_input(
+            self.data / "d2d-javascript" / "to" / "partial.js.map",
+            to_dir,
+        )
+
+        from_dir = self.project1.codebase_path / "from/src"
+        from_dir.mkdir(parents=True)
+        copy_input(
+            self.data / "d2d-javascript" / "from" / "queue.ts",
+            from_dir,
+        )
+
+        pipes.collect_and_create_codebase_resources(self.project1)
+
+        to_map = self.project1.codebaseresources.get(path="to/dist/partial.js.map")
+
+        buffer = io.StringIO()
+        d2d.map_javascript_source_map_sources(self.project1, logger=buffer.write)
+
+        self.assertIn("0 .map source-map files mapped", buffer.getvalue())
+
+        to_map.refresh_from_db()
+        self.assertEqual(flag.REQUIRES_REVIEW, to_map.status)
+        self.assertEqual(0, self.project1.codebaserelations.count())