diff --git a/ruff_errors.txt b/ruff_errors.txt new file mode 100644 index 0000000000..83a1006ceb Binary files /dev/null and b/ruff_errors.txt differ diff --git a/scanpipe/pipelines/deploy_to_develop.py b/scanpipe/pipelines/deploy_to_develop.py index fa481fa95f..bc377d873c 100644 --- a/scanpipe/pipelines/deploy_to_develop.py +++ b/scanpipe/pipelines/deploy_to_develop.py @@ -109,6 +109,7 @@ def steps(cls): cls.map_javascript_post_purldb_match, cls.map_javascript_path, cls.map_javascript_colocation, + cls.map_javascript_source_map_sources, cls.map_thirdparty_npm_packages, cls.map_path, cls.flag_mapped_resources_archives_and_ignored_directories, @@ -449,6 +450,11 @@ def map_javascript_colocation(self): """Map JavaScript files based on neighborhood file mapping.""" d2d.map_javascript_colocation(project=self.project, logger=self.log) + @optional_step("JavaScript") + def map_javascript_source_map_sources(self): + """Map .map files by resolving listed sources against the from/ codebase.""" + d2d.map_javascript_source_map_sources(project=self.project, logger=self.log) + @optional_step("JavaScript") def map_thirdparty_npm_packages(self): """Map thirdparty package using package.json metadata.""" diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index c8e3b64294..563a4da007 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -1297,6 +1297,79 @@ def _map_javascript_colocation_resource( ) +def _map_javascript_source_map_resource(to_map, from_resources, from_resources_index): + """Map a `.map` file by resolving its `sources` against the `from/` codebase.""" + sources = js.get_map_sources(to_map) + if not sources: + return 0 + + matched_from_resources = [] + for source_path in sources: + match = pathmap.find_paths(source_path, from_resources_index) + if not match: + to_map.update(status=flag.REQUIRES_REVIEW) + return 0 + + # Reject ambiguous matches where there are more candidate resources + # than the number of path segments actually matched. + if len(match.resource_ids) > match.matched_path_length: + to_map.update(status=flag.REQUIRES_REVIEW) + return 0 + + from_resource = from_resources.get(id=match.resource_ids[0]) + matched_from_resources.append(from_resource) + + # All sources resolved – create relations and mark the .map file as mapped. + for from_resource in matched_from_resources: + pipes.make_relation( + from_resource=from_resource, + to_resource=to_map, + map_type="js_source_map", + extra_data={"source_count": len(sources)}, + ) + + to_map.update(status=flag.MAPPED) + return 1 + + +def map_javascript_source_map_sources(project, logger=None): + """Map .map files by resolving their sources against the from/ codebase.""" + project_files = project.codebaseresources.files() + + to_resources_dot_map = ( + project_files.to_codebase() + .no_status() + .filter(extension=".map") + .exclude(name__startswith=".") + .exclude(path__contains="/node_modules/") + ) + + from_resources = project_files.from_codebase().exclude(path__contains="/test/") + resource_count = to_resources_dot_map.count() + + if logger: + logger( + f"Mapping {resource_count:,d} .map source-map files by resolving their " + f"sources against the from/ codebase." + ) + + from_resources_index = pathmap.build_index( + from_resources.values_list("id", "path"), with_subpaths=True + ) + + resource_iterator = to_resources_dot_map.iterator(chunk_size=2000) + progress = LoopProgress(resource_count, logger) + map_count = 0 + + for to_map in progress.iter(resource_iterator): + map_count += _map_javascript_source_map_resource( + to_map, from_resources, from_resources_index + ) + + if logger: + logger(f"{map_count:,d} .map source-map files mapped") + + def flag_processed_archives(project): """ Flag package archives as processed if they meet the following criteria: diff --git a/scanpipe/pipes/d2d_config.py b/scanpipe/pipes/d2d_config.py index 917e004bf7..d2fc589b2e 100644 --- a/scanpipe/pipes/d2d_config.py +++ b/scanpipe/pipes/d2d_config.py @@ -164,7 +164,16 @@ class EcosystemConfig: ), "MacOS": EcosystemConfig( ecosystem_option="MacOS", - source_symbol_extensions=[".c", ".cpp", ".h", ".m", ".swift"], + source_symbol_extensions=[ + ".c", + ".cpp", + ".h", + ".m", + ".swift", + ".go", + ".ts", + ".tsx", + ], ), "Windows": EcosystemConfig( ecosystem_option="Windows", @@ -174,6 +183,18 @@ class EcosystemConfig: ecosystem_option="Python", source_symbol_extensions=[".pyx", ".pxd", ".py", ".pyi"], matchable_resource_extensions=[".py", ".pyi"], + deployed_resource_path_exclusions=[ + "*.cmake", + "*.mo", + "*.toml", + "*.txt", + "*.db", + "*.conf", + "*.pth", + "*.dist-info/*", + "*.pc", + "*.la", + ], ), } diff --git a/scanpipe/tests/data/d2d-javascript/from/error.ts b/scanpipe/tests/data/d2d-javascript/from/error.ts new file mode 100644 index 0000000000..aca086b448 --- /dev/null +++ b/scanpipe/tests/data/d2d-javascript/from/error.ts @@ -0,0 +1 @@ +export function error() { } diff --git a/scanpipe/tests/data/d2d-javascript/from/queue.ts b/scanpipe/tests/data/d2d-javascript/from/queue.ts new file mode 100644 index 0000000000..0e3cc96870 --- /dev/null +++ b/scanpipe/tests/data/d2d-javascript/from/queue.ts @@ -0,0 +1 @@ +export function queue() { } diff --git a/scanpipe/tests/data/d2d-javascript/to/bundle.js.map b/scanpipe/tests/data/d2d-javascript/to/bundle.js.map new file mode 100644 index 0000000000..9f82c98c94 --- /dev/null +++ b/scanpipe/tests/data/d2d-javascript/to/bundle.js.map @@ -0,0 +1,10 @@ +{ + "version": 3, + "sources": ["src/queue.ts", "src/error.ts"], + "sourcesContent": [ + "export function queue() {}", + "export function error() {}" + ], + "mappings": "AAAA", + "file": "bundle.js" +} diff --git a/scanpipe/tests/data/d2d-javascript/to/partial.js.map b/scanpipe/tests/data/d2d-javascript/to/partial.js.map new file mode 100644 index 0000000000..a5b9eaa3e6 --- /dev/null +++ b/scanpipe/tests/data/d2d-javascript/to/partial.js.map @@ -0,0 +1,10 @@ +{ + "version": 3, + "sources": ["src/queue.ts", "src/missing.ts"], + "sourcesContent": [ + "export function queue() {}", + null + ], + "mappings": "AAAA", + "file": "partial.js" +} diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index 73db8c0c9e..dd70ff318b 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -2500,3 +2500,89 @@ def test_scanpipe_pipes_d2d_map_python_protobuf_files_no_py_files(self): d2d.map_python_protobuf_files(self.project1) relations = self.project1.codebaserelations.filter(map_type="protobuf_mapping") self.assertEqual(0, relations.count()) + + def test_scanpipe_pipes_d2d_map_javascript_source_map_all_found(self): + """ + Test .map file with all sources present. + + It creates relations and is flagged MAPPED. + """ + to_dir = self.project1.codebase_path / "to/dist" + to_dir.mkdir(parents=True) + copy_input( + self.data / "d2d-javascript" / "to" / "bundle.js.map", + to_dir, + ) + + from_dir = self.project1.codebase_path / "from/src" + from_dir.mkdir(parents=True) + copy_inputs( + [ + self.data / "d2d-javascript" / "from" / "queue.ts", + self.data / "d2d-javascript" / "from" / "error.ts", + ], + from_dir, + ) + + pipes.collect_and_create_codebase_resources(self.project1) + + to_map = self.project1.codebaseresources.get(path="to/dist/bundle.js.map") + from_queue = self.project1.codebaseresources.get(path="from/src/queue.ts") + from_error = self.project1.codebaseresources.get(path="from/src/error.ts") + + buffer = io.StringIO() + d2d.map_javascript_source_map_sources(self.project1, logger=buffer.write) + + self.assertIn( + "Mapping 1 .map source-map files by resolving their sources", + buffer.getvalue(), + ) + self.assertIn("1 .map source-map files mapped", buffer.getvalue()) + + to_map.refresh_from_db() + self.assertEqual(flag.MAPPED, to_map.status) + + self.assertEqual(2, self.project1.codebaserelations.count()) + relation_types = set( + self.project1.codebaserelations.values_list("map_type", flat=True) + ) + self.assertEqual({"js_source_map"}, relation_types) + + related_from_ids = set( + self.project1.codebaserelations.values_list("from_resource_id", flat=True) + ) + self.assertIn(from_queue.id, related_from_ids) + self.assertIn(from_error.id, related_from_ids) + + def test_scanpipe_pipes_d2d_map_javascript_source_map_partial(self): + """ + Test .map file with missing source. + + It is flagged REQUIRES_REVIEW with no relations. + """ + to_dir = self.project1.codebase_path / "to/dist" + to_dir.mkdir(parents=True) + copy_input( + self.data / "d2d-javascript" / "to" / "partial.js.map", + to_dir, + ) + + from_dir = self.project1.codebase_path / "from/src" + from_dir.mkdir(parents=True) + copy_input( + self.data / "d2d-javascript" / "from" / "queue.ts", + from_dir, + ) + + pipes.collect_and_create_codebase_resources(self.project1) + + to_map = self.project1.codebaseresources.get(path="to/dist/partial.js.map") + + buffer = io.StringIO() + d2d.map_javascript_source_map_sources(self.project1, logger=buffer.write) + + self.assertIn("0 .map source-map files mapped", buffer.getvalue()) + + to_map.refresh_from_db() + self.assertEqual(flag.REQUIRES_REVIEW, to_map.status) + self.assertEqual(0, self.project1.codebaserelations.count())