diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index d3c48b6e25..44d8ad368a 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -25,6 +25,7 @@ from packagedcode import freebsd from packagedcode import godeps from packagedcode import golang +from packagedcode import gradle from packagedcode import haxe from packagedcode import maven from packagedcode import misc @@ -40,6 +41,7 @@ from packagedcode import swift from packagedcode import win_pe from packagedcode import windows +from packagedcode.gradle import GradleModuleHandler if on_linux: from packagedcode import msi @@ -56,6 +58,7 @@ bower.BowerJsonHandler, build_gradle.BuildGradleHandler, + gradle.GradleModuleHandler, build.AutotoolsConfigureHandler, build.BazelBuildHandler, diff --git a/src/packagedcode/gradle.py b/src/packagedcode/gradle.py new file mode 100644 index 0000000000..788a84c3ba --- /dev/null +++ b/src/packagedcode/gradle.py @@ -0,0 +1,118 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json + +from packageurl import PackageURL + +from packagedcode import models + + +class GradleModuleHandler(models.DatafileHandler): + datasource_id = 'gradle_module' + path_patterns = ('*.module',) + default_package_type = 'maven' + description = 'Gradle module metadata file' + + @classmethod + def is_datafile(cls, location, filetypes=tuple()): + if super().is_datafile(location, filetypes=filetypes): + return True + return str(location).endswith('.module') + + @classmethod + def parse(cls, location, package_only=False): + try: + with open(location, 'r') as f: + data = json.load(f) + except Exception: + return + + if not data or not isinstance(data, dict): + return + + component = data.get('component', {}) + if not component: + return + + namespace = component.get('group', '') + name = component.get('module', '') + version = component.get('version', '') + + created_by = data.get('createdBy', {}) + gradle_version = created_by.get('gradle', {}).get('version') + + variants = data.get('variants', []) + seen_deps = {} + files = [] + + for variant in variants: + variant_name = variant.get('name', '') + attributes = variant.get('attributes', {}) + usage = attributes.get('org.gradle.usage', variant_name) + + is_runtime = 'runtime' in usage.lower() + scope = 'runtime' if is_runtime else 'api' + + for dep in variant.get('dependencies', []): + dep_namespace = dep.get('group', '') + dep_name = dep.get('module', '') + dep_version_info = dep.get('version', {}) + + if isinstance(dep_version_info, dict): + dep_version = dep_version_info.get('requires') or \ + dep_version_info.get('prefers') or \ + dep_version_info.get('strictly', '') + else: + dep_version = str(dep_version_info) if dep_version_info else '' + + key = (dep_namespace, dep_name, dep_version) + if key not in seen_deps: + seen_deps[key] = models.DependentPackage( + purl=str(PackageURL( + type='maven', + namespace=dep_namespace, + name=dep_name, + version=dep_version + )), + extracted_requirement=dep_version, + scope=scope, + is_runtime=is_runtime, + is_optional=False, + ) + + # Extract files from the first variant that has them + if not files and variant.get('files'): + files = variant.get('files', []) + + extra_data = {} + if gradle_version: + extra_data['gradle_version'] = gradle_version + format_version = data.get('formatVersion') + if format_version: + extra_data['format_version'] = format_version + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + namespace=namespace, + name=name, + version=version, + dependencies=list(seen_deps.values()), + extra_data=extra_data, + ) + + # Store file checksums on the PackageData if available from the first variant files + for f in files: + for algo in ('sha1', 'sha256', 'sha512', 'md5'): + if f.get(algo): + package_data[algo] = f.get(algo) + break + + yield models.PackageData.from_data(package_data, package_only) diff --git a/tests/packagedcode/data/gradle/module/dependency-management-plugin-1.1.3.module b/tests/packagedcode/data/gradle/module/dependency-management-plugin-1.1.3.module new file mode 100644 index 0000000000..c7fa075980 --- /dev/null +++ b/tests/packagedcode/data/gradle/module/dependency-management-plugin-1.1.3.module @@ -0,0 +1,42 @@ +{ + "formatVersion": "1.1", + "component": { + "group": "io.spring.gradle", + "module": "dependency-management-plugin", + "version": "1.1.3", + "attributes": { + "org.gradle.status": "release" + } + }, + "createdBy": { + "gradle": { + "version": "7.6.1" + } + }, + "variants": [ + { + "name": "apiElements", + "attributes": { + "org.gradle.category": "library", + "org.gradle.usage": "java-api" + }, + "dependencies": [ + { + "group": "org.springframework.boot", + "module": "spring-boot", + "version": { + "requires": "3.1.0" + } + } + ], + "files": [ + { + "name": "dependency-management-plugin-1.1.3.jar", + "url": "dependency-management-plugin-1.1.3.jar", + "sha1": "3209385654a7e661d68de95a5ea8fc11d8ce015e", + "md5": "abc123" + } + ] + } + ] +} diff --git a/tests/packagedcode/data/gradle/module/material-1.9.0.module b/tests/packagedcode/data/gradle/module/material-1.9.0.module new file mode 100644 index 0000000000..2c36f37252 --- /dev/null +++ b/tests/packagedcode/data/gradle/module/material-1.9.0.module @@ -0,0 +1,74 @@ +{ + "formatVersion": "1.1", + "component": { + "group": "com.google.android.material", + "module": "material", + "version": "1.9.0", + "attributes": { + "org.gradle.status": "release" + } + }, + "createdBy": { + "gradle": { + "version": "7.3.3" + } + }, + "variants": [ + { + "name": "releaseApiElements", + "attributes": { + "org.gradle.category": "library", + "org.gradle.usage": "java-api" + }, + "dependencies": [ + { + "group": "androidx.annotation", + "module": "annotation", + "version": { + "requires": "1.2.0" + } + }, + { + "group": "androidx.appcompat", + "module": "appcompat", + "version": { + "requires": "1.5.0" + } + } + ], + "files": [ + { + "name": "material-1.9.0.aar", + "url": "material-1.9.0.aar", + "sha512": "7630aacb9e3073b2064397ed080b8d5bf7db06ba2022d6c927e05b7d53c5787d", + "sha256": "6cc2359979269e4d9eddce7d84682d2bb06a35a14edce806bf0da6e8d4d31806", + "sha1": "08f4a93a381be223a5bbaacd46eaab92381ab6a8", + "md5": "3287103cfb083fb998a35ef8a1983c58" + } + ] + }, + { + "name": "releaseRuntimeElements", + "attributes": { + "org.gradle.category": "library", + "org.gradle.usage": "java-runtime" + }, + "dependencies": [ + { + "group": "com.google.errorprone", + "module": "error_prone_annotations", + "version": { + "requires": "2.15.0" + } + }, + { + "group": "androidx.annotation", + "module": "annotation", + "version": { + "requires": "1.2.0" + } + } + ] + } + ] +} diff --git a/tests/packagedcode/data/gradle/module/no-deps.module b/tests/packagedcode/data/gradle/module/no-deps.module new file mode 100644 index 0000000000..f03dc7bdf9 --- /dev/null +++ b/tests/packagedcode/data/gradle/module/no-deps.module @@ -0,0 +1,16 @@ +{ + "formatVersion": "1.1", + "component": { + "group": "org.example", + "module": "standalone", + "version": "2.0.0" + }, + "variants": [ + { + "name": "apiElements", + "attributes": {}, + "dependencies": [], + "files": [] + } + ] +} diff --git a/tests/packagedcode/data/gradle/module/simple.module b/tests/packagedcode/data/gradle/module/simple.module new file mode 100644 index 0000000000..48fe3feac7 --- /dev/null +++ b/tests/packagedcode/data/gradle/module/simple.module @@ -0,0 +1,9 @@ +{ + "formatVersion": "1.1", + "component": { + "group": "com.example", + "module": "mylib", + "version": "1.0.0" + }, + "variants": [] +} diff --git a/tests/packagedcode/test_gradle_module.py b/tests/packagedcode/test_gradle_module.py new file mode 100644 index 0000000000..974c2d7d71 --- /dev/null +++ b/tests/packagedcode/test_gradle_module.py @@ -0,0 +1,112 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import io +import json +import os.path + +import pytest + +from commoncode import fileutils +from commoncode import text +from commoncode import testcase + +from packagedcode import gradle +from packagedcode import models +from scancode.cli_test_utils import check_json_scan +from scancode.cli_test_utils import run_scan_click +from scancode_config import REGEN_TEST_FIXTURES + + +def parse_module(location=None): + """ + Return a PackageData mapping from the Gradle module file at location. + """ + packages = list(gradle.GradleModuleHandler.parse(location=location)) + if not packages: + return {} + package = packages[0] + return package.to_dict() + + +class TestGradleModule(testcase.FileBasedTesting): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_gradle_module_is_datafile(self): + test_dir = self.get_test_loc('gradle/module') + material_module = os.path.join(test_dir, 'material-1.9.0.module') + simple_module = os.path.join(test_dir, 'simple.module') + + assert gradle.GradleModuleHandler.is_datafile(material_module) + assert gradle.GradleModuleHandler.is_datafile(simple_module) + + # Create a fake JSON file + fake_json = self.get_temp_file('json') + with open(fake_json, 'w') as f: + f.write('{"foo": "bar"}') + assert not gradle.GradleModuleHandler.is_datafile(fake_json) + + def test_parse_material_module(self): + test_loc = self.get_test_loc('gradle/module/material-1.9.0.module') + result = parse_module(test_loc) + + assert result.get('type') == 'maven' + assert result.get('namespace') == 'com.google.android.material' + assert result.get('name') == 'material' + assert result.get('version') == '1.9.0' + + deps = result.get('dependencies', []) + assert len(deps) == 3 + + purls = [d.get('purl') for d in deps] + assert 'pkg:maven/androidx.annotation/annotation@1.2.0' in purls + assert 'pkg:maven/androidx.appcompat/appcompat@1.5.0' in purls + assert 'pkg:maven/com.google.errorprone/error_prone_annotations@2.15.0' in purls + + extra = result.get('extra_data', {}) + assert extra.get('gradle_version') == '7.3.3' + assert extra.get('format_version') == '1.1' + + def test_parse_simple_module(self): + test_loc = self.get_test_loc('gradle/module/simple.module') + result = parse_module(test_loc) + + assert result.get('namespace') == 'com.example' + assert result.get('name') == 'mylib' + assert result.get('version') == '1.0.0' + assert result.get('dependencies') == [] + + def test_parse_no_deps_module(self): + test_loc = self.get_test_loc('gradle/module/no-deps.module') + # Does not crash + result = parse_module(test_loc) + + assert result.get('namespace') == 'org.example' + assert result.get('name') == 'standalone' + assert result.get('dependencies') == [] + + def test_parse_spring_module(self): + test_loc = self.get_test_loc('gradle/module/dependency-management-plugin-1.1.3.module') + result = parse_module(test_loc) + + assert result.get('namespace') == 'io.spring.gradle' + assert result.get('name') == 'dependency-management-plugin' + assert result.get('version') == '1.1.3' + + deps = result.get('dependencies', []) + assert len(deps) == 1 + assert deps[0].get('purl') == 'pkg:maven/org.springframework.boot/spring-boot@3.1.0' + + def test_dependencies_deduplicated(self): + test_loc = self.get_test_loc('gradle/module/material-1.9.0.module') + result = parse_module(test_loc) + + deps = result.get('dependencies', []) + purls = [d.get('purl') for d in deps] + assert len(purls) == len(set(purls))