Tejas morbagal/generate prr collection

.github/workflows/unittest-workflow.yaml

@@ -25,7 +25,8 @@ jobs:
 
       - name: Lint with ruff
         run: |
-            pip install ruff
+            # TODO upgrade to latest later
+            pip install ruff==0.9.10
             ruff check
 
       - name: Run unit tests

CHANGES.md

@@ -90,3 +90,7 @@
 - Generated project collection now includes required STAC extensions (`osc`, `themes`, `contacts`) and OSC-mandatory fields (`osc:type`, `osc:status`, `themes`, `contacts`) to pass OSC catalog validation.
 - Added optional `osc_project_url` field to the dataset config; used as the `via` link in the project collection. Falls back to `documentation_link` if omitted; defaults to the existing DeepESDL project collection when neither is provided.
 - `dataset_status` now defaults to `"ongoing"` when not specified in the dataset config.
+- Added a new CLI command `generate-prr-collection` that writes a Project Results Repository (PRR) STAC collection as local files, ready for submission to the ESA EarthCODE PRR endpoint. It reuses the dataset config and needs no GitHub credentials or S3 write access. See [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html).
+- The PRR command produces a self-contained `Collection → Item → Assets` tree: the Item carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) with `zarr-data` and `zarr-consolidated-metadata` assets, and the Collection declares the OSC, Scientific, Processing, Themes and CF extensions.
+- Added PRR-specific dataset config fields: `osc_initiative` (default `earthcode`), `osc_missions`, `osc_contract_number`, `osc_project_website`, `osc_project_description`, `thumbnail`, `thumbnail_media_type`, `sci_doi`, `sci_citation`, and `prr_output_dir`. Missing PRR-required fields produce a warning rather than a failure.
+- `generate-config` templates now document the PRR fields under a dedicated section.

deep_code/cli/generate_config.py

@@ -18,5 +18,5 @@
     help="Output directory for templates",
 )
 def generate_config(output_dir):
-    TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow_config.yaml")
-    TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset_config.yaml")
+    TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow.yaml")
+    TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset.yaml")

deep_code/cli/main.py

@@ -8,18 +8,19 @@
 
 from deep_code.cli.generate_config import generate_config
 from deep_code.cli.lint import lint_dataset
+from deep_code.cli.prr import generate_prr_collection_cmd
 from deep_code.cli.publish import publish
 
 
 @click.group()
 def main():
     """Deep Code CLI."""
-    pass
 
 
 main.add_command(publish)
 main.add_command(generate_config)
 main.add_command(lint_dataset)
+main.add_command(generate_prr_collection_cmd)
 
 if __name__ == "__main__":
     main()

deep_code/cli/prr.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 by Brockmann Consult GmbH
+# Permissions are hereby granted under the terms of the MIT License:
+# https://opensource.org/licenses/MIT.
+
+import click
+
+from deep_code.tools.prr import generate_prr_collection
+
+
+@click.command(name="generate-prr-collection")
+@click.argument("dataset_config", type=click.Path(exists=True))
+@click.option(
+    "--output-dir",
+    "-o",
+    type=click.Path(file_okay=False),
+    default=None,
+    help=(
+        "Directory to write the PRR collection tree into "
+        "(default: 'prr_output_dir' from the config, else 'prr/<collection_id>')."
+    ),
+)
+def generate_prr_collection_cmd(dataset_config, output_dir):
+    """Generate a PRR-style STAC Collection as local files.
+
+    Reads a dataset config (the same YAML used by 'publish') and writes a
+    self-contained Collection -> Item -> Assets tree for submission to the
+    ESA EarthCODE Project Results Repository.
+
+    Example:
+      deep-code generate-prr-collection dataset-config.yaml -o ./prr
+    """
+    out_dir = generate_prr_collection(dataset_config, output_dir=output_dir)
+    click.echo(f"PRR collection written to: {out_dir}")

deep_code/constants.py

@@ -41,3 +41,10 @@
     "https://stac-extensions.github.io/application/v0.1.0/schema.json"
 )
 ZARR_MEDIA_TYPE = "application/vnd+zarr"
+DATACUBE_SCHEMA_URI = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json"
+PROCESSING_SCHEMA_URI = (
+    "https://stac-extensions.github.io/processing/v1.2.0/schema.json"
+)
+SCIENTIFIC_SCHEMA_URI = (
+    "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
+)

deep_code/tests/tools/test_publish.py

@@ -24,7 +24,7 @@ def setUp(self, mock_github_publisher, mock_fsspec_open):
         # Mock dataset and workflow config files
         self.dataset_config = {
             "collection_id": "test-collection",
-            "dataset_id": "test-dataset",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
         }
         self.workflow_config = {
             "properties": {"title": "Test Workflow"},
@@ -94,7 +94,7 @@ def test_read_config_files(self):
         # Mock dataset and workflow config files
         dataset_config = {
             "collection_id": "test-collection",
-            "dataset_id": "test-dataset",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
         }
         workflow_config = {
             "properties": {"title": "Test Workflow"},
@@ -138,7 +138,7 @@ def test_publish_mode_routing(self, mock_wf, mock_ds, mock_s3):
         self.publisher.dataset_config = {
             "stac_catalog_s3_root": "s3://bucket/stac/",
             "collection_id": "test-collection",
-            "dataset_id": "test-dataset",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
         }
         self.publisher.gh_publisher.publish_files.return_value = "PR_URL"
 
@@ -198,7 +198,6 @@ def test_publish_builds_pr_params(self, mock_wf, mock_ds, mock_s3):
         assert "dataset: col" in kwargs["pr_title"]
         assert "workflow/experiment: wf" in kwargs["pr_title"]
 
-
     # ------------------------------------------------------------------
     # S3 credential resolution
     # ------------------------------------------------------------------
@@ -262,9 +261,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open):
             "s3://bucket/catalog.json": {"type": "Catalog", "id": "test"},
             "s3://bucket/col/item.json": {"type": "Feature", "id": "item"},
         }
-        self.publisher._write_stac_catalog_to_s3(
-            file_dict, {"key": "k", "secret": "s"}
-        )
+        self.publisher._write_stac_catalog_to_s3(file_dict, {"key": "k", "secret": "s"})
 
         self.assertEqual(mock_fsspec_open.call_count, 2)
         mock_fsspec_open.assert_any_call(
@@ -283,9 +280,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open):
     def test_publish_writes_zarr_stac_to_s3_when_configured(
         self, mock_publish_ds, mock_fsspec_open
     ):
-        self.publisher.dataset_config["stac_catalog_s3_root"] = (
-            "s3://test-bucket/stac/"
-        )
+        self.publisher.dataset_config["stac_catalog_s3_root"] = "s3://test-bucket/stac/"
 
         mock_ctx = MagicMock()
         mock_ctx.__enter__ = MagicMock(return_value=MagicMock())
@@ -295,7 +290,9 @@ def test_publish_writes_zarr_stac_to_s3_when_configured(
         mock_generator = MagicMock()
         mock_generator.build_zarr_stac_catalog_file_dict.return_value = {
             "s3://test-bucket/stac/catalog.json": {"type": "Catalog"},
-            "s3://test-bucket/stac/test-collection/item.json": {"type": "Feature"},
+            "s3://test-bucket/stac/test-collection/items/test-collection.json": {
+                "type": "Feature"
+            },
         }
         # Simulate what publish_dataset() normally does: store the generator
         self.publisher._last_generator = mock_generator
@@ -330,8 +327,10 @@ def test_publish_dataset_creates_project_collection_when_missing(
         MockGenerator.return_value = mock_gen
 
         self.publisher.dataset_config = {
-            "dataset_id": "test-dataset",
             "collection_id": "test-collection",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
+            "osc_project": "test-project",
+            "osc_project_url": "https://example.com/projects/test-project",
             "license_type": "CC-BY-4.0",
             "stac_catalog_s3_root": "s3://bucket/stac/test-collection/",
         }
@@ -340,9 +339,14 @@ def test_publish_dataset_creates_project_collection_when_missing(
         # Project collection is missing; all other file_exists calls return True
         self.publisher.gh_publisher.github_automation.file_exists.return_value = False
 
-        with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \
-                patch.object(self.publisher, "_update_variable_catalogs"):
-            file_dict = self.publisher.publish_dataset(write_to_file=False)
+        with patch("deep_code.tools.publish.open_dataset", return_value=object()):
+            with (
+                patch.object(
+                    self.publisher, "_update_and_add_to_file_dict"
+                ) as mock_update,
+                patch.object(self.publisher, "_update_variable_catalogs"),
+            ):
+                file_dict = self.publisher.publish_dataset(write_to_file=False)
 
         mock_gen.build_project_collection.assert_called_once()
         self.assertIn("projects/test-project/collection.json", file_dict)
@@ -365,8 +369,10 @@ def test_publish_dataset_updates_project_collection_when_exists(
         MockGenerator.return_value = mock_gen
 
         self.publisher.dataset_config = {
-            "dataset_id": "test-dataset",
             "collection_id": "test-collection",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
+            "osc_project": "test-project",
+            "osc_project_url": "https://example.com/projects/test-project",
             "license_type": "CC-BY-4.0",
             "stac_catalog_s3_root": "s3://bucket/stac/test-collection/",
         }
@@ -375,9 +381,14 @@ def test_publish_dataset_updates_project_collection_when_exists(
         # Project collection already exists
         self.publisher.gh_publisher.github_automation.file_exists.return_value = True
 
-        with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \
-                patch.object(self.publisher, "_update_variable_catalogs"):
-            self.publisher.publish_dataset(write_to_file=False)
+        with patch("deep_code.tools.publish.open_dataset", return_value=object()):
+            with (
+                patch.object(
+                    self.publisher, "_update_and_add_to_file_dict"
+                ) as mock_update,
+                patch.object(self.publisher, "_update_variable_catalogs"),
+            ):
+                self.publisher.publish_dataset(write_to_file=False)
 
         mock_gen.build_project_collection.assert_not_called()
 
@@ -388,7 +399,9 @@ def test_publish_dataset_updates_project_collection_when_exists(
     def test_publish_dataset_raises_when_stac_root_missing(self):
         self.publisher.dataset_config = {
             "collection_id": "test-collection",
-            "dataset_id": "test-dataset",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
+            "osc_project": "test-project",
+            "osc_project_url": "https://example.com/projects/test-project",
             "license_type": "CC-BY-4.0",
         }
         with pytest.raises(ValueError, match="stac_catalog_s3_root"):
@@ -400,14 +413,24 @@ def test_publish_dataset_raises_when_no_dataset_config(self):
             self.publisher.publish_dataset(write_to_file=False)
 
     def test_publish_dataset_raises_when_ids_missing(self):
-        self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""}
-        with pytest.raises(ValueError, match="Dataset ID or Collection ID missing"):
+        self.publisher.dataset_config = {
+            "collection_id": "",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
+            "osc_project": "test-project",
+            "osc_project_url": "https://example.com/projects/test-project",
+            "license_type": "CC-BY-4.0",
+            "stac_catalog_s3_root": "s3://bucket/stac/test-collection/",
+        }
+        with pytest.raises(ValueError, match="collection_id missing"):
             self.publisher.publish_dataset(write_to_file=False)
 
     def test_publish_dataset_raises_when_license_missing(self):
         self.publisher.dataset_config = {
             "collection_id": "test-collection",
-            "dataset_id": "test-dataset",
+            "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}],
+            "osc_project": "test-project",
+            "osc_project_url": "https://example.com/projects/test-project",
+            "stac_catalog_s3_root": "s3://bucket/stac/test-collection/",
         }
         with pytest.raises(ValueError, match="license_type is required"):
             self.publisher.publish_dataset(write_to_file=False)
@@ -431,14 +454,18 @@ def test_update_and_add_to_file_dict(self):
         file_dict = {}
         self.publisher.gh_publisher.github_automation.local_clone_dir = "/tmp"
         update_method = MagicMock(return_value={"key": "value"})
-        self.publisher._update_and_add_to_file_dict(file_dict, "some/catalog.json", update_method)
+        self.publisher._update_and_add_to_file_dict(
+            file_dict, "some/catalog.json", update_method
+        )
         update_method.assert_called_once()
         assert any("some/catalog.json" in str(k) for k in file_dict)
 
     def test_update_variable_catalogs_creates_new_when_missing(self):
         mock_gen = MagicMock()
         mock_gen.variables_metadata = {"var1": {"variable_id": "var1"}}
-        mock_gen.build_variable_catalog.return_value.to_dict.return_value = {"id": "var1"}
+        mock_gen.build_variable_catalog.return_value.to_dict.return_value = {
+            "id": "var1"
+        }
         self.publisher.gh_publisher.github_automation.file_exists.return_value = False
 
         file_dict = {}
@@ -486,7 +513,7 @@ def _setup_workflow_mocks(self):
     @patch("deep_code.tools.publish.LinksBuilder")
     @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator")
     def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF):
-        mock_rg, mock_props, mock_wf_record, _ = self._setup_workflow_mocks()
+        mock_rg, _mock_props, mock_wf_record, _ = self._setup_workflow_mocks()
         MockRG.return_value = mock_rg
         MockWF.return_value = mock_wf_record
 
@@ -507,8 +534,12 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF
     @patch("deep_code.tools.publish.WorkflowAsOgcRecord")
     @patch("deep_code.tools.publish.LinksBuilder")
     @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator")
-    def test_generate_workflow_records_mode_all(self, MockRG, MockLinks, MockWF, MockExp):
-        mock_rg, mock_props, mock_wf_record, mock_exp_record = self._setup_workflow_mocks()
+    def test_generate_workflow_records_mode_all(
+        self, MockRG, MockLinks, MockWF, MockExp
+    ):
+        mock_rg, _mock_props, mock_wf_record, mock_exp_record = (
+            self._setup_workflow_mocks()
+        )
         MockRG.return_value = mock_rg
         MockWF.return_value = mock_wf_record
         MockExp.return_value = mock_exp_record
@@ -583,8 +614,8 @@ class TestParseGithubNotebookUrl:
         ],
     )
     def test_valid_urls(self, url, repo_url, repo_name, branch, file_path):
-        got_repo_url, got_repo_name, got_branch, got_file_path = LinksBuilder._parse_github_notebook_url(
-            url
+        got_repo_url, got_repo_name, got_branch, got_file_path = (
+            LinksBuilder._parse_github_notebook_url(url)
         )
         assert got_repo_url == repo_url
         assert got_repo_name == repo_name