diff --git a/.github/workflows/unittest-workflow.yaml b/.github/workflows/unittest-workflow.yaml index d83721a..1e143cd 100644 --- a/.github/workflows/unittest-workflow.yaml +++ b/.github/workflows/unittest-workflow.yaml @@ -25,7 +25,8 @@ jobs: - name: Lint with ruff run: | - pip install ruff + # TODO upgrade to latest later + pip install ruff==0.9.10 ruff check - name: Run unit tests diff --git a/CHANGES.md b/CHANGES.md index 2c2b89c..5c594e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -90,3 +90,7 @@ - Generated project collection now includes required STAC extensions (`osc`, `themes`, `contacts`) and OSC-mandatory fields (`osc:type`, `osc:status`, `themes`, `contacts`) to pass OSC catalog validation. - Added optional `osc_project_url` field to the dataset config; used as the `via` link in the project collection. Falls back to `documentation_link` if omitted; defaults to the existing DeepESDL project collection when neither is provided. - `dataset_status` now defaults to `"ongoing"` when not specified in the dataset config. +- Added a new CLI command `generate-prr-collection` that writes a Project Results Repository (PRR) STAC collection as local files, ready for submission to the ESA EarthCODE PRR endpoint. It reuses the dataset config and needs no GitHub credentials or S3 write access. See [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html). +- The PRR command produces a self-contained `Collection → Item → Assets` tree: the Item carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) with `zarr-data` and `zarr-consolidated-metadata` assets, and the Collection declares the OSC, Scientific, Processing, Themes and CF extensions. +- Added PRR-specific dataset config fields: `osc_initiative` (default `earthcode`), `osc_missions`, `osc_contract_number`, `osc_project_website`, `osc_project_description`, `thumbnail`, `thumbnail_media_type`, `sci_doi`, `sci_citation`, and `prr_output_dir`. Missing PRR-required fields produce a warning rather than a failure. +- `generate-config` templates now document the PRR fields under a dedicated section. diff --git a/deep_code/cli/generate_config.py b/deep_code/cli/generate_config.py index ce1c286..ba63d0d 100644 --- a/deep_code/cli/generate_config.py +++ b/deep_code/cli/generate_config.py @@ -18,5 +18,5 @@ help="Output directory for templates", ) def generate_config(output_dir): - TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow_config.yaml") - TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset_config.yaml") + TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow.yaml") + TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset.yaml") diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index e267305..dfc8295 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -8,18 +8,19 @@ from deep_code.cli.generate_config import generate_config from deep_code.cli.lint import lint_dataset +from deep_code.cli.prr import generate_prr_collection_cmd from deep_code.cli.publish import publish @click.group() def main(): """Deep Code CLI.""" - pass main.add_command(publish) main.add_command(generate_config) main.add_command(lint_dataset) +main.add_command(generate_prr_collection_cmd) if __name__ == "__main__": main() diff --git a/deep_code/cli/prr.py b/deep_code/cli/prr.py new file mode 100644 index 0000000..25e0ac5 --- /dev/null +++ b/deep_code/cli/prr.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 by Brockmann Consult GmbH +# Permissions are hereby granted under the terms of the MIT License: +# https://opensource.org/licenses/MIT. + +import click + +from deep_code.tools.prr import generate_prr_collection + + +@click.command(name="generate-prr-collection") +@click.argument("dataset_config", type=click.Path(exists=True)) +@click.option( + "--output-dir", + "-o", + type=click.Path(file_okay=False), + default=None, + help=( + "Directory to write the PRR collection tree into " + "(default: 'prr_output_dir' from the config, else 'prr/')." + ), +) +def generate_prr_collection_cmd(dataset_config, output_dir): + """Generate a PRR-style STAC Collection as local files. + + Reads a dataset config (the same YAML used by 'publish') and writes a + self-contained Collection -> Item -> Assets tree for submission to the + ESA EarthCODE Project Results Repository. + + Example: + deep-code generate-prr-collection dataset-config.yaml -o ./prr + """ + out_dir = generate_prr_collection(dataset_config, output_dir=output_dir) + click.echo(f"PRR collection written to: {out_dir}") diff --git a/deep_code/constants.py b/deep_code/constants.py index fa9e4b8..e1fee57 100644 --- a/deep_code/constants.py +++ b/deep_code/constants.py @@ -41,3 +41,10 @@ "https://stac-extensions.github.io/application/v0.1.0/schema.json" ) ZARR_MEDIA_TYPE = "application/vnd+zarr" +DATACUBE_SCHEMA_URI = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json" +PROCESSING_SCHEMA_URI = ( + "https://stac-extensions.github.io/processing/v1.2.0/schema.json" +) +SCIENTIFIC_SCHEMA_URI = ( + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" +) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 6ccc8eb..4048b09 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -24,7 +24,7 @@ def setUp(self, mock_github_publisher, mock_fsspec_open): # Mock dataset and workflow config files self.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.workflow_config = { "properties": {"title": "Test Workflow"}, @@ -94,7 +94,7 @@ def test_read_config_files(self): # Mock dataset and workflow config files dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } workflow_config = { "properties": {"title": "Test Workflow"}, @@ -138,7 +138,7 @@ def test_publish_mode_routing(self, mock_wf, mock_ds, mock_s3): self.publisher.dataset_config = { "stac_catalog_s3_root": "s3://bucket/stac/", "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.publisher.gh_publisher.publish_files.return_value = "PR_URL" @@ -198,7 +198,6 @@ def test_publish_builds_pr_params(self, mock_wf, mock_ds, mock_s3): assert "dataset: col" in kwargs["pr_title"] assert "workflow/experiment: wf" in kwargs["pr_title"] - # ------------------------------------------------------------------ # S3 credential resolution # ------------------------------------------------------------------ @@ -262,9 +261,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): "s3://bucket/catalog.json": {"type": "Catalog", "id": "test"}, "s3://bucket/col/item.json": {"type": "Feature", "id": "item"}, } - self.publisher._write_stac_catalog_to_s3( - file_dict, {"key": "k", "secret": "s"} - ) + self.publisher._write_stac_catalog_to_s3(file_dict, {"key": "k", "secret": "s"}) self.assertEqual(mock_fsspec_open.call_count, 2) mock_fsspec_open.assert_any_call( @@ -283,9 +280,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): def test_publish_writes_zarr_stac_to_s3_when_configured( self, mock_publish_ds, mock_fsspec_open ): - self.publisher.dataset_config["stac_catalog_s3_root"] = ( - "s3://test-bucket/stac/" - ) + self.publisher.dataset_config["stac_catalog_s3_root"] = "s3://test-bucket/stac/" mock_ctx = MagicMock() mock_ctx.__enter__ = MagicMock(return_value=MagicMock()) @@ -295,7 +290,9 @@ def test_publish_writes_zarr_stac_to_s3_when_configured( mock_generator = MagicMock() mock_generator.build_zarr_stac_catalog_file_dict.return_value = { "s3://test-bucket/stac/catalog.json": {"type": "Catalog"}, - "s3://test-bucket/stac/test-collection/item.json": {"type": "Feature"}, + "s3://test-bucket/stac/test-collection/items/test-collection.json": { + "type": "Feature" + }, } # Simulate what publish_dataset() normally does: store the generator self.publisher._last_generator = mock_generator @@ -330,8 +327,10 @@ def test_publish_dataset_creates_project_collection_when_missing( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -340,9 +339,14 @@ def test_publish_dataset_creates_project_collection_when_missing( # Project collection is missing; all other file_exists calls return True self.publisher.gh_publisher.github_automation.file_exists.return_value = False - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - file_dict = self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + file_dict = self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_called_once() self.assertIn("projects/test-project/collection.json", file_dict) @@ -365,8 +369,10 @@ def test_publish_dataset_updates_project_collection_when_exists( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -375,9 +381,14 @@ def test_publish_dataset_updates_project_collection_when_exists( # Project collection already exists self.publisher.gh_publisher.github_automation.file_exists.return_value = True - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_not_called() @@ -388,7 +399,9 @@ def test_publish_dataset_updates_project_collection_when_exists( def test_publish_dataset_raises_when_stac_root_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", } with pytest.raises(ValueError, match="stac_catalog_s3_root"): @@ -400,14 +413,24 @@ def test_publish_dataset_raises_when_no_dataset_config(self): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_ids_missing(self): - self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""} - with pytest.raises(ValueError, match="Dataset ID or Collection ID missing"): + self.publisher.dataset_config = { + "collection_id": "", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "license_type": "CC-BY-4.0", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", + } + with pytest.raises(ValueError, match="collection_id missing"): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_license_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } with pytest.raises(ValueError, match="license_type is required"): self.publisher.publish_dataset(write_to_file=False) @@ -431,14 +454,18 @@ def test_update_and_add_to_file_dict(self): file_dict = {} self.publisher.gh_publisher.github_automation.local_clone_dir = "/tmp" update_method = MagicMock(return_value={"key": "value"}) - self.publisher._update_and_add_to_file_dict(file_dict, "some/catalog.json", update_method) + self.publisher._update_and_add_to_file_dict( + file_dict, "some/catalog.json", update_method + ) update_method.assert_called_once() assert any("some/catalog.json" in str(k) for k in file_dict) def test_update_variable_catalogs_creates_new_when_missing(self): mock_gen = MagicMock() mock_gen.variables_metadata = {"var1": {"variable_id": "var1"}} - mock_gen.build_variable_catalog.return_value.to_dict.return_value = {"id": "var1"} + mock_gen.build_variable_catalog.return_value.to_dict.return_value = { + "id": "var1" + } self.publisher.gh_publisher.github_automation.file_exists.return_value = False file_dict = {} @@ -486,7 +513,7 @@ def _setup_workflow_mocks(self): @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF): - mock_rg, mock_props, mock_wf_record, _ = self._setup_workflow_mocks() + mock_rg, _mock_props, mock_wf_record, _ = self._setup_workflow_mocks() MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record @@ -507,8 +534,12 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF @patch("deep_code.tools.publish.WorkflowAsOgcRecord") @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") - def test_generate_workflow_records_mode_all(self, MockRG, MockLinks, MockWF, MockExp): - mock_rg, mock_props, mock_wf_record, mock_exp_record = self._setup_workflow_mocks() + def test_generate_workflow_records_mode_all( + self, MockRG, MockLinks, MockWF, MockExp + ): + mock_rg, _mock_props, mock_wf_record, mock_exp_record = ( + self._setup_workflow_mocks() + ) MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record MockExp.return_value = mock_exp_record @@ -583,8 +614,8 @@ class TestParseGithubNotebookUrl: ], ) def test_valid_urls(self, url, repo_url, repo_name, branch, file_path): - got_repo_url, got_repo_name, got_branch, got_file_path = LinksBuilder._parse_github_notebook_url( - url + got_repo_url, got_repo_name, got_branch, got_file_path = ( + LinksBuilder._parse_github_notebook_url(url) ) assert got_repo_url == repo_url assert got_repo_name == repo_name diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 0b806cd..9b0a2d7 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -3,27 +3,36 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. +import json +import os +import tempfile import unittest from datetime import datetime -from unittest.mock import MagicMock, patch +from unittest.mock import patch import numpy as np -from pystac import Catalog, Item +from pystac import Catalog, Collection, Item from xarray import DataArray, Dataset from deep_code.constants import ( + DATACUBE_SCHEMA_URI, DEEPESDL_COLLECTION_SELF_HREF, + OSC_SCHEMA_URI, OSC_THEME_SCHEME, + PROCESSING_SCHEMA_URI, PRODUCT_BASE_CATALOG_SELF_HREF, VARIABLE_BASE_CATALOG_SELF_HREF, ZARR_MEDIA_TYPE, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator, Theme +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + Theme, +) class TestOSCProductSTACGenerator(unittest.TestCase): - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_data_store): + def setUp(self): """Set up a mock dataset and generator.""" self.mock_dataset = Dataset( coords={ @@ -59,18 +68,27 @@ def setUp(self, mock_data_store): ), }, ) - mock_store = MagicMock() - mock_store.open_data.return_value = self.mock_dataset - mock_data_store.return_value = self.mock_dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.mock_dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.generator = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", - access_link="s3://mock-bucket/mock-dataset", + access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", + osc_project="deep-earth-system-data-lab", osc_status="ongoing", osc_region="Global", osc_themes=["climate", "environment"], @@ -78,18 +96,19 @@ def setUp(self, mock_data_store): def test_open_dataset(self): """Test if the dataset is opened correctly.""" - self.assertIsInstance(self.generator.dataset, Dataset) + dataset = self.mock_dataset + self.assertIsInstance(dataset, Dataset) for coord in ("lon", "lat", "time"): - self.assertIn(coord, self.generator.dataset.coords) + self.assertIn(coord, dataset.coords) def test_get_spatial_extent(self): """Test spatial extent extraction.""" - extent = self.generator._get_spatial_extent() + extent = self.generator._get_spatial_extent(self.mock_dataset) self.assertEqual(extent.bboxes[0], [-180.0, -90.0, 180.0, 90.0]) def test_get_temporal_extent(self): """Test temporal extent extraction.""" - extent = self.generator._get_temporal_extent() + extent = self.generator._get_temporal_extent(self.mock_dataset) # TemporalExtent.intervals is a list of [start, end] interval = extent.intervals[0] self.assertEqual(interval[0], datetime(2023, 1, 1, 0, 0)) @@ -97,12 +116,12 @@ def test_get_temporal_extent(self): def test_get_variables(self): """Test variable ID extraction.""" - vars_ = self.generator.get_variable_ids() + vars_ = self.generator.get_variable_ids(self.mock_dataset) self.assertCountEqual(vars_, ["var1", "var2"]) def test_get_general_metadata(self): """Test general metadata extraction.""" - meta = self.generator._get_general_metadata() + meta = self.generator._get_general_metadata(self.mock_dataset) self.assertEqual(meta.get("description"), "Mock dataset for testing.") def test_extract_metadata_for_variable(self): @@ -115,7 +134,7 @@ def test_extract_metadata_for_variable(self): def test_get_variables_metadata(self): """Test metadata dict for all variables.""" - meta_dict = self.generator.get_variables_metadata() + meta_dict = self.generator.get_variables_metadata(self.mock_dataset) self.assertIn("var1", meta_dict) self.assertIn("var2", meta_dict) self.assertIsInstance(meta_dict["var1"], dict) @@ -128,11 +147,10 @@ def test_build_theme(self): ids = [tc.id for tc in theme_obj.concepts] self.assertListEqual(ids, ["a", "b"]) - @patch.object(OscDatasetStacGenerator, "_add_gcmd_link_to_var_catalog") - @patch.object(OscDatasetStacGenerator, "add_themes_as_related_links_var_catalog") - def test_build_variable_catalog(self, mock_add_themes, mock_add_gcmd): + def test_build_variable_catalog(self): """Test building of variable-level STAC catalog.""" - var_meta = self.generator.variables_metadata["var1"] + variables_metadata = self.generator.get_variables_metadata(self.mock_dataset) + var_meta = variables_metadata["var1"] catalog = self.generator.build_variable_catalog(var_meta) self.assertIsInstance(catalog, Catalog) self.assertEqual(catalog.id, "var1") @@ -156,12 +174,10 @@ def test_update_product_base_catalog(self): } ], } - import tempfile import json as _json + import tempfile - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -196,13 +212,11 @@ def test_update_variable_base_catalog(self): } ], } - import tempfile import json as _json import os + import tempfile - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -213,9 +227,7 @@ def test_update_variable_base_catalog(self): os.unlink(tmp_path) self.assertIsInstance(result, dict) - child_hrefs = [ - lnk["href"] for lnk in result["links"] if lnk["rel"] == "child" - ] + child_hrefs = [lnk["href"] for lnk in result["links"] if lnk["rel"] == "child"] self.assertEqual(len(child_hrefs), len(vars_)) # self link must remain in place self.assertEqual(result["links"][0]["rel"], "self") @@ -233,8 +245,13 @@ def test_osc_project_custom(self, mock_open_ds): """A custom osc_project is stored on the generator.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -281,8 +298,13 @@ def test_build_project_collection_custom_project(self, mock_open_ds): """build_project_collection reflects a custom osc_project.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -383,13 +405,11 @@ def test_update_deepesdl_collection(self): } ], } - import tempfile import json as _json import os + import tempfile - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -413,7 +433,9 @@ def test_update_deepesdl_collection(self): def test_build_zarr_stac_item_structure(self): """Item has correct geometry, bbox, datetime range, assets, and links.""" s3_root = "s3://test-bucket/stac/my-collection/" - item = self.generator.build_zarr_stac_item(s3_root) + item = self.generator.build_zarr_stac_item( + self.generator.items_config[0], s3_root + ) self.assertIsInstance(item, Item) self.assertEqual(item.id, "mock-collection-id") @@ -437,13 +459,14 @@ def test_build_zarr_stac_item_structure(self): self.assertIn("zarr-consolidated-metadata", item.assets) zarr_asset = item.assets["zarr-data"] - self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset") + self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset-id") self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE) self.assertIn("data", zarr_asset.roles) meta_asset = item.assets["zarr-consolidated-metadata"] self.assertEqual( - meta_asset.href, "s3://mock-bucket/mock-dataset/.zmetadata" + meta_asset.href, + "s3://mock-bucket/mock-dataset-id/.zmetadata", ) self.assertIn("metadata", meta_asset.roles) @@ -473,8 +496,12 @@ def test_build_zarr_stac_item_structure(self): def test_build_zarr_stac_item_trailing_slash_normalised(self): """Trailing slash on s3_root should not produce double slashes.""" - item_with = self.generator.build_zarr_stac_item("s3://bucket/stac/") - item_without = self.generator.build_zarr_stac_item("s3://bucket/stac") + item_with = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac/" + ) + item_without = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac" + ) self.assertEqual(item_with.self_href, item_without.self_href) def test_build_zarr_stac_catalog_file_dict_keys(self): @@ -484,7 +511,8 @@ def test_build_zarr_stac_catalog_file_dict_keys(self): catalog_path = "s3://test-bucket/stac/my-collection/catalog.json" item_path = ( - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ) self.assertIn(catalog_path, file_dict) self.assertIn(item_path, file_dict) @@ -500,7 +528,8 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertEqual(catalog_dict["id"], "mock-collection-id-stac-catalog") item_dict = file_dict[ - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ] self.assertEqual(item_dict["type"], "Feature") self.assertEqual(item_dict["id"], "mock-collection-id") @@ -508,6 +537,40 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertIn("zarr-data", item_dict["assets"]) self.assertIn("zarr-consolidated-metadata", item_dict["assets"]) + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): + """Only the first item configuration is emitted in the STAC file dict.""" + mock_open_ds.return_value = self.mock_dataset + gen = OscDatasetStacGenerator( + collection_id="multi-collection", + items_config=[ + ItemConfig(dataset_id="first.zarr", item_id="first-item"), + ItemConfig(dataset_id="second.zarr", item_id="second-item"), + ], + workflow_id="dummy", + workflow_title="test", + license_type="proprietary", + osc_project="deep-earth-system-data-lab", + ) + + file_dict = gen.build_zarr_stac_catalog_file_dict( + "s3://test-bucket/stac/multi-collection/" + ) + + self.assertIn( + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/first-item.json", + file_dict, + ) + self.assertNotIn( + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/second-item.json", + file_dict, + ) + catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] + item_links = [lnk for lnk in catalog["links"] if lnk["rel"] == "item"] + self.assertEqual(len(item_links), 1) + def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): """A 'via' link (STAC browser) and a 'child' link (HTTPS catalog) are added when stac_catalog_s3_root is provided. @@ -520,21 +583,31 @@ def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): collection = self.generator.build_dataset_stac_collection( mode="dataset", stac_catalog_s3_root=s3_root ) - https_catalog = "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + https_catalog = ( + "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + ) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" + https_catalog.replace("https://", "") ) via_link = next( - (lnk for lnk in collection.links if lnk.rel == "via" and "stac-browser" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "via" and "stac-browser" in str(lnk.target) + ), None, ) self.assertIsNotNone(via_link, "Expected a 'via' STAC browser link") self.assertEqual(via_link.target, stac_browser_href) child_link = next( - (lnk for lnk in collection.links if lnk.rel == "child" and "catalog.json" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "child" and "catalog.json" in str(lnk.target) + ), None, ) self.assertIsNotNone(child_link, "Expected a 'child' HTTPS catalog link") @@ -612,20 +685,27 @@ def test_edge_cases(self): class TestOscDatasetStacGeneratorExtra(unittest.TestCase): """Additional tests to cover branches not exercised by TestOSCProductSTACGenerator.""" - def _make_generator(self, mock_ds, collection_id="my-collection", **kwargs): - with patch("deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds): + @staticmethod + def _make_generator(mock_ds, collection_id="my-collection", **kwargs): + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds + ): + kwargs.setdefault("osc_project", "deep-earth-system-data-lab") return OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id=collection_id, + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id=collection_id), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", **kwargs, ) - def _make_dataset(self, coord_type="lon_lat"): - import numpy as np + @staticmethod + def _make_dataset(coord_type="lon_lat"): from datetime import datetime + if coord_type == "lon_lat": coords = { "lon": ("lon", np.linspace(-10, 10, 3)), @@ -647,6 +727,7 @@ def _make_dataset(self, coord_type="lon_lat"): else: coords = {} from xarray import Dataset + return Dataset(coords=coords) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -654,11 +735,14 @@ def test_collection_id_with_space_raises(self, mock_open_ds): mock_open_ds.return_value = self._make_dataset() with self.assertRaisesRegex(ValueError, "must not contain spaces"): OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="bad id", + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id="bad-id"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -666,7 +750,7 @@ def test_spatial_extent_longitude_latitude(self, mock_open_ds): ds = self._make_dataset("longitude_latitude") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], -10.0) self.assertAlmostEqual(extent.bboxes[0][1], -5.0) @@ -675,7 +759,7 @@ def test_spatial_extent_x_y(self, mock_open_ds): ds = self._make_dataset("x_y") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], 0.0) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -684,7 +768,7 @@ def test_spatial_extent_unknown_coords_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "recognized spatial coordinates"): - gen._get_spatial_extent() + gen._get_spatial_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_temporal_extent_no_time_raises(self, mock_open_ds): @@ -692,7 +776,7 @@ def test_temporal_extent_no_time_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "time"): - gen._get_temporal_extent() + gen._get_temporal_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_normalize_name_none_returns_none(self, mock_open_ds): @@ -704,9 +788,14 @@ def test_normalize_name_none_returns_none(self, mock_open_ds): def test_build_collection_with_cf_params(self, mock_open_ds): ds = self._make_dataset() mock_open_ds.return_value = ds - gen = self._make_generator(ds, cf_params=[{"name": "temperature", "units": "K"}]) + gen = self._make_generator( + ds, cf_params=[{"name": "temperature", "units": "K"}] + ) collection = gen.build_dataset_stac_collection(mode="dataset") - self.assertEqual(collection.extra_fields.get("cf:parameter"), [{"name": "temperature", "units": "K"}]) + self.assertEqual( + collection.extra_fields.get("cf:parameter"), + [{"name": "temperature", "units": "K"}], + ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_build_collection_with_visualisation_link(self, mock_open_ds): @@ -725,11 +814,12 @@ def test_build_collection_mode_all_adds_experiment_link(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) collection = gen.build_dataset_stac_collection(mode="all") - exp_links = [lnk for lnk in collection.links if "experiments" in str(lnk.target)] + exp_links = [ + lnk for lnk in collection.links if "experiments" in str(lnk.target) + ] self.assertEqual(len(exp_links), 1) - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def test_s3_to_https(self, mock_open_ds): + def test_s3_to_https(self): self.assertEqual( OscDatasetStacGenerator._s3_to_https("s3://my-bucket/path/to/file.json"), "https://my-bucket.s3.amazonaws.com/path/to/file.json", @@ -756,10 +846,310 @@ def test_update_existing_variable_catalog(self, mock_open_ds): json.dump(base, f) tmp_path = f.name try: - result = gen.update_existing_variable_catalog(tmp_path, "var1") + result = gen.update_existing_variable_catalog(tmp_path) finally: os.unlink(tmp_path) rels = [lnk["rel"] for lnk in result["links"]] self.assertIn("child", rels) self.assertIn("related", rels) # theme link + + +class TestPRRCollection(unittest.TestCase): + """Tests for the PRR-style Collection -> Item -> Assets generation.""" + + def setUp(self): + self.dataset = Dataset( + coords={ + "lon": ("lon", np.linspace(-20, 20, 4)), + "lat": ("lat", np.linspace(-10, 10, 3)), + "time": ( + "time", + [ + np.datetime64(datetime(2021, 1, 1), "ns"), + np.datetime64(datetime(2021, 1, 3), "ns"), + ], + ), + }, + data_vars={ + "sst": ( + ("time", "lat", "lon"), + np.random.rand(2, 3, 4), + {"units": "K", "long_name": "Sea surface temperature"}, + ), + "chl": ( + ("time", "lat", "lon"), + np.random.rand(2, 3, 4), + {"units": "mg m-3"}, + ), + # A CRS var that must be excluded from cube:variables and drive EPSG. + "spatial_ref": ((), 0, {"spatial_epsg": 3035}), + }, + attrs={"description": "PRR test cube"}, + ) + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) + self.gen = OscDatasetStacGenerator( + collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", + access_link_root="s3://bucket", + osc_status="ongoing", + osc_region="Global", + osc_themes=["oceans"], + osc_missions=["sentinel-3"], + documentation_link="https://example.org/doc", + ) + + # ---- helpers ---- + + def test_get_crs_from_spatial_ref(self): + self.assertEqual(self.gen._get_crs(self.dataset).to_epsg(), 3035) + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_get_epsg_default_4326(self, mock_open_ds): + ds = Dataset( + coords={ + "lon": ("lon", np.linspace(-1, 1, 2)), + "lat": ("lat", np.linspace(-1, 1, 2)), + "time": ("time", [np.datetime64(datetime(2020, 1, 1), "ns")]), + }, + data_vars={"v": (("time", "lat", "lon"), np.random.rand(1, 2, 2))}, + ) + mock_open_ds.return_value = ds + gen = OscDatasetStacGenerator( + collection_id="c", + items_config=[ + ItemConfig(dataset_id="t.zarr", item_id="c"), + ], + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", + ) + self.assertEqual(gen._get_crs(ds).to_epsg(), 4326) + + def test_get_cube_dimensions(self): + dims = self.gen._get_cube_dimensions(self.dataset) + self.assertEqual(set(dims), {"lon", "lat", "time"}) + self.assertEqual(dims["lon"]["type"], "spatial") + self.assertEqual(dims["lon"]["axis"], "x") + self.assertEqual(dims["lon"]["reference_system"], 3035) + self.assertEqual(dims["lon"]["extent"], [-20.0, 20.0]) + self.assertEqual(dims["lat"]["axis"], "y") + self.assertEqual(dims["lat"]["extent"], [-10.0, 10.0]) + self.assertEqual(dims["time"]["type"], "temporal") + self.assertEqual(len(dims["time"]["extent"]), 2) + + def test_get_cube_variables(self): + variables = self.gen._get_cube_variables(self.dataset) + # CRS variable must be excluded. + self.assertEqual(set(variables), {"sst", "chl"}) + self.assertEqual(variables["sst"]["type"], "data") + self.assertEqual(variables["sst"]["dimensions"], ["time", "lat", "lon"]) + self.assertEqual(variables["sst"]["unit"], "K") + self.assertEqual(variables["sst"]["description"], "Sea surface temperature") + # chl has a unit but no long_name/description. + self.assertEqual(variables["chl"]["unit"], "mg m-3") + self.assertNotIn("description", variables["chl"]) + + # ---- item ---- + + def test_build_prr_stac_item(self): + item = self.gen.build_prr_stac_item(self.gen.items_config[0]) + self.assertIsInstance(item, Item) + self.assertEqual(item.id, "prr-collection") + self.assertIn(DATACUBE_SCHEMA_URI, item.stac_extensions) + self.assertIn("cube:dimensions", item.properties) + self.assertIn("cube:variables", item.properties) + + # datetime is null; start/end are timezone-aware ISO strings. + self.assertIsNone(item.datetime) + self.assertTrue(item.properties["start_datetime"].endswith("+00:00")) + self.assertTrue(item.properties["end_datetime"].endswith("+00:00")) + + self.assertEqual(set(item.assets), {"zarr-data", "zarr-consolidated-metadata"}) + self.assertEqual(item.assets["zarr-data"].href, "s3://bucket/test.zarr") + self.assertEqual(item.assets["zarr-data"].media_type, ZARR_MEDIA_TYPE) + self.assertEqual( + item.assets["zarr-consolidated-metadata"].href, + "s3://bucket/test.zarr/.zmetadata", + ) + + # ---- collection ---- + + def test_build_prr_collection(self): + coll = self.gen.build_prr_collection() + self.assertIsInstance(coll, Collection) + self.assertEqual(coll.id, "prr-collection") + self.assertEqual(coll.license, "CC-BY-4.0") + + ef = coll.extra_fields + self.assertEqual(ef["osc:type"], "product") + self.assertEqual(ef["osc:status"], "ongoing") + self.assertEqual(ef["osc:region"], "Global") + self.assertCountEqual(ef["osc:variables"], ["sst", "chl"]) + self.assertEqual(ef["osc:missions"], ["sentinel-3"]) + self.assertEqual(ef["cf:parameter"], [{"name": "prr-collection"}]) + self.assertIn("processing:datetime", ef) + + # Extensions declared. + self.assertIn(OSC_SCHEMA_URI, coll.stac_extensions) + self.assertIn(PROCESSING_SCHEMA_URI, coll.stac_extensions) + + # Themes are plain dicts (not Theme objects). + themes = ef["themes"] + self.assertEqual(themes[0]["scheme"], OSC_THEME_SCHEME) + self.assertEqual(themes[0]["concepts"], [{"id": "oceans"}]) + self.assertIsInstance(themes[0], dict) + + # The single Item is attached as a child. + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertEqual(items[0].id, "prr-collection") + + def test_build_prr_collection_conformant_fields(self): + """A fully configured generator emits every PRR-required field.""" + from deep_code.constants import SCIENTIFIC_SCHEMA_URI + + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ): + gen = OscDatasetStacGenerator( + collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link_root="s3://bucket/", + osc_status="ongoing", + osc_region="Global", + osc_themes=["oceans"], + osc_missions=["sentinel-3"], + osc_project="deep-earth-system-data-lab", + osc_project_description="A detailed project description.", + osc_project_website="https://project.example.org", + osc_contract_number="4000114410/15/NL/BW", + thumbnail="https://example.org/logo.jpeg", + sci_doi="10.1000/xyz123", + ) + coll = gen.build_prr_collection() + ef = coll.extra_fields + + # Scientific extension declared alongside the others. + self.assertIn(SCIENTIFIC_SCHEMA_URI, coll.stac_extensions) + # Required PRR project fields present. + self.assertEqual(ef["osc:initiative"], "earthcode") + self.assertEqual(ef["osc:project_website"], "https://project.example.org") + self.assertEqual( + ef["osc:project_description"], "A detailed project description." + ) + self.assertEqual(ef["osc:contract-number"], "4000114410/15/NL/BW") + self.assertEqual(ef["sci:doi"], "10.1000/xyz123") + # Thumbnail asset with correct role and guessed media type. + self.assertIn("thumbnail", coll.assets) + thumb = coll.assets["thumbnail"] + self.assertEqual(thumb.roles, ["thumbnail"]) + self.assertEqual(thumb.media_type, "image/jpeg") + + def test_build_prr_collection_fallbacks(self): + """project_website/description fall back to doc link / description.""" + coll = self.gen.build_prr_collection() + ef = coll.extra_fields + # Default initiative. + self.assertEqual(ef["osc:initiative"], "earthcode") + # Fallbacks: website -> documentation_link, description -> dataset description. + self.assertEqual(ef["osc:project_website"], "https://example.org/doc") + self.assertEqual(ef["osc:project_description"], "No description provided.") + # No thumbnail / contract number configured -> absent. + self.assertNotIn("thumbnail", coll.assets) + self.assertNotIn("osc:contract-number", ef) + + def test_thumbnail_media_type_guess(self): + self.gen.thumbnail = "https://x/logo.png" + self.assertEqual(self.gen._thumbnail_media_type(), "image/png") + self.gen.thumbnail = "https://x/logo.JPG" + self.assertEqual(self.gen._thumbnail_media_type(), "image/jpeg") + self.gen.thumbnail = "https://x/logo.webp" + self.assertEqual(self.gen._thumbnail_media_type(), "image/webp") + self.gen.thumbnail_media_type = "image/tiff" + self.assertEqual(self.gen._thumbnail_media_type(), "image/tiff") + + def test_build_prr_collection_cf_params_override(self): + self.gen.cf_params = [{"name": "sst", "units": "K"}] + coll = self.gen.build_prr_collection() + self.assertEqual( + coll.extra_fields["cf:parameter"], [{"name": "sst", "units": "K"}] + ) + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_prr_collection_no_themes(self, mock_open_ds): + mock_open_ds.return_value = self.dataset + gen = OscDatasetStacGenerator( + collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", + access_link_root="s3://bucket", + ) + coll = gen.build_prr_collection() + self.assertNotIn("themes", coll.extra_fields) + + # ---- save (local self-contained tree) ---- + + def test_save_prr_collection_writes_tree(self): + with tempfile.TemporaryDirectory() as tmp: + out = self.gen.save_prr_collection(tmp) + self.assertEqual(out, tmp) + + collection_path = os.path.join(tmp, "prr-collection", "collection.json") + item_path = os.path.join( + tmp, "prr-collection", "items", "prr-collection.json" + ) + self.assertTrue(os.path.isfile(collection_path)) + self.assertTrue(os.path.isfile(item_path)) + + # Files are plain-JSON serialisable (no leftover Python objects). + with open(collection_path) as f: + coll_dict = json.load(f) + with open(item_path) as f: + item_dict = json.load(f) + + self.assertEqual(coll_dict["type"], "Collection") + self.assertEqual(item_dict["type"], "Feature") + + # Structural links are relative; the Item link points at the child. + item_link = next(lnk for lnk in coll_dict["links"] if lnk["rel"] == "item") + self.assertFalse(item_link["href"].startswith("s3://")) + self.assertTrue(item_link["href"].endswith(".json")) + + # Asset hrefs stay absolute (the data lives on S3). + self.assertEqual( + item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" + ) + + def test_save_prr_collection_readable_by_pystac(self): + with tempfile.TemporaryDirectory() as tmp: + self.gen.save_prr_collection(tmp) + coll = Collection.from_file( + os.path.join(tmp, "prr-collection", "collection.json") + ) + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/tests/utils/test_github_automation.py b/deep_code/tests/utils/test_github_automation.py index 671adb3..25ca33b 100644 --- a/deep_code/tests/utils/test_github_automation.py +++ b/deep_code/tests/utils/test_github_automation.py @@ -47,6 +47,7 @@ def test_clone_sync_repository_new(self, mock_run): """ No .git directory → we clone and then ensure upstream remote gets added. """ + # Simulate: "git remote -v" returns nothing so we add 'upstream' def run_side_effect(args, cwd, check, capture_output=False, text=True): if args[:3] == ["git", "remote", "-v"]: @@ -55,8 +56,9 @@ def run_side_effect(args, cwd, check, capture_output=False, text=True): mock_run.side_effect = run_side_effect - with patch.object(Path, "mkdir") as _mk, patch( - "pathlib.Path.exists", side_effect=lambda p=None: False + with ( + patch.object(Path, "mkdir") as _mk, + patch("pathlib.Path.exists", side_effect=lambda p=None: False), ): self.gha.clone_sync_repository() @@ -239,9 +241,10 @@ def test_create_branch(self, mock_run): @patch("subprocess.run") def test_add_file(self, mock_run): mock_run.return_value = make_cp() - with patch.object(Path, "mkdir") as _mk, patch.object( - Path, "write_text" - ) as _wt: + with ( + patch.object(Path, "mkdir") as _mk, + patch.object(Path, "write_text") as _wt, + ): # Ensure .git exists with patch( "pathlib.Path.exists", diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 8fb49f3..5558501 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -4,14 +4,13 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. -from typing import Optional import yaml class TemplateGenerator: @staticmethod - def generate_workflow_template(output_path: Optional[str] = None) -> str: + def generate_workflow_template(output_path: str | None = None) -> str: """Generate a complete template with all possible keys and placeholder values""" workflow_template = { @@ -48,38 +47,74 @@ def generate_workflow_template(output_path: Optional[str] = None) -> str: with open(output_path, "w") as f: f.write("# Workflow Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") - f.write(yaml.dump(workflow_template, sort_keys=False, width=1000, - default_flow_style=False)) + f.write( + yaml.dump( + workflow_template, + sort_keys=False, + width=1000, + default_flow_style=False, + ) + ) @staticmethod - def generate_dataset_template(output_path: Optional[str] = None) -> str: + def generate_dataset_template(output_path: str | None = None) -> str: """Generate a complete dataset template with all possible keys and placeholder values""" required = { - "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", "collection_id": "[REQUIRED: unique identifier, no spaces — use hyphens (e.g. My-Dataset-2024)]", "license_type": "[REQUIRED: SPDX license identifier, e.g. CC-BY-4.0, MIT, proprietary]", "stac_catalog_s3_root": "[REQUIRED: S3 root for the STAC Catalog + Item, e.g. s3://my-bucket/stac/my-collection/]", + "items_config": [ + { + "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", + "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", + } + ], + "osc_project": "[REQUIRED: OSC project ID (e.g. deep-earth-system-data-lab)]", + "osc_project_url": "[REQUIRED: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", } optional = { - "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection. Defaults to the existing DeepESDL project collection]", - "osc_themes": ["[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]"], + "osc_themes": [ + "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" + ], "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]", "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]", "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", - "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", - "access_link": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public/{dataset_id}]", - "cf_parameter": [{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}], + "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", + "collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]", + "cf_parameter": [ + {"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"} + ], + } + + # Fields used only by `deep-code generate-prr-collection` to build a + # PRR (Project Results Repository) collection that conforms to + # https://eoresults.esa.int/prr_collection_specifications.html + prr = { + "prr_output_dir": "[OPTIONAL: local dir for the PRR collection tree — defaults to prr/{collection_id}]", + "osc_initiative": "[OPTIONAL: PRR initiative — 'earthcode' or 'apex' (default: earthcode)]", + "osc_contract_number": "[PRR-REQUIRED: ESA contract identifier, e.g. 4000114410/15/NL/BW]", + "osc_project_website": "[PRR-REQUIRED: project website URL — falls back to osc_project_url / documentation_link]", + "osc_project_description": "[PRR-REQUIRED: multi-line project description — falls back to description]", + "osc_missions": [ + "[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]" + ], + "thumbnail": "[PRR-REQUIRED: URL to a collection thumbnail image (jpeg/png/webp)]", + "thumbnail_media_type": "[OPTIONAL: thumbnail MIME type — guessed from the URL suffix if omitted]", + "sci_doi": "[OPTIONAL: dataset DOI, e.g. 10.1000/xyz123 (not a DOI link)]", + "sci_citation": "[OPTIONAL: human-readable citation for the dataset]", } stac_catalog_comment = ( "\n# stac_catalog_s3_root: deep-code writes the following files to this S3 root:\n" "# {stac_catalog_s3_root}/catalog.json (STAC Catalog root)\n" - "# {stac_catalog_s3_root}/{collection_id}/item.json (STAC Item for the whole Zarr)\n" + "# {stac_catalog_s3_root}/{collection_id}/items/{item_id}.json (STAC Item for each Zarr)\n" + "# items_config can contain multiple dataset/item pairs, but publish\n" + "# currently only consumes one item configuration.\n" "# S3 write credentials are resolved in order:\n" "# 1. STAC_S3_KEY / STAC_S3_SECRET env vars (STAC-specific, any bucket)\n" "# 2. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars\n" @@ -91,7 +126,23 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: f.write("# Dataset Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") f.write("# --- REQUIRED fields ---\n") - f.write(yaml.dump(required, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + required, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write("\n# --- OPTIONAL fields ---\n") - f.write(yaml.dump(optional, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + optional, sort_keys=False, width=1000, default_flow_style=False + ) + ) + f.write( + "\n# --- PRR fields (for `deep-code generate-prr-collection`) ---\n" + ) + f.write( + yaml.dump( + prr, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write(stac_catalog_comment) diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py new file mode 100644 index 0000000..6a65035 --- /dev/null +++ b/deep_code/tools/prr.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 by Brockmann Consult GmbH +# Permissions are hereby granted under the terms of the MIT License: +# https://opensource.org/licenses/MIT. + +"""Generate a PRR-style STAC Collection -> Item -> Assets tree as local files. + +The PRR (Project Results Repository) collection is a self-contained STAC tree +suitable for submission to the ESA EarthCODE PRR endpoint. It reuses the same +dataset config as the ``publish`` command but writes only local files and does +not require GitHub credentials or S3 access beyond reading the Zarr store. +""" + +import logging + +import fsspec +import yaml + +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator + +logger = logging.getLogger(__name__) + + +def generate_prr_collection( + dataset_config_path: str, output_dir: str | None = None +) -> str: + """Build and write a PRR STAC collection from a dataset config. + + Args: + dataset_config_path: Path to the dataset config YAML (the same file used + by ``deep-code publish``). + output_dir: Directory to write the collection tree into. Falls back to + ``prr_output_dir`` in the config, then ``prr/``. + + Returns: + The output directory the collection tree was written to. + """ + with fsspec.open(dataset_config_path, "r") as file: + config = yaml.safe_load(file) or {} + + collection_id = config.get("collection_id") + osc_project = config.get("osc_project") + osc_project_url = config.get("osc_project_url") + license_type = config.get("license_type") + items_config_raw = config.get("items_config") + if not collection_id: + raise ValueError("collection_id is required in the dataset config.") + if not osc_project: + raise ValueError("osc_project is required in the dataset config.") + if not osc_project_url: + raise ValueError("osc_project_url is required in the dataset config.") + if not license_type: + raise ValueError( + "license_type is required in the dataset config. " + "Provide an SPDX identifier (e.g. 'CC-BY-4.0', 'MIT', 'proprietary')." + ) + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + + logger.info(f"Generating PRR STAC collection for '{collection_id}'.") + generator = OscDatasetStacGenerator( + collection_id=collection_id, + items_config=items_config, + workflow_id=config.get("workflow_id") or "", + workflow_title=config.get("workflow_title") or "", + license_type=license_type, + documentation_link=config.get("documentation_link"), + collection_title=config.get("collection_title"), + access_link_root=config.get("access_link_root"), + osc_status=config.get("dataset_status") or "ongoing", + osc_region=config.get("osc_region") or "Global", + osc_themes=config.get("osc_themes"), + osc_missions=config.get("osc_missions"), + cf_params=config.get("cf_parameter"), + visualisation_link=config.get("visualisation_link"), + description=config.get("description"), + osc_project=config.get("osc_project"), + osc_project_title=config.get("osc_project_title"), + osc_project_url=config.get("osc_project_url"), + # PRR-specific project metadata. + osc_initiative=config.get("osc_initiative") or "earthcode", + osc_contract_number=config.get("osc_contract_number"), + osc_project_website=config.get("osc_project_website"), + osc_project_description=config.get("osc_project_description"), + thumbnail=config.get("thumbnail") or config.get("thumbnail_link"), + thumbnail_media_type=config.get("thumbnail_media_type"), + sci_doi=config.get("sci_doi"), + sci_citation=config.get("sci_citation"), + ) + + out_dir = output_dir or config.get("prr_output_dir") or "prr" + generator.save_prr_collection(out_dir) + return out_dir diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 3b62cbd..399f102 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,11 @@ OSC_REPO_OWNER, WORKFLOW_BASE_CATALOG_SELF_HREF, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + open_dataset, +) from deep_code.utils.github_automation import GitHubAutomation from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, @@ -160,6 +164,8 @@ def __init__( # Values that may be set from configs self.collection_id: str | None = None + self.osc_project: str | None = None + self.osc_project_url: str | None = None self.workflow_title: str | None = None self.workflow_id: str | None = None @@ -245,9 +251,24 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): / var_file_path ) file_dict[var_file_path] = generator.update_existing_variable_catalog( - full_path, var_id + full_path ) + @staticmethod + def _build_items_config(dataset_config: dict[str, Any]) -> list[ItemConfig]: + """Build item configs from the dataset config.""" + items_config_raw = dataset_config.get("items_config") + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + return items_config + def publish_dataset( self, write_to_file: bool = False, @@ -260,23 +281,31 @@ def publish_dataset( raise ValueError( "No dataset config loaded. Provide dataset_config_path to publish dataset." ) - dataset_id = self.dataset_config.get("dataset_id") + items_config = self._build_items_config(self.dataset_config) + if len(items_config) != 1: + raise ValueError( + "publish currently supports exactly one item configuration." + ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") - access_link = self.dataset_config.get("access_link") + access_link_root = self.dataset_config.get("access_link_root") dataset_status = self.dataset_config.get("dataset_status") or "ongoing" osc_region = self.dataset_config.get("osc_region") osc_themes = self.dataset_config.get("osc_themes") cf_params = self.dataset_config.get("cf_parameter") license_type = self.dataset_config.get("license_type") visualisation_link = self.dataset_config.get("visualisation_link") - osc_project = self.dataset_config.get("osc_project") + self.osc_project = self.dataset_config.get("osc_project") osc_project_title = self.dataset_config.get("osc_project_title") - osc_project_url = self.dataset_config.get("osc_project_url") + self.osc_project_url = self.dataset_config.get("osc_project_url") description = self.dataset_config.get("description") - if not dataset_id or not self.collection_id: - raise ValueError("Dataset ID or Collection ID missing in the config.") + if not self.collection_id: + raise ValueError("collection_id missing in the config.") + if not self.osc_project: + raise ValueError("osc_project missing in the config.") + if not self.osc_project_url: + raise ValueError("osc_project missing in the config.") if not license_type: raise ValueError( @@ -295,27 +324,28 @@ def publish_dataset( logger.info("Generating STAC collection...") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, + items_config=items_config, collection_id=self.collection_id, workflow_id=self.workflow_id, workflow_title=self.workflow_title, license_type=license_type, documentation_link=documentation_link, - access_link=access_link, + access_link_root=access_link_root, osc_status=dataset_status, osc_region=osc_region, osc_themes=osc_themes, cf_params=cf_params, visualisation_link=visualisation_link, - **({"osc_project": osc_project} if osc_project else {}), + osc_project=self.osc_project, osc_project_title=osc_project_title, - osc_project_url=osc_project_url, + osc_project_url=self.osc_project_url, description=description, ) # Store so publish() can reuse it for zarr STAC catalog generation self._last_generator = generator - variable_ids = generator.get_variable_ids() + dataset = open_dataset(generator.items_config[0].dataset_id) + variable_ids = generator.get_variable_ids(dataset) ds_collection = generator.build_dataset_stac_collection( mode=mode, stac_catalog_s3_root=stac_catalog_s3_root ) @@ -352,7 +382,9 @@ def publish_dataset( file_dict[project_collection_path] = generator.build_project_collection() # Add child link in the projects base catalog self._update_and_add_to_file_dict( - file_dict, "projects/catalog.json", generator.update_project_base_catalog + file_dict, + "projects/catalog.json", + generator.update_project_base_catalog, ) else: self._update_and_add_to_file_dict( diff --git a/deep_code/tools/test.py b/deep_code/tools/test.py index 5bdf092..b42e21f 100644 --- a/deep_code/tools/test.py +++ b/deep_code/tools/test.py @@ -1,2 +1,2 @@ -""" Execute the application package of a published experiment on a subset of input data +"""Execute the application package of a published experiment on a subset of input data to verify the reproducibility is achieved""" diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 716d509..373029b 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -5,15 +5,33 @@ import json import logging +from dataclasses import dataclass from datetime import datetime, timezone +from typing import Any +import numpy as np import pandas as pd -from pystac import Catalog, Collection, Extent, Item, Asset, Link, SpatialExtent, TemporalExtent +import pyproj +import xarray as xr +from pystac import ( + Asset, + Catalog, + CatalogType, + Collection, + Extent, + Item, + Link, + SpatialExtent, + TemporalExtent, +) from deep_code.constants import ( CONTACTS_SCHEMA_URI, + DATACUBE_SCHEMA_URI, OSC_SCHEMA_URI, OSC_THEME_SCHEME, + PROCESSING_SCHEMA_URI, + SCIENTIFIC_SCHEMA_URI, THEMES_SCHEMA_URI, ZARR_MEDIA_TYPE, ) @@ -22,13 +40,21 @@ from deep_code.utils.osc_extension import OscExtension +@dataclass +class ItemConfig: + dataset_id: str + item_id: str + + class OscDatasetStacGenerator: """Generates OSC STAC Collections for a product from Zarr datasets. Args: - dataset_id: ID of the Zarr dataset. collection_id: Unique identifier for the STAC collection. - access_link: Public access link to the dataset. + items_config: List of item configuration entries. Each item maps one + dataset_id to one item_id + collection_title: Title present in the collection and in the STAC browser + access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -40,96 +66,109 @@ class OscDatasetStacGenerator: def __init__( self, - dataset_id: str, collection_id: str, + items_config: list[ItemConfig], workflow_id: str, workflow_title: str, license_type: str, - access_link: str | None = None, + osc_project: str, + collection_title: str | None = None, + access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", osc_region: str = "Global", osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, - cf_params: list[dict[str]] | None = None, - osc_project: str = "deep-earth-system-data-lab", - osc_project_title: str = "DeepESDL", + cf_params: list[dict[str, Any]] | None = None, + osc_project_title: str | None = None, osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, + osc_initiative: str = "earthcode", + osc_contract_number: str | None = None, + osc_project_website: str | None = None, + osc_project_description: str | None = None, + thumbnail: str | None = None, + thumbnail_media_type: str | None = None, + sci_doi: str | None = None, + sci_citation: str | None = None, ): if " " in collection_id: raise ValueError( f"collection_id must not contain spaces: {collection_id!r}. " - "Use hyphens as word separators (e.g. 'My-Dataset-2024')." + "Use hyphens as word separators (e.g. 'My-Collection-2024')." ) - self.dataset_id = dataset_id self.collection_id = collection_id + self.items_config = items_config self.workflow_id = workflow_id self.workflow_title = workflow_title self.license_type = license_type self.osc_project = osc_project - self.osc_project_title = osc_project_title + self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url - self.access_link = access_link or f"s3://deep-esdl-public/{dataset_id}" + self.access_link_root = access_link_root or "s3://deep-esdl-public/" + self.collection_title = collection_title or collection_id self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region - self.osc_themes = [t.lower() for t in (osc_themes or [])] + if osc_themes is None: + osc_themes = [] + assert isinstance(osc_themes, list) + self.osc_themes = [t.lower() for t in osc_themes] self.osc_missions = osc_missions or [] self.cf_params = cf_params or {} self.visualisation_link = visualisation_link self.description = description + # PRR-specific project metadata (see PRR collection specification). + self.osc_initiative = osc_initiative or "earthcode" + self.osc_contract_number = osc_contract_number + self.osc_project_website = osc_project_website + self.osc_project_description = osc_project_description + self.thumbnail = thumbnail + self.thumbnail_media_type = thumbnail_media_type + self.sci_doi = sci_doi + self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - self.dataset = open_dataset(dataset_id=dataset_id, logger=self.logger) - self.variables_metadata = self.get_variables_metadata() - - def _get_spatial_extent(self) -> SpatialExtent: - """Extract spatial extent from the dataset.""" - if {"lon", "lat"}.issubset(self.dataset.coords): - # For regular gridding - lon_min, lon_max = ( - float(self.dataset.lon.min()), - float(self.dataset.lon.max()), - ) - lat_min, lat_max = ( - float(self.dataset.lat.min()), - float(self.dataset.lat.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"longitude", "latitude"}.issubset(self.dataset.coords): - # For regular gridding with 'longitude' and 'latitude' - lon_min, lon_max = ( - float(self.dataset.longitude.min()), - float(self.dataset.longitude.max()), - ) - lat_min, lat_max = ( - float(self.dataset.latitude.min()), - float(self.dataset.latitude.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"x", "y"}.issubset(self.dataset.coords): - # For irregular gridding - x_min, x_max = (float(self.dataset.x.min()), float(self.dataset.x.max())) - y_min, y_max = (float(self.dataset.y.min()), float(self.dataset.y.max())) - return SpatialExtent([[x_min, y_min, x_max, y_max]]) + + def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: + """Extract the spatial extent and return it in EPSG:4326.""" + + if {"lon", "lat"}.issubset(dataset.coords): + x_name, y_name = "lon", "lat" + elif {"longitude", "latitude"}.issubset(dataset.coords): + x_name, y_name = "longitude", "latitude" + elif {"x", "y"}.issubset(dataset.coords): + x_name, y_name = "x", "y" else: raise ValueError( "Dataset does not have recognized spatial coordinates " - "('lon', 'lat' or 'x', 'y')." + "('lon', 'lat'), ('longitude', 'latitude'), or ('x', 'y')." ) - def _get_temporal_extent(self) -> TemporalExtent: + x_min = float(dataset[x_name].min()) + x_max = float(dataset[x_name].max()) + y_min = float(dataset[y_name].min()) + y_max = float(dataset[y_name].max()) + + crs = self._get_crs(dataset) + + if crs.to_epsg() != 4326: + transformer = pyproj.Transformer.from_crs(crs, 4326, always_xy=True) + x_min, y_min, x_max, y_max = transformer.transform_bounds( + x_min, y_min, x_max, y_max + ) + + return SpatialExtent([[x_min, y_min, x_max, y_max]]) + + @staticmethod + def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: """Extract temporal extent from the dataset.""" - if "time" in self.dataset.coords: + dataset = dataset + if "time" in dataset.coords: try: # Convert the time bounds to datetime objects - time_min = pd.to_datetime( - self.dataset.time.min().values - ).to_pydatetime() - time_max = pd.to_datetime( - self.dataset.time.max().values - ).to_pydatetime() + time_min = pd.to_datetime(dataset.time.min().values).to_pydatetime() + time_max = pd.to_datetime(dataset.time.max().values).to_pydatetime() return TemporalExtent([[time_min, time_max]]) except Exception as e: raise ValueError(f"Failed to parse temporal extent: {e}") @@ -142,11 +181,47 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - def _get_general_metadata(self) -> dict: + def _build_access_link(self, item_config: ItemConfig) -> str: + """Return the asset href for an item, supporting prefix and full URLs.""" + root = self.access_link_root + if root.endswith("/"): + root = root.rstrip("/") + return f"{root}/{item_config.dataset_id}" + + @staticmethod + def _union_spatial_extent(items: list[Item]) -> SpatialExtent: + """Merge multiple dataset spatial extents into a single bounding box.""" + bboxes = [item.bbox for item in items] + return SpatialExtent( + [ + [ + min(bbox[0] for bbox in bboxes), + min(bbox[1] for bbox in bboxes), + max(bbox[2] for bbox in bboxes), + max(bbox[3] for bbox in bboxes), + ] + ] + ) + + @staticmethod + def _union_temporal_extent(items: list[Item]) -> TemporalExtent: + """Merge multiple dataset temporal extents into a single interval.""" + intervals = np.array( + [ + [ + datetime.fromisoformat(item.properties["start_datetime"]), + datetime.fromisoformat(item.properties["end_datetime"]), + ] + for item in items + ] + ) + return TemporalExtent([[min(intervals[:, 0]), max(intervals[:, 1])]]) + + def _get_general_metadata(self, dataset: xr.Dataset) -> dict: return { "description": ( self.description - or self.dataset.attrs.get("description") + or dataset.attrs.get("description") or "No description available." ) } @@ -164,19 +239,19 @@ def extract_metadata_for_variable(self, variable_data) -> dict: "gcmd_keyword_url": gcmd_keyword_url, } - def get_variable_ids(self) -> list[str]: + def get_variable_ids(self, dataset: xr.Dataset) -> list[str]: """Get variable IDs for all variables in the dataset.""" - variable_ids = list(self.variables_metadata.keys()) + variable_ids = list(self.get_variables_metadata(dataset).keys()) # Remove 'crs' and 'spatial_ref' from the list if they exist, note that # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. return [ var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"] ] - def get_variables_metadata(self) -> dict[str, dict]: + def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" variables_metadata = {} - for var_name, variable in self.dataset.data_vars.items(): + for variable in dataset.data_vars.values(): var_metadata = self.extract_metadata_for_variable(variable) variables_metadata[var_metadata.get("variable_id")] = var_metadata return variables_metadata @@ -415,7 +490,9 @@ def build_project_collection(self) -> dict: CONTACTS_SCHEMA_URI, ], "title": self.format_string(self.osc_project_title or self.osc_project), - "description": self.format_string(self.osc_project_title or self.osc_project), + "description": self.format_string( + self.osc_project_title or self.osc_project + ), "keywords": [], "license": "various", "extent": { @@ -472,7 +549,7 @@ def update_deepesdl_collection(self, deepesdl_collection_full_path) -> dict: ) return data - def update_existing_variable_catalog(self, var_file_path, var_id) -> dict: + def update_existing_variable_catalog(self, var_file_path) -> dict: """Append child and theme links to an existing variable catalog.""" with open(var_file_path, encoding="utf-8") as f: data = json.load(f) @@ -506,7 +583,7 @@ def _s3_to_https(s3_url: str) -> str: Example: s3://my-bucket/path/to/file → https://my-bucket.s3.amazonaws.com/path/to/file """ - without_scheme = s3_url[len("s3://"):] + without_scheme = s3_url[len("s3://") :] bucket, _, key = without_scheme.partition("/") return f"https://{bucket}.s3.amazonaws.com/{key}" @@ -519,40 +596,50 @@ def format_string(s: str) -> str: @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) - def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: + def build_zarr_stac_item( + self, + item_config: ItemConfig, + stac_catalog_s3_root: str, + ) -> Item: """Build a single STAC Item representing the entire Zarr store. One item covers the full spatiotemporal extent of the dataset. Assets point to the Zarr store and its consolidated metadata. Args: + item_config: object containing `dataset_id` and `item_id` stac_catalog_s3_root: S3 root URL where the STAC catalog will be hosted (e.g. ``s3://my-bucket/stac/``). Used to build self/root/parent hrefs. Returns: A :class:`pystac.Item` ready to be serialised to S3. """ - self.logger.info(f"Building STAC Item for collection '{self.collection_id}'.") - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + self.logger.info( + f"Building STAC Item {item_config.item_id} " + f"for collection '{self.collection_id}'." + ) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -572,11 +659,12 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: ) item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, properties={ + "title": self.format_string(item_config.item_id), "start_datetime": start_dt.isoformat() if start_dt else None, "end_datetime": end_dt.isoformat() if end_dt else None, "description": general_metadata.get("description", ""), @@ -586,26 +674,39 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: ) item.collection_id = self.collection_id item.set_self_href(item_href) - item.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - item.add_link(Link(rel="parent", target=catalog_href, media_type="application/json")) - item.add_link(Link( - rel="collection", - target=osc_collection_href, - media_type="application/json", - title=self.collection_id, - )) - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) + item.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link(rel="parent", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link( + rel="collection", + target=osc_collection_href, + media_type="application/json", + title=self.collection_id, + ) + ) + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) self.logger.info(f"STAC Item built: {item_href}") return item @@ -620,7 +721,7 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ - └── item.json # STAC Item (whole Zarr) + └── item.json # STAC Item (whole Zarr) Args: stac_catalog_s3_root: S3 root URL (e.g. ``s3://my-bucket/stac/``). @@ -632,42 +733,437 @@ def build_zarr_stac_catalog_file_dict( f"Building STAC Catalog file dict for collection '{self.collection_id}' " f"at root '{stac_catalog_s3_root}'." ) + root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - - item = self.build_zarr_stac_item(stac_catalog_s3_root) - catalog = Catalog( id=f"{self.collection_id}-stac-catalog", description=f"STAC Catalog for {self.collection_id}", ) catalog.set_self_href(catalog_href) - catalog.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - catalog.add_link(Link( - rel="item", - target=f"./{self.collection_id}/item.json", - media_type="application/json", - title=self.collection_id, - )) + catalog.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + + item_config = self.items_config[0] + item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) + catalog.add_link( + Link( + rel="item", + target=f"./{self.collection_id}/items/{item_config.item_id}.json", + media_type="application/json", + title=item_config.item_id, + ) + ) + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" - item_href = f"{root}/{self.collection_id}/item.json" self.logger.info(f"STAC Catalog file dict ready: {catalog_href}, {item_href}") return { catalog_href: catalog.to_dict(transform_hrefs=False), item_href: item.to_dict(transform_hrefs=False), } - def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | None = None) -> Collection: + # --------------------------------------------------------------------- # + # PRR (Project Results Repository) style output # + # # + # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # + # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # + # the plain {collection_id}/items/{item_id}.json under ``{root}/prr/``. # + # --------------------------------------------------------------------- # + + @staticmethod + def _get_crs(dataset: xr.Dataset) -> pyproj.CRS: + """Best-effort EPSG code for the dataset, defaulting to 4326. + + Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or + ``spatial_ref`` variable when present; otherwise assumes geographic + WGS 84 (EPSG:4326). + """ + for var_name in ("spatial_ref", "crs"): + if var_name in dataset.variables: + attrs = dataset[var_name].attrs + try: + return pyproj.CRS.from_cf(attrs) + except pyproj.exceptions.CRSError: + for key in ("spatial_epsg", "epsg", "EPSG"): + if key in attrs: + try: + return pyproj.CRS.from_epsg(attrs[key]) + except (TypeError, ValueError): + pass + return pyproj.CRS.from_epsg(4326) + + def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: + """Build the ``cube:dimensions`` object from the dataset coordinates. + + Follows the datacube STAC extension: horizontal spatial dimensions are + classified by axis (x/y), the ``time`` coordinate becomes a temporal + dimension, and any remaining index coordinate is emitted as an + additional dimension. + """ + crs = self._get_crs(dataset) + x_names = {"lon", "longitude", "x"} + y_names = {"lat", "latitude", "y"} + dimensions: dict[str, dict] = {} + for name, coord in dataset.coords.items(): + if name not in dataset.dims: + # Skip non-dimension coordinates (e.g. scalar or auxiliary coords). + continue + name = str(name) + lname = name.lower() + if lname in x_names: + dimensions[name] = { + "type": "spatial", + "axis": "x", + "extent": [float(coord.min()), float(coord.max())], + "reference_system": crs.to_epsg(), + } + elif lname in y_names: + dimensions[name] = { + "type": "spatial", + "axis": "y", + "extent": [float(coord.min()), float(coord.max())], + "reference_system": crs.to_epsg(), + } + elif lname == "time": + time_min = pd.to_datetime(coord.min().values).to_pydatetime() + time_max = pd.to_datetime(coord.max().values).to_pydatetime() + dimensions[name] = { + "type": "temporal", + "extent": [time_min.isoformat(), time_max.isoformat()], + } + else: + try: + dimensions[name] = { + "type": lname, + "extent": [float(coord.min()), float(coord.max())], + } + except (TypeError, ValueError): + dimensions[name] = { + "type": lname, + "values": [str(v) for v in coord.values.tolist()], + } + return dimensions + + @staticmethod + def _get_cube_variables(dataset: xr.Dataset) -> dict[str, dict]: + """Build the ``cube:variables`` object from the dataset data variables.""" + skip = {"crs", "spatial_ref"} + variables: dict[str, dict] = {} + for name, var in dataset.data_vars.items(): + if name in skip: + continue + entry: dict = { + "type": "data", + "dimensions": [str(d) for d in var.dims], + } + unit = var.attrs.get("units") + if unit: + entry["unit"] = unit + description = var.attrs.get("long_name") or var.attrs.get("description") + if description: + entry["description"] = description + variables[str(name)] = entry + return variables + + def build_prr_stac_item(self, item_config: ItemConfig) -> Item: + """Build the single datacube Item for the PRR collection. + + One Item covers the full spatiotemporal extent of the Zarr store. It + carries the datacube extension (``cube:dimensions`` / ``cube:variables``) + and two assets pointing at the Zarr store. Structural links + (root/parent/collection/self) are left for :meth:`save_prr_collection` + to fill in via ``Collection.add_item`` + ``normalize_hrefs``. + """ + self.logger.info( + f"Building PRR STAC Item '{item_config.item_id}' " + f"for collection '{self.collection_id}'." + ) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) + + bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] + lon_min, lat_min, lon_max, lat_max = bbox + geometry = { + "type": "Polygon", + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], + } + + start_dt, end_dt = temporal_extent.intervals[0] + if start_dt is not None and start_dt.tzinfo is None: + start_dt = start_dt.replace(tzinfo=timezone.utc) + if end_dt is not None and end_dt.tzinfo is None: + end_dt = end_dt.replace(tzinfo=timezone.utc) + + now_iso = datetime.now(timezone.utc).isoformat() + + item = Item( + id=item_config.item_id, + geometry=geometry, + bbox=bbox, + datetime=None, + properties={ + "start_datetime": start_dt.isoformat() if start_dt else None, + "end_datetime": end_dt.isoformat() if end_dt else None, + "description": general_metadata.get("description", ""), + "created": now_iso, + "updated": now_iso, + "cube:dimensions": self._get_cube_dimensions(dataset), + "cube:variables": self._get_cube_variables(dataset), + }, + ) + item.stac_extensions.append(DATACUBE_SCHEMA_URI) + # Asset hrefs stay absolute (the Zarr lives on S3); only the structural + # links become relative when the tree is normalised locally. + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) + + self.logger.info( + f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." + ) + return item + + def build_prr_collection(self) -> Collection: + """Build the PRR parent Collection with its datacube Item(s) attached. + + The Collection carries OSC extension fields (``osc:type``, ``osc:status``, + ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list + and ``processing:datetime`` — aligning with the ESA EarthCODE PRR + endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child, so + it produces a self-contained tree. + """ + items = [ + self.build_prr_stac_item(item_config) for item_config in self.items_config + ] + spatial_extent = self._union_spatial_extent(items) + temporal_extent = self._union_temporal_extent(items) + dataset_ref = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + variables = self.get_variable_ids(dataset_ref) + + collection = Collection( + id=self.collection_id, + description=self.description or "No description provided.", + extent=Extent(spatial=spatial_extent, temporal=temporal_extent), + license=self.license_type, + title=self.collection_title, + ) + collection.stac_version = "1.0.0" + + osc_extension = OscExtension.add_to(collection) + osc_extension.osc_project = self.osc_project + osc_extension.osc_type = "product" + osc_extension.osc_status = self.osc_status + osc_extension.osc_region = self.osc_region + osc_extension.osc_variables = variables + osc_extension.osc_missions = self.osc_missions + osc_extension.cf_parameter = self.cf_params or [{"name": self.collection_id}] + + now_iso = datetime.now(timezone.utc).isoformat() + collection.extra_fields["created"] = now_iso + collection.extra_fields["updated"] = now_iso + + # Processing extension — the PRR reference collection declares this and + # exposes 'processing:datetime' (the ingestion/generation timestamp). + if PROCESSING_SCHEMA_URI not in collection.stac_extensions: + collection.stac_extensions.append(PROCESSING_SCHEMA_URI) + collection.extra_fields["processing:datetime"] = now_iso + + if self.osc_themes: + # Plain dicts (not Theme objects) so the collection serialises with a + # plain json encoder as well as pystac's own writer. + collection.extra_fields["themes"] = [ + { + "concepts": [{"id": theme} for theme in self.osc_themes], + "scheme": OSC_THEME_SCHEME, + } + ] + + # ---- Required PRR project-level metadata ---- + # Scientific extension is REQUIRED in the PRR profile's extension list; + # 'sci:*' values themselves remain optional. + if SCIENTIFIC_SCHEMA_URI not in collection.stac_extensions: + collection.stac_extensions.append(SCIENTIFIC_SCHEMA_URI) + + collection.extra_fields["osc:initiative"] = self.osc_initiative + project_website = ( + self.osc_project_website or self.osc_project_url or self.documentation_link + ) + if project_website: + collection.extra_fields["osc:project_website"] = project_website + project_description = ( + self.osc_project_description + or self.description + or "No description provided." + ) + if project_description: + collection.extra_fields["osc:project_description"] = project_description + if self.osc_contract_number: + collection.extra_fields["osc:contract-number"] = self.osc_contract_number + + # Optional scientific metadata. + if self.sci_doi: + collection.extra_fields["sci:doi"] = self.sci_doi + if self.sci_citation: + collection.extra_fields["sci:citation"] = self.sci_citation + + # Thumbnail asset (REQUIRED by the PRR spec: an asset named 'thumbnail' + # with the 'thumbnail' role). + if self.thumbnail: + collection.add_asset( + "thumbnail", + Asset( + href=self.thumbnail, + media_type=self._thumbnail_media_type(), + title="Collection Thumbnail", + roles=["thumbnail"], + ), + ) + + if self.documentation_link: + collection.add_link( + Link(rel="via", target=self.documentation_link, title="Documentation") + ) + if self.visualisation_link: + collection.add_link( + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) + ) + + try: + osc_extension.validate_extension() + except ValueError as e: + raise ValueError(f"OSC Extension validation failed: {e}") + + self._warn_missing_prr_fields(collection, variables) + for item in items: + collection.add_item(item) + return collection + + def _thumbnail_media_type(self) -> str: + """Return the thumbnail media type, guessed from the href suffix.""" + if self.thumbnail_media_type: + return self.thumbnail_media_type + href = (self.thumbnail or "").lower() + if href.endswith((".jpg", ".jpeg")): + return "image/jpeg" + if href.endswith(".webp"): + return "image/webp" + return "image/png" + + def _warn_missing_prr_fields( + self, collection: Collection, variables: list[str] + ) -> None: + """Log a warning for PRR-required fields that are absent, so the caller + knows the output is not yet spec-conformant.""" + ef = collection.extra_fields + missing = [] + if "thumbnail" not in collection.assets: + missing.append("thumbnail asset") + if not ef.get("osc:contract-number"): + missing.append("osc:contract-number") + if not ef.get("osc:project_website"): + missing.append("osc:project_website") + if not ef.get("osc:project_description"): + missing.append("osc:project_description") + if not ef.get("themes"): + missing.append("themes") + if not variables: + missing.append("osc:variables") + if not self.osc_missions: + missing.append("osc:missions") + if missing: + self.logger.warning( + "PRR collection is missing required field(s): %s. " + "The collection will not fully conform to the PRR specification " + "until these are provided in the dataset config.", + ", ".join(missing), + ) + + def save_prr_collection(self, output_dir: str) -> str: + """Write the PRR ``Collection -> Item -> Assets`` tree to a local folder. + + Produces a self-contained STAC tree with relative structural links, + ready to inspect or submit to the ESA EarthCODE PRR endpoint:: + + {output_dir}/ + └── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) + + Zarr asset hrefs remain absolute (``s3://…``) since that is where the + data lives. + + Args: + output_dir: Local directory to write the collection tree into. + + Returns: + The ``output_dir`` that was written. + """ + self.logger.info( + f"Writing PRR STAC collection for '{self.collection_id}' to " + f"'{output_dir}'." + ) + collection = self.build_prr_collection() + + # Set absolute HREFs for writing. + collection_dir = f"{output_dir}/{self.collection_id}" + items_dir = f"{collection_dir}/items" + collection.set_self_href(f"{collection_dir}/collection.json") + for item in collection.get_items(): + item.set_self_href(f"{items_dir}/{item.id}.json") + + # Write the collection and its children. + collection.save(catalog_type=CatalogType.SELF_CONTAINED) + self.logger.info(f"PRR STAC collection written to '{output_dir}'.") + return output_dir + + def build_dataset_stac_collection( + self, mode: str, stac_catalog_s3_root: str | None = None + ) -> Collection: """Build an OSC STAC Collection for the dataset. Returns: A pystac.Collection object. """ try: - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + assert len(self.items_config) == 1 + dataset = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + variables = self.get_variable_ids(dataset) + general_metadata = self._get_general_metadata(dataset) except ValueError as e: raise ValueError(f"Metadata extraction failed: {e}") @@ -714,7 +1210,11 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) if self.visualisation_link: collection.add_link( - Link(rel="visualisation", target=self.visualisation_link, title="Dataset visualisation") + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) ) collection.add_link( Link( @@ -767,7 +1267,7 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) ) - if mode in "all": + if mode == "all": collection.add_link( Link( rel="related", @@ -789,19 +1289,23 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N catalog_https = self._s3_to_https(catalog_s3) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" - + catalog_https[len("https://"):] + + catalog_https[len("https://") :] + ) + collection.add_link( + Link( + rel="via", + target=stac_browser_href, + title="Access", + ) + ) + collection.add_link( + Link( + rel="child", + target=catalog_https, + media_type="application/json", + title="Items", + ) ) - collection.add_link(Link( - rel="via", - target=stac_browser_href, - title="Access", - )) - collection.add_link(Link( - rel="child", - target=catalog_https, - media_type="application/json", - title="Items", - )) # Validate OSC extension fields try: diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 9452b81..a27a786 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -1,6 +1,5 @@ import logging import os -from typing import Optional import xarray as xr from xcube.core.store import new_data_store @@ -25,8 +24,8 @@ def serialize(obj): def open_dataset( dataset_id: str, root: str = "deep-esdl-public", - storage_configs: Optional[list[dict]] = None, - logger: Optional[logging.Logger] = None, + storage_configs: list[dict] | None = None, + logger: logging.Logger | None = None, ) -> xr.Dataset: """Open an xarray dataset from a specified store. @@ -63,10 +62,15 @@ def open_dataset( "root": os.environ.get("S3_USER_STORAGE_BUCKET", root), "storage_options": { "anon": False, - **({ - "key": os.environ["S3_USER_STORAGE_KEY"], - "secret": os.environ["S3_USER_STORAGE_SECRET"], - } if os.environ.get("S3_USER_STORAGE_KEY") and os.environ.get("S3_USER_STORAGE_SECRET") else {}), + **( + { + "key": os.environ["S3_USER_STORAGE_KEY"], + "secret": os.environ["S3_USER_STORAGE_SECRET"], + } + if os.environ.get("S3_USER_STORAGE_KEY") + and os.environ.get("S3_USER_STORAGE_SECRET") + else {} + ), }, }, }, diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 4a46fb9..9d80152 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from urllib.parse import quote, urlencode, urlparse from xrlint.util.constructible import MappingConstructible @@ -72,13 +72,13 @@ def __init__( description: str, osc_project: str, jupyter_kernel_info: JupyterKernelInfo = None, - osc_workflow: str = None, - updated: str = None, - contacts: list[Contact] = None, - themes: list[Theme] = None, + osc_workflow: str | None = None, + updated: str | None = None, + contacts: list[Contact] | None = None, + themes: list[Theme] | None = None, keywords: list[str] | None = None, formats: list[dict] | None = None, - license: str = None, + license: str | None = None, ): self.created = created self.updated = updated @@ -110,7 +110,7 @@ def to_dict(self, value_name: str | None = None) -> dict[str, JsonValue]: class LinksBuilder: - def __init__(self, themes: list[str], jupyter_kernel_info: dict[str]): + def __init__(self, themes: list[str], jupyter_kernel_info: dict[str, Any]): self.themes = themes self.jupyter_kernel_info = jupyter_kernel_info self.theme_links = [] @@ -152,7 +152,7 @@ def build_child_link_to_related_experiment( } ] - def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, Any]]: + def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> list[dict[str, Any]]: return [ { "rel": "application", @@ -171,7 +171,7 @@ def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, An ] @staticmethod - def _parse_github_notebook_url(url: str) -> Tuple[str, str, str, str]: + def _parse_github_notebook_url(url: str) -> tuple[str, str, str, str]: """ Returns (repo_url, repo_name, branch, file_path_in_repo) from a GitHub URL. @@ -233,7 +233,7 @@ def make_related_link_for_opening_jnb_from_github( jupyter_notebook_url: str, title: str = "Open notebook on the DeepESDL platform", branch_override: str | None = None, - ) -> dict[str, str]: + ) -> list[dict[str, str]]: return [ { "rel": "related", @@ -255,11 +255,13 @@ def __init__( jupyter_notebook_url: str, properties: RecordProperties, links: list[dict], - linkTemplates: list = [], - conformsTo: list[str] = None, - geometry: Optional[Any] = None, - themes: Optional[Any] = None, + linkTemplates: list | None = None, + conformsTo: list[str] | None = None, + geometry: Any | None = None, + themes: Any | None = None, ): + if linkTemplates is None: + linkTemplates = [] if conformsTo is None: conformsTo = [ OGC_API_RECORD_SPEC, @@ -331,8 +333,8 @@ def __init__( properties: RecordProperties, links: list[dict], linkTemplates=None, - conformsTo: list[str] = None, - geometry: Optional[Any] = None, + conformsTo: list[str] | None = None, + geometry: Any | None = None, ): if linkTemplates is None: linkTemplates = [] diff --git a/deep_code/utils/ogc_record_generator.py b/deep_code/utils/ogc_record_generator.py index 83bd08a..e300a60 100644 --- a/deep_code/utils/ogc_record_generator.py +++ b/deep_code/utils/ogc_record_generator.py @@ -16,26 +16,24 @@ class OSCWorkflowOGCApiRecordGenerator: - """Generates OGC API record for a workflow - """ + """Generates OGC API record for a workflow""" @staticmethod def build_contact_objects(contacts_list: list[dict]) -> list[Contact]: """Build a list of Contact objects from a list of contact dictionaries. - Uses the inherited MappingConstructible logic to parse each dict. + Uses the inherited MappingConstructible logic to parse each dict. - Args: - contacts_list: A list of dictionaries, each containing contact information. + Args: + contacts_list: A list of dictionaries, each containing contact information. - Returns: - A list of Contact instances. - """ + Returns: + A list of Contact instances. + """ return [Contact.from_value(cdict) for cdict in contacts_list] @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) diff --git a/docs/cli.md b/docs/cli.md index 734927a..6c5c89b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -45,3 +45,43 @@ Options: 3. Forks/clones the target metadata repo (production, staging, or testing), commits generated JSON, and opens a pull request on your behalf. The pull request description includes a "Generated with deep-code" attribution note. + +## Generate a PRR collection + +Generate a **Project Results Repository (PRR)** STAC collection as local files, ready +to submit to the [ESA EarthCODE PRR endpoint](https://eoresults.esa.int): + +```bash +deep-code generate-prr-collection dataset.yaml # writes to prr/ +deep-code generate-prr-collection dataset.yaml -o ./prr # custom output directory +``` + +This reuses the **same dataset config** as `publish`, but writes only local files and +needs no GitHub credentials or S3 write access (it only reads the Zarr store). It +produces a self-contained `Collection → Item → Assets` tree: + +``` +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) +``` + +- The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` + extracted from the Zarr) plus `zarr-data` and `zarr-consolidated-metadata` assets. +- The **Collection** carries the OSC, Scientific, Processing, Themes and CF extensions + and the PRR-mandatory fields. +- `deep-code publish` still publishes one dataset/item at a time; the multi-item + generator support is exposed first through the lower-level API and the PRR helper. + +The output conforms to the +[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html) +when the PRR fields are set in the config. If any required field is missing, the command +still runs but logs a warning listing what is needed for full conformance. See +[PRR collection fields](configuration.md#prr-collection-fields). + +Options: + +- `--output-dir/-o`: directory to write the tree into. Defaults to `prr`. diff --git a/docs/configuration.md b/docs/configuration.md index d8d90d6..cd4020d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,7 +7,7 @@ deep-code generate-config # writes to current directory deep-code generate-config -o ./configs # custom output folder ``` -This creates `dataset_config.yaml` and `workflow_config.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). +This creates `dataset.yaml` and `workflow.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). The sections below document every field in those templates. @@ -16,10 +16,14 @@ The sections below document every field in those templates. ## Dataset config (YAML) ```yaml # Required -dataset_id: your-dataset.zarr collection_id: your-collection # no spaces — use hyphens license_type: CC-BY-4.0 stac_catalog_s3_root: s3://bucket/stac/your-collection/ +items_config: + - dataset_id: your-dataset.zarr + item_id: your-item # no spaces — use hyphens +osc_project: osc-project-name +osc_project_url: osc-project-url # Optional osc_themes: [cryosphere] # must match slugs at opensciencedata.esa.int/themes/catalog — auto-lowercased @@ -27,32 +31,64 @@ osc_region: global dataset_status: completed # ongoing | completed | planned (default: ongoing) documentation_link: https://example.com/docs visualisation_link: https://example.com/viewer # URL to a visualisation of the dataset -osc_project: deep-earth-system-data-lab # defaults to deep-earth-system-data-lab -access_link: s3://bucket/your-dataset.zarr # defaults to s3://deep-esdl-public/{dataset_id} +access_link_root: s3://bucket/ # defaults to s3://deep-esdl-public # CF parameter overrides (list of {name, units, ...} dicts) cf_parameter: - name: sea_surface_temperature units: kelvin + +# PRR fields (only used by `deep-code generate-prr-collection`) +osc_initiative: earthcode # earthcode | apex (default: earthcode) +osc_missions: [sentinel-3] +osc_contract_number: 4000114410/15/NL/BW +osc_project_website: https://project.example.org +osc_project_description: A detailed multi-line description of the project. +thumbnail: https://example.org/thumbnail.jpeg +sci_doi: 10.1000/xyz123 +prr_output_dir: ./prr/your-collection ``` ### Field reference +| Field | Required | Description | +|------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | +| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | +| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | +| `osc_project` | Yes | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). | +| `osc_project_url` | Yes | OSC project url used to link to project. | +| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | +| `osc_region` | No | Geographical region label (default: `Global`). | +| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | +| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | +| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | +| `documentation_link` | No | URL to dataset documentation. | +| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | +| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | +| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | +| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | + +> The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). + +### PRR collection fields + | Field | Required | Description | |---|---|---| -| `dataset_id` | Yes | Zarr store identifier (used to open the dataset). | -| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Dataset-2024`). | -| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | -| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | -| `osc_region` | No | Geographical region label (default: `Global`). | -| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | -| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | -| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | -| `documentation_link` | No | URL to dataset documentation. | -| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | -| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | -| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | -| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +| `osc_initiative` | No | PRR initiative: `earthcode` or `apex` (default: `earthcode`). | +| `osc_missions` | PRR-required | List of satellite mission name(s), e.g. `[sentinel-3]`. | +| `osc_contract_number` | PRR-required | ESA contract identifier, e.g. `4000114410/15/NL/BW`. | +| `osc_project_website` | PRR-required | Project website URL. Falls back to `osc_project_url`, then `documentation_link`. | +| `osc_project_description` | PRR-required | Multi-line project description. Falls back to `description`. | +| `thumbnail` | PRR-required | URL to a collection thumbnail image (jpeg/png/webp). Added as an asset named `thumbnail` with role `thumbnail`. | +| `thumbnail_media_type` | No | Thumbnail MIME type. Guessed from the URL suffix when omitted. | +| `sci_doi` | No | Dataset DOI, e.g. `10.1000/xyz123` (a DOI name, not a link). | +| `sci_citation` | No | Human-readable citation for the dataset. | +| `prr_output_dir` | No | Local directory for the PRR collection tree. Defaults to `prr/{collection_id}`. | + +`themes` for a PRR collection must be drawn from the allowed set: +`atmosphere`, `cryosphere`, `land`, `magnetosphere-ionosphere`, `oceans`, `solid-earth` +(configured via `osc_themes`). See [PRR collection output](#prr-collection-output). ### STAC Catalog on S3 @@ -62,7 +98,8 @@ cf_parameter: s3://bucket/stac/your-collection/ ├── catalog.json # STAC Catalog (root) └── your-collection/ - └── item.json # STAC Item covering the full Zarr store + └── items/ + └── your-item.json # STAC Item covering the full Zarr store ``` The item has two assets: @@ -79,6 +116,36 @@ S3 credentials for writing the STAC catalog are resolved in this order: then `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`, then the boto3 default chain (IAM role, `~/.aws/credentials`). +### PRR collection output + +[`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection) writes a +self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): + +``` +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) +``` + +- **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, + and carries the PRR-mandatory fields (`osc:project`, `osc:initiative`, + `osc:contract-number`, `osc:project_website`, `osc:project_description`, + `osc:variables`, `osc:missions`, `cf:parameter`, `processing:datetime`, `themes`, + `license`) plus a required `thumbnail` asset. +- **Item** — carries the `datacube` extension (`cube:dimensions` / `cube:variables` + read from the Zarr) and the `zarr-data` / `zarr-consolidated-metadata` assets. Asset + hrefs stay absolute (`s3://…`) since that is where the data lives; structural links + are relative so the folder is portable. + +The output targets the +[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html). +Fields that can't be defaulted (`osc_missions`, `osc_contract_number`, `thumbnail`) must +be supplied in the config — otherwise the command still generates the tree but warns that +it is not yet fully conformant. + ## Workflow config (YAML) ```yaml # Required @@ -126,4 +193,4 @@ links: | `contact` | No | List of contact objects with `name`, `organization`, and `links`. | | `links` | No | Additional OGC API record links (e.g. `related`, `describedby`). | -More templates and examples live in `dataset_config.yaml`, `workflow_config.yaml`, and `example-config/`. +More templates and examples live in `dataset.yaml`, `workflow.yaml`, and `example-config/`. diff --git a/docs/examples.md b/docs/examples.md index bf3a596..c1eac29 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,5 +1,5 @@ # Examples -- Templates: `dataset_config.yaml`, `workflow_config.yaml` +- Templates: `dataset.yaml`, `workflow.yaml` - Example configs: `examples/example-config/` - Notebooks on publishing: `examples/notebooks` diff --git a/docs/index.md b/docs/index.md index 843f4a5..657bb7c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,7 @@ - Build STAC collections and catalogs for Datasets and their corresponding variables automatically from the dataset metadata. - Generate STAC catalog and item for the product (Zarr store) and publish them to S3. - Build OGC API records for Workflows and Experiments from your configs. +- Generate a Project Results Repository (PRR) STAC collection as local files for submission to the ESA EarthCODE PRR endpoint. - Flexible publishling targets i.e production/staging/testing EarthCODE metadata repositories with GitHub automation. ```mermaid @@ -15,7 +16,7 @@ flowchart LR subgraph User A["Config files
(dataset.yaml, workflow.yaml)"] - B["deep-code CLI
(generate-config, publish)"] + B["deep-code CLI
(generate-config, publish, generate-prr-collection)"] end subgraph App["deep-code internals"] diff --git a/docs/python-api.md b/docs/python-api.md index 5254f61..2203e93 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -71,14 +71,14 @@ publisher.publish(write_to_file=False, mode="dataset") over individual artifacts. ```python -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator generator = OscDatasetStacGenerator( - dataset_id="my-dataset.zarr", collection_id="my-collection", workflow_id="my-workflow", workflow_title="My Workflow", license_type="CC-BY-4.0", + items_config=[ItemConfig(dataset_id="my-dataset.zarr", item_id="my-item")], osc_themes=["cryosphere"], osc_region="Global", osc_status="completed", @@ -124,3 +124,26 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the generated structure. + +### PRR collection generation + +Build a self-contained PRR (Project Results Repository) `Collection → Item → Assets` +tree as local files. The high-level helper reads the same dataset config as the CLI: + +```python +from deep_code.tools.prr import generate_prr_collection + +out_dir = generate_prr_collection("dataset.yaml", output_dir="./prr") +# ./prr/collection.json + ./prr//items/.json +``` + +Or drive the generator directly: + +```python +generator.save_prr_collection("./prr") # writes a self-contained tree +``` + +The Item includes the `datacube` extension; the Collection declares the OSC, Scientific, +Processing, Themes and CF extensions and the PRR-mandatory fields. See +[PRR collection output](configuration.md#prr-collection-output) and +[Generate a PRR collection](cli.md#generate-a-prr-collection). diff --git a/examples/notebooks/publish_to_EarthCODE.ipynb b/examples/notebooks/publish_to_EarthCODE.ipynb index eb8d520..2fec356 100644 --- a/examples/notebooks/publish_to_EarthCODE.ipynb +++ b/examples/notebooks/publish_to_EarthCODE.ipynb @@ -234,11 +234,11 @@ "team_store = new_data_store(\n", " \"s3\", \n", " root=S3_USER_STORAGE_BUCKET, \n", - " storage_options=dict(\n", - " anon=False, \n", - " key=S3_USER_STORAGE_KEY, \n", - " secret=S3_USER_STORAGE_SECRET\n", - " )\n", + " storage_options={\n", + " 'anon': False, \n", + " 'key': S3_USER_STORAGE_KEY, \n", + " 'secret': S3_USER_STORAGE_SECRET\n", + " }\n", ")" ] },