From e9e6344b03905c8e7721f7275a33b36491337268 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:12:29 +0200 Subject: [PATCH 01/20] generate prr collection --- deep_code/cli/main.py | 2 + deep_code/constants.py | 7 + .../utils/test_dataset_stac_generator.py | 290 ++++++++++++- deep_code/tools/new.py | 20 + deep_code/utils/dataset_stac_generator.py | 381 +++++++++++++++++- 5 files changed, 698 insertions(+), 2 deletions(-) diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index e267305..ac306f5 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -8,6 +8,7 @@ from deep_code.cli.generate_config import generate_config from deep_code.cli.lint import lint_dataset +from deep_code.cli.prr import generate_prr_collection_cmd from deep_code.cli.publish import publish @@ -20,6 +21,7 @@ def main(): main.add_command(publish) main.add_command(generate_config) main.add_command(lint_dataset) +main.add_command(generate_prr_collection_cmd) if __name__ == "__main__": main() diff --git a/deep_code/constants.py b/deep_code/constants.py index fa9e4b8..e1fee57 100644 --- a/deep_code/constants.py +++ b/deep_code/constants.py @@ -41,3 +41,10 @@ "https://stac-extensions.github.io/application/v0.1.0/schema.json" ) ZARR_MEDIA_TYPE = "application/vnd+zarr" +DATACUBE_SCHEMA_URI = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json" +PROCESSING_SCHEMA_URI = ( + "https://stac-extensions.github.io/processing/v1.2.0/schema.json" +) +SCIENTIFIC_SCHEMA_URI = ( + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" +) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 0b806cd..e981df6 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -3,17 +3,23 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. +import json +import os +import tempfile import unittest from datetime import datetime from unittest.mock import MagicMock, patch import numpy as np -from pystac import Catalog, Item +from pystac import Catalog, Collection, Item from xarray import DataArray, Dataset from deep_code.constants import ( + DATACUBE_SCHEMA_URI, DEEPESDL_COLLECTION_SELF_HREF, + OSC_SCHEMA_URI, OSC_THEME_SCHEME, + PROCESSING_SCHEMA_URI, PRODUCT_BASE_CATALOG_SELF_HREF, VARIABLE_BASE_CATALOG_SELF_HREF, ZARR_MEDIA_TYPE, @@ -763,3 +769,285 @@ def test_update_existing_variable_catalog(self, mock_open_ds): rels = [lnk["rel"] for lnk in result["links"]] self.assertIn("child", rels) self.assertIn("related", rels) # theme link + + +class TestPRRCollection(unittest.TestCase): + """Tests for the PRR-style Collection -> Item -> Assets generation.""" + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def setUp(self, mock_open_ds): + self.dataset = Dataset( + coords={ + "lon": ("lon", np.linspace(-20, 20, 4)), + "lat": ("lat", np.linspace(-10, 10, 3)), + "time": ( + "time", + [ + np.datetime64(datetime(2021, 1, 1), "ns"), + np.datetime64(datetime(2021, 1, 3), "ns"), + ], + ), + }, + data_vars={ + "sst": ( + ("time", "lat", "lon"), + np.random.rand(2, 3, 4), + {"units": "K", "long_name": "Sea surface temperature"}, + ), + "chl": ( + ("time", "lat", "lon"), + np.random.rand(2, 3, 4), + {"units": "mg m-3"}, + ), + # A CRS var that must be excluded from cube:variables and drive EPSG. + "spatial_ref": ((), 0, {"spatial_epsg": 3035}), + }, + attrs={"description": "PRR test cube"}, + ) + mock_open_ds.return_value = self.dataset + self.gen = OscDatasetStacGenerator( + dataset_id="test.zarr", + collection_id="prr-collection", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link="s3://bucket/test.zarr", + osc_status="ongoing", + osc_region="Global", + osc_themes=["oceans"], + osc_missions=["sentinel-3"], + documentation_link="https://example.org/doc", + ) + + # ---- helpers ---- + + def test_get_epsg_from_spatial_ref(self): + self.assertEqual(self.gen._get_epsg(), 3035) + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_get_epsg_default_4326(self, mock_open_ds): + ds = Dataset( + coords={ + "lon": ("lon", np.linspace(-1, 1, 2)), + "lat": ("lat", np.linspace(-1, 1, 2)), + "time": ("time", [np.datetime64(datetime(2020, 1, 1), "ns")]), + }, + data_vars={"v": (("time", "lat", "lon"), np.random.rand(1, 2, 2))}, + ) + mock_open_ds.return_value = ds + gen = OscDatasetStacGenerator( + dataset_id="t.zarr", + collection_id="c", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + ) + self.assertEqual(gen._get_epsg(), 4326) + + def test_get_cube_dimensions(self): + dims = self.gen._get_cube_dimensions() + self.assertEqual(set(dims), {"lon", "lat", "time"}) + self.assertEqual(dims["lon"]["type"], "spatial") + self.assertEqual(dims["lon"]["axis"], "x") + self.assertEqual(dims["lon"]["reference_system"], 3035) + self.assertEqual(dims["lon"]["extent"], [-20.0, 20.0]) + self.assertEqual(dims["lat"]["axis"], "y") + self.assertEqual(dims["lat"]["extent"], [-10.0, 10.0]) + self.assertEqual(dims["time"]["type"], "temporal") + self.assertEqual(len(dims["time"]["extent"]), 2) + + def test_get_cube_variables(self): + variables = self.gen._get_cube_variables() + # CRS variable must be excluded. + self.assertEqual(set(variables), {"sst", "chl"}) + self.assertEqual(variables["sst"]["type"], "data") + self.assertEqual(variables["sst"]["dimensions"], ["time", "lat", "lon"]) + self.assertEqual(variables["sst"]["unit"], "K") + self.assertEqual(variables["sst"]["description"], "Sea surface temperature") + # chl has a unit but no long_name/description. + self.assertEqual(variables["chl"]["unit"], "mg m-3") + self.assertNotIn("description", variables["chl"]) + + # ---- item ---- + + def test_build_prr_stac_item(self): + item = self.gen.build_prr_stac_item() + self.assertIsInstance(item, Item) + self.assertEqual(item.id, "prr-collection") + self.assertIn(DATACUBE_SCHEMA_URI, item.stac_extensions) + self.assertIn("cube:dimensions", item.properties) + self.assertIn("cube:variables", item.properties) + + # datetime is null; start/end are timezone-aware ISO strings. + self.assertIsNone(item.datetime) + self.assertTrue(item.properties["start_datetime"].endswith("+00:00")) + self.assertTrue(item.properties["end_datetime"].endswith("+00:00")) + + self.assertEqual(set(item.assets), {"zarr-data", "zarr-consolidated-metadata"}) + self.assertEqual(item.assets["zarr-data"].href, "s3://bucket/test.zarr") + self.assertEqual(item.assets["zarr-data"].media_type, ZARR_MEDIA_TYPE) + self.assertEqual( + item.assets["zarr-consolidated-metadata"].href, + "s3://bucket/test.zarr/.zmetadata", + ) + + # ---- collection ---- + + def test_build_prr_collection(self): + coll = self.gen.build_prr_collection() + self.assertIsInstance(coll, Collection) + self.assertEqual(coll.id, "prr-collection") + self.assertEqual(coll.license, "CC-BY-4.0") + + ef = coll.extra_fields + self.assertEqual(ef["osc:type"], "product") + self.assertEqual(ef["osc:status"], "ongoing") + self.assertEqual(ef["osc:region"], "Global") + self.assertCountEqual(ef["osc:variables"], ["sst", "chl"]) + self.assertEqual(ef["osc:missions"], ["sentinel-3"]) + self.assertEqual(ef["cf:parameter"], [{"name": "prr-collection"}]) + self.assertIn("processing:datetime", ef) + + # Extensions declared. + self.assertIn(OSC_SCHEMA_URI, coll.stac_extensions) + self.assertIn(PROCESSING_SCHEMA_URI, coll.stac_extensions) + + # Themes are plain dicts (not Theme objects). + themes = ef["themes"] + self.assertEqual(themes[0]["scheme"], OSC_THEME_SCHEME) + self.assertEqual(themes[0]["concepts"], [{"id": "oceans"}]) + self.assertIsInstance(themes[0], dict) + + # The single Item is attached as a child. + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertEqual(items[0].id, "prr-collection") + + def test_build_prr_collection_conformant_fields(self): + """A fully configured generator emits every PRR-required field.""" + from deep_code.constants import SCIENTIFIC_SCHEMA_URI + + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ): + gen = OscDatasetStacGenerator( + dataset_id="test.zarr", + collection_id="prr-collection", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link="s3://bucket/test.zarr", + osc_status="ongoing", + osc_region="Global", + osc_themes=["oceans"], + osc_missions=["sentinel-3"], + osc_project_description="A detailed project description.", + osc_project_website="https://project.example.org", + osc_contract_number="4000114410/15/NL/BW", + thumbnail="https://example.org/logo.jpeg", + sci_doi="10.1000/xyz123", + ) + coll = gen.build_prr_collection() + ef = coll.extra_fields + + # Scientific extension declared alongside the others. + self.assertIn(SCIENTIFIC_SCHEMA_URI, coll.stac_extensions) + # Required PRR project fields present. + self.assertEqual(ef["osc:initiative"], "earthcode") + self.assertEqual(ef["osc:project_website"], "https://project.example.org") + self.assertEqual( + ef["osc:project_description"], "A detailed project description." + ) + self.assertEqual(ef["osc:contract-number"], "4000114410/15/NL/BW") + self.assertEqual(ef["sci:doi"], "10.1000/xyz123") + # Thumbnail asset with correct role and guessed media type. + self.assertIn("thumbnail", coll.assets) + thumb = coll.assets["thumbnail"] + self.assertEqual(thumb.roles, ["thumbnail"]) + self.assertEqual(thumb.media_type, "image/jpeg") + + def test_build_prr_collection_fallbacks(self): + """project_website/description fall back to doc link / description.""" + coll = self.gen.build_prr_collection() + ef = coll.extra_fields + # Default initiative. + self.assertEqual(ef["osc:initiative"], "earthcode") + # Fallbacks: website -> documentation_link, description -> dataset description. + self.assertEqual(ef["osc:project_website"], "https://example.org/doc") + self.assertEqual(ef["osc:project_description"], "PRR test cube") + # No thumbnail / contract number configured -> absent. + self.assertNotIn("thumbnail", coll.assets) + self.assertNotIn("osc:contract-number", ef) + + def test_thumbnail_media_type_guess(self): + self.gen.thumbnail = "https://x/logo.png" + self.assertEqual(self.gen._thumbnail_media_type(), "image/png") + self.gen.thumbnail = "https://x/logo.JPG" + self.assertEqual(self.gen._thumbnail_media_type(), "image/jpeg") + self.gen.thumbnail = "https://x/logo.webp" + self.assertEqual(self.gen._thumbnail_media_type(), "image/webp") + self.gen.thumbnail_media_type = "image/tiff" + self.assertEqual(self.gen._thumbnail_media_type(), "image/tiff") + + def test_build_prr_collection_cf_params_override(self): + self.gen.cf_params = [{"name": "sst", "units": "K"}] + coll = self.gen.build_prr_collection() + self.assertEqual( + coll.extra_fields["cf:parameter"], [{"name": "sst", "units": "K"}] + ) + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_prr_collection_no_themes(self, mock_open_ds): + mock_open_ds.return_value = self.dataset + gen = OscDatasetStacGenerator( + dataset_id="test.zarr", + collection_id="prr-collection", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link="s3://bucket/test.zarr", + ) + coll = gen.build_prr_collection() + self.assertNotIn("themes", coll.extra_fields) + + # ---- save (local self-contained tree) ---- + + def test_save_prr_collection_writes_tree(self): + with tempfile.TemporaryDirectory() as tmp: + out = self.gen.save_prr_collection(tmp) + self.assertEqual(out, tmp) + + collection_path = os.path.join(tmp, "collection.json") + item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + self.assertTrue(os.path.isfile(collection_path)) + self.assertTrue(os.path.isfile(item_path)) + + # Files are plain-JSON serialisable (no leftover Python objects). + with open(collection_path) as f: + coll_dict = json.load(f) + with open(item_path) as f: + item_dict = json.load(f) + + self.assertEqual(coll_dict["type"], "Collection") + self.assertEqual(item_dict["type"], "Feature") + + # Structural links are relative; the Item link points at the child. + item_link = next( + lnk for lnk in coll_dict["links"] if lnk["rel"] == "item" + ) + self.assertFalse(item_link["href"].startswith("s3://")) + self.assertTrue(item_link["href"].endswith(".json")) + + # Asset hrefs stay absolute (the data lives on S3). + self.assertEqual( + item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" + ) + + def test_save_prr_collection_readable_by_pystac(self): + with tempfile.TemporaryDirectory() as tmp: + self.gen.save_prr_collection(tmp) + coll = Collection.from_file(os.path.join(tmp, "collection.json")) + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 8fb49f3..fc53ed0 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -76,6 +76,22 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "cf_parameter": [{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}], } + # Fields used only by `deep-code generate-prr-collection` to build a + # PRR (Product Readiness Review) collection that conforms to + # https://eoresults.esa.int/prr_collection_specifications.html + prr = { + "prr_output_dir": "[OPTIONAL: local dir for the PRR collection tree — defaults to prr/{collection_id}]", + "osc_initiative": "[OPTIONAL: PRR initiative — 'earthcode' or 'apex' (default: earthcode)]", + "osc_contract_number": "[PRR-REQUIRED: ESA contract identifier, e.g. 4000114410/15/NL/BW]", + "osc_project_website": "[PRR-REQUIRED: project website URL — falls back to osc_project_url / documentation_link]", + "osc_project_description": "[PRR-REQUIRED: multi-line project description — falls back to description]", + "osc_missions": ["[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]"], + "thumbnail": "[PRR-REQUIRED: URL to a collection thumbnail image (jpeg/png/webp)]", + "thumbnail_media_type": "[OPTIONAL: thumbnail MIME type — guessed from the URL suffix if omitted]", + "sci_doi": "[OPTIONAL: dataset DOI, e.g. 10.1000/xyz123 (not a DOI link)]", + "sci_citation": "[OPTIONAL: human-readable citation for the dataset]", + } + stac_catalog_comment = ( "\n# stac_catalog_s3_root: deep-code writes the following files to this S3 root:\n" "# {stac_catalog_s3_root}/catalog.json (STAC Catalog root)\n" @@ -94,4 +110,8 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: f.write(yaml.dump(required, sort_keys=False, width=1000, default_flow_style=False)) f.write("\n# --- OPTIONAL fields ---\n") f.write(yaml.dump(optional, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + "\n# --- PRR fields (for `deep-code generate-prr-collection`) ---\n" + ) + f.write(yaml.dump(prr, sort_keys=False, width=1000, default_flow_style=False)) f.write(stac_catalog_comment) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 716d509..57b1ea9 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -8,12 +8,25 @@ from datetime import datetime, timezone import pandas as pd -from pystac import Catalog, Collection, Extent, Item, Asset, Link, SpatialExtent, TemporalExtent +from pystac import ( + Asset, + Catalog, + CatalogType, + Collection, + Extent, + Item, + Link, + SpatialExtent, + TemporalExtent, +) from deep_code.constants import ( CONTACTS_SCHEMA_URI, + DATACUBE_SCHEMA_URI, OSC_SCHEMA_URI, OSC_THEME_SCHEME, + PROCESSING_SCHEMA_URI, + SCIENTIFIC_SCHEMA_URI, THEMES_SCHEMA_URI, ZARR_MEDIA_TYPE, ) @@ -57,6 +70,14 @@ def __init__( osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, + osc_initiative: str = "earthcode", + osc_contract_number: str | None = None, + osc_project_website: str | None = None, + osc_project_description: str | None = None, + thumbnail: str | None = None, + thumbnail_media_type: str | None = None, + sci_doi: str | None = None, + sci_citation: str | None = None, ): if " " in collection_id: raise ValueError( @@ -80,6 +101,15 @@ def __init__( self.cf_params = cf_params or {} self.visualisation_link = visualisation_link self.description = description + # PRR-specific project metadata (see PRR collection specification). + self.osc_initiative = osc_initiative or "earthcode" + self.osc_contract_number = osc_contract_number + self.osc_project_website = osc_project_website + self.osc_project_description = osc_project_description + self.thumbnail = thumbnail + self.thumbnail_media_type = thumbnail_media_type + self.sci_doi = sci_doi + self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) self.dataset = open_dataset(dataset_id=dataset_id, logger=self.logger) self.variables_metadata = self.get_variables_metadata() @@ -657,6 +687,355 @@ def build_zarr_stac_catalog_file_dict( item_href: item.to_dict(transform_hrefs=False), } + # ------------------------------------------------------------------ # + # PRR (Product Readiness Review) style output # + # # + # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # + # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # + # the plain catalog.json/item.json under ``{root}/prr/``. # + # ------------------------------------------------------------------ # + + def _get_epsg(self) -> int: + """Best-effort EPSG code for the dataset, defaulting to 4326. + + Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or + ``spatial_ref`` variable when present; otherwise assumes geographic + WGS 84 (EPSG:4326). + """ + for var_name in ("spatial_ref", "crs"): + if var_name in self.dataset.variables: + attrs = self.dataset[var_name].attrs + for key in ("spatial_epsg", "epsg", "EPSG"): + if key in attrs: + try: + return int(attrs[key]) + except (TypeError, ValueError): + pass + return 4326 + + def _get_cube_dimensions(self) -> dict[str, dict]: + """Build the ``cube:dimensions`` object from the dataset coordinates. + + Follows the datacube STAC extension: horizontal spatial dimensions are + classified by axis (x/y), the ``time`` coordinate becomes a temporal + dimension, and any remaining index coordinate is emitted as an + additional dimension. + """ + ds = self.dataset + epsg = self._get_epsg() + x_names = {"lon", "longitude", "x"} + y_names = {"lat", "latitude", "y"} + dimensions: dict[str, dict] = {} + for name, coord in ds.coords.items(): + if name not in ds.dims: + # Skip non-dimension coordinates (e.g. scalar or auxiliary coords). + continue + lname = str(name).lower() + if lname in x_names: + dimensions[name] = { + "type": "spatial", + "axis": "x", + "extent": [float(coord.min()), float(coord.max())], + "reference_system": epsg, + } + elif lname in y_names: + dimensions[name] = { + "type": "spatial", + "axis": "y", + "extent": [float(coord.min()), float(coord.max())], + "reference_system": epsg, + } + elif lname == "time": + time_min = pd.to_datetime(coord.min().values).to_pydatetime() + time_max = pd.to_datetime(coord.max().values).to_pydatetime() + dimensions[name] = { + "type": "temporal", + "extent": [time_min.isoformat(), time_max.isoformat()], + } + else: + try: + dimensions[name] = { + "type": lname, + "extent": [float(coord.min()), float(coord.max())], + } + except (TypeError, ValueError): + dimensions[name] = { + "type": lname, + "values": [str(v) for v in coord.values.tolist()], + } + return dimensions + + def _get_cube_variables(self) -> dict[str, dict]: + """Build the ``cube:variables`` object from the dataset data variables.""" + skip = {"crs", "spatial_ref"} + variables: dict[str, dict] = {} + for name, var in self.dataset.data_vars.items(): + if name in skip: + continue + entry: dict = { + "type": "data", + "dimensions": [str(d) for d in var.dims], + } + unit = var.attrs.get("units") + if unit: + entry["unit"] = unit + description = var.attrs.get("long_name") or var.attrs.get("description") + if description: + entry["description"] = description + variables[name] = entry + return variables + + def build_prr_stac_item(self) -> Item: + """Build the single datacube Item for the PRR collection. + + One Item covers the full spatiotemporal extent of the Zarr store. It + carries the datacube extension (``cube:dimensions`` / ``cube:variables``) + and two assets pointing at the Zarr store. Structural links + (root/parent/collection/self) are left for :meth:`save_prr_collection` + to fill in via ``Collection.add_item`` + ``normalize_hrefs``. + """ + self.logger.info( + f"Building PRR STAC Item for collection '{self.collection_id}'." + ) + spatial_extent = self._get_spatial_extent() + temporal_extent = self._get_temporal_extent() + general_metadata = self._get_general_metadata() + + bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] + lon_min, lat_min, lon_max, lat_max = bbox + geometry = { + "type": "Polygon", + "coordinates": [[ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ]], + } + + start_dt, end_dt = temporal_extent.intervals[0] + if start_dt is not None and start_dt.tzinfo is None: + start_dt = start_dt.replace(tzinfo=timezone.utc) + if end_dt is not None and end_dt.tzinfo is None: + end_dt = end_dt.replace(tzinfo=timezone.utc) + + now_iso = datetime.now(timezone.utc).isoformat() + + item = Item( + id=self.collection_id, + geometry=geometry, + bbox=bbox, + datetime=None, + properties={ + "start_datetime": start_dt.isoformat() if start_dt else None, + "end_datetime": end_dt.isoformat() if end_dt else None, + "description": general_metadata.get("description", ""), + "created": now_iso, + "updated": now_iso, + "cube:dimensions": self._get_cube_dimensions(), + "cube:variables": self._get_cube_variables(), + }, + ) + item.stac_extensions.append(DATACUBE_SCHEMA_URI) + # Asset hrefs stay absolute (the Zarr lives on S3); only the structural + # links become relative when the tree is normalised locally. + item.add_asset("zarr-data", Asset( + href=self.access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + )) + item.add_asset("zarr-consolidated-metadata", Asset( + href=f"{self.access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + )) + self.logger.info(f"PRR STAC Item built for '{self.collection_id}'.") + return item + + def build_prr_collection(self) -> Collection: + """Build the PRR parent Collection with its single datacube Item attached. + + The Collection carries OSC extension fields (``osc:type``, ``osc:status``, + ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list + and ``processing:datetime`` — aligning with the ESA EarthCODE PRR + endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a + subsequent ``normalize_hrefs`` produces a self-contained tree. + """ + spatial_extent = self._get_spatial_extent() + temporal_extent = self._get_temporal_extent() + variables = self.get_variable_ids() + general_metadata = self._get_general_metadata() + + collection = Collection( + id=self.collection_id, + description=general_metadata.get("description", "No description provided."), + extent=Extent(spatial=spatial_extent, temporal=temporal_extent), + license=self.license_type, + title=self.collection_id, + ) + + osc_extension = OscExtension.add_to(collection) + osc_extension.osc_project = self.osc_project + osc_extension.osc_type = "product" + osc_extension.osc_status = self.osc_status + osc_extension.osc_region = self.osc_region + osc_extension.osc_variables = variables + osc_extension.osc_missions = self.osc_missions + osc_extension.cf_parameter = self.cf_params or [{"name": self.collection_id}] + + now_iso = datetime.now(timezone.utc).isoformat() + collection.extra_fields["created"] = now_iso + collection.extra_fields["updated"] = now_iso + + # Processing extension — the PRR reference collection declares this and + # exposes 'processing:datetime' (the ingestion/generation timestamp). + if PROCESSING_SCHEMA_URI not in collection.stac_extensions: + collection.stac_extensions.append(PROCESSING_SCHEMA_URI) + collection.extra_fields["processing:datetime"] = now_iso + + if self.osc_themes: + # Plain dicts (not Theme objects) so the collection serialises with a + # plain json encoder as well as pystac's own writer. + collection.extra_fields["themes"] = [ + { + "concepts": [{"id": theme} for theme in self.osc_themes], + "scheme": OSC_THEME_SCHEME, + } + ] + + # ---- Required PRR project-level metadata ---- + # Scientific extension is REQUIRED in the PRR profile's extension list; + # 'sci:*' values themselves remain optional. + if SCIENTIFIC_SCHEMA_URI not in collection.stac_extensions: + collection.stac_extensions.append(SCIENTIFIC_SCHEMA_URI) + + collection.extra_fields["osc:initiative"] = self.osc_initiative + project_website = ( + self.osc_project_website + or self.osc_project_url + or self.documentation_link + ) + if project_website: + collection.extra_fields["osc:project_website"] = project_website + project_description = ( + self.osc_project_description + or self.description + or general_metadata.get("description") + ) + if project_description: + collection.extra_fields["osc:project_description"] = project_description + if self.osc_contract_number: + collection.extra_fields["osc:contract-number"] = self.osc_contract_number + + # Optional scientific metadata. + if self.sci_doi: + collection.extra_fields["sci:doi"] = self.sci_doi + if self.sci_citation: + collection.extra_fields["sci:citation"] = self.sci_citation + + # Thumbnail asset (REQUIRED by the PRR spec: an asset named 'thumbnail' + # with the 'thumbnail' role). + if self.thumbnail: + collection.add_asset("thumbnail", Asset( + href=self.thumbnail, + media_type=self._thumbnail_media_type(), + title="Collection Thumbnail", + roles=["thumbnail"], + )) + + if self.documentation_link: + collection.add_link( + Link(rel="via", target=self.documentation_link, title="Documentation") + ) + if self.visualisation_link: + collection.add_link(Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + )) + + try: + osc_extension.validate_extension() + except ValueError as e: + raise ValueError(f"OSC Extension validation failed: {e}") + + self._warn_missing_prr_fields(collection, variables) + collection.add_item(self.build_prr_stac_item()) + return collection + + def _thumbnail_media_type(self) -> str: + """Return the thumbnail media type, guessed from the href suffix.""" + if self.thumbnail_media_type: + return self.thumbnail_media_type + href = (self.thumbnail or "").lower() + if href.endswith((".jpg", ".jpeg")): + return "image/jpeg" + if href.endswith(".webp"): + return "image/webp" + return "image/png" + + def _warn_missing_prr_fields( + self, collection: Collection, variables: list[str] + ) -> None: + """Log a warning for PRR-required fields that are absent, so the caller + knows the output is not yet spec-conformant.""" + ef = collection.extra_fields + missing = [] + if "thumbnail" not in collection.assets: + missing.append("thumbnail asset") + if not ef.get("osc:contract-number"): + missing.append("osc:contract-number") + if not ef.get("osc:project_website"): + missing.append("osc:project_website") + if not ef.get("osc:project_description"): + missing.append("osc:project_description") + if not ef.get("themes"): + missing.append("themes") + if not variables: + missing.append("osc:variables") + if not self.osc_missions: + missing.append("osc:missions") + if missing: + self.logger.warning( + "PRR collection is missing required field(s): %s. " + "The collection will not fully conform to the PRR specification " + "until these are provided in the dataset config.", + ", ".join(missing), + ) + + def save_prr_collection(self, output_dir: str) -> str: + """Write the PRR ``Collection -> Item -> Assets`` tree to a local folder. + + Produces a self-contained STAC tree with relative structural links, + ready to inspect or submit to the ESA EarthCODE PRR endpoint:: + + {output_dir}/ + ├── collection.json # STAC Collection (root) + └── {collection_id}/ + └── {collection_id}.json # datacube Item (whole Zarr) + + Zarr asset hrefs remain absolute (``s3://…``) since that is where the + data lives. + + Args: + output_dir: Local directory to write the collection tree into. + + Returns: + The ``output_dir`` that was written. + """ + self.logger.info( + f"Writing PRR STAC collection for '{self.collection_id}' to " + f"'{output_dir}'." + ) + collection = self.build_prr_collection() + collection.normalize_hrefs(output_dir) + collection.save(catalog_type=CatalogType.SELF_CONTAINED) + self.logger.info(f"PRR STAC collection written to '{output_dir}'.") + return output_dir + def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | None = None) -> Collection: """Build an OSC STAC Collection for the dataset. From d9a765ddad6a7ce8ab8be1cf0c78356b60acd324 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:16:28 +0200 Subject: [PATCH 02/20] updated docs --- deep_code/tools/new.py | 2 +- docs/cli.md | 37 ++++++++++++++++++++++++++ docs/configuration.md | 59 ++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 3 ++- docs/python-api.md | 23 ++++++++++++++++ 5 files changed, 122 insertions(+), 2 deletions(-) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index fc53ed0..d910db0 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -77,7 +77,7 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: } # Fields used only by `deep-code generate-prr-collection` to build a - # PRR (Product Readiness Review) collection that conforms to + # PRR (Project Results Repository) collection that conforms to # https://eoresults.esa.int/prr_collection_specifications.html prr = { "prr_output_dir": "[OPTIONAL: local dir for the PRR collection tree — defaults to prr/{collection_id}]", diff --git a/docs/cli.md b/docs/cli.md index 734927a..a269479 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -45,3 +45,40 @@ Options: 3. Forks/clones the target metadata repo (production, staging, or testing), commits generated JSON, and opens a pull request on your behalf. The pull request description includes a "Generated with deep-code" attribution note. + +## Generate a PRR collection + +Generate a **Project Results Repository (PRR)** STAC collection as local files, ready +to submit to the [ESA EarthCODE PRR endpoint](https://eoresults.esa.int): + +```bash +deep-code generate-prr-collection dataset.yaml # writes to prr/ +deep-code generate-prr-collection dataset.yaml -o ./prr # custom output directory +``` + +This reuses the **same dataset config** as `publish`, but writes only local files and +needs no GitHub credentials or S3 write access (it only reads the Zarr store). It +produces a self-contained `Collection → Item → Assets` tree: + +``` +prr// +├── collection.json # STAC Collection (root, relative links) +└── / + └── .json # datacube Item covering the whole Zarr store +``` + +- The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` + extracted from the Zarr) plus `zarr-data` and `zarr-consolidated-metadata` assets. +- The **Collection** carries the OSC, Scientific, Processing, Themes and CF extensions + and the PRR-mandatory fields. + +The output conforms to the +[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html) +when the PRR fields are set in the config. If any required field is missing, the command +still runs but logs a warning listing what is needed for full conformance. See +[PRR collection fields](configuration.md#prr-collection-fields). + +Options: + +- `--output-dir/-o`: directory to write the tree into. Defaults to `prr_output_dir` + from the config, then `prr/`. diff --git a/docs/configuration.md b/docs/configuration.md index d8d90d6..881c572 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,6 +34,16 @@ access_link: s3://bucket/your-dataset.zarr # defaults to s3://deep-esdl-public cf_parameter: - name: sea_surface_temperature units: kelvin + +# PRR fields (only used by `deep-code generate-prr-collection`) +osc_initiative: earthcode # earthcode | apex (default: earthcode) +osc_missions: [sentinel-3] +osc_contract_number: 4000114410/15/NL/BW +osc_project_website: https://project.example.org +osc_project_description: A detailed multi-line description of the project. +thumbnail: https://example.org/thumbnail.jpeg +sci_doi: 10.1000/xyz123 +prr_output_dir: ./prr/your-collection ``` ### Field reference @@ -54,6 +64,27 @@ cf_parameter: | `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | | `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +> The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). + +### PRR collection fields + +| Field | Required | Description | +|---|---|---| +| `osc_initiative` | No | PRR initiative: `earthcode` or `apex` (default: `earthcode`). | +| `osc_missions` | PRR-required | List of satellite mission name(s), e.g. `[sentinel-3]`. | +| `osc_contract_number` | PRR-required | ESA contract identifier, e.g. `4000114410/15/NL/BW`. | +| `osc_project_website` | PRR-required | Project website URL. Falls back to `osc_project_url`, then `documentation_link`. | +| `osc_project_description` | PRR-required | Multi-line project description. Falls back to `description`. | +| `thumbnail` | PRR-required | URL to a collection thumbnail image (jpeg/png/webp). Added as an asset named `thumbnail` with role `thumbnail`. | +| `thumbnail_media_type` | No | Thumbnail MIME type. Guessed from the URL suffix when omitted. | +| `sci_doi` | No | Dataset DOI, e.g. `10.1000/xyz123` (a DOI name, not a link). | +| `sci_citation` | No | Human-readable citation for the dataset. | +| `prr_output_dir` | No | Local directory for the PRR collection tree. Defaults to `prr/{collection_id}`. | + +`themes` for a PRR collection must be drawn from the allowed set: +`atmosphere`, `cryosphere`, `land`, `magnetosphere-ionosphere`, `oceans`, `solid-earth` +(configured via `osc_themes`). See [PRR collection output](#prr-collection-output). + ### STAC Catalog on S3 `stac_catalog_s3_root` is required. deep-code writes a two-file STAC hierarchy to S3 alongside the data: @@ -79,6 +110,34 @@ S3 credentials for writing the STAC catalog are resolved in this order: then `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`, then the boto3 default chain (IAM role, `~/.aws/credentials`). +### PRR collection output + +[`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection) writes a +self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): + +``` +prr/your-collection/ +├── collection.json # STAC Collection (root, relative links) +└── your-collection/ + └── your-collection.json # datacube Item covering the full Zarr store +``` + +- **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, + and carries the PRR-mandatory fields (`osc:project`, `osc:initiative`, + `osc:contract-number`, `osc:project_website`, `osc:project_description`, + `osc:variables`, `osc:missions`, `cf:parameter`, `processing:datetime`, `themes`, + `license`) plus a required `thumbnail` asset. +- **Item** — carries the `datacube` extension (`cube:dimensions` / `cube:variables` + read from the Zarr) and the `zarr-data` / `zarr-consolidated-metadata` assets. Asset + hrefs stay absolute (`s3://…`) since that is where the data lives; structural links + are relative so the folder is portable. + +The output targets the +[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html). +Fields that can't be defaulted (`osc_missions`, `osc_contract_number`, `thumbnail`) must +be supplied in the config — otherwise the command still generates the tree but warns that +it is not yet fully conformant. + ## Workflow config (YAML) ```yaml # Required diff --git a/docs/index.md b/docs/index.md index 843f4a5..657bb7c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,7 @@ - Build STAC collections and catalogs for Datasets and their corresponding variables automatically from the dataset metadata. - Generate STAC catalog and item for the product (Zarr store) and publish them to S3. - Build OGC API records for Workflows and Experiments from your configs. +- Generate a Project Results Repository (PRR) STAC collection as local files for submission to the ESA EarthCODE PRR endpoint. - Flexible publishling targets i.e production/staging/testing EarthCODE metadata repositories with GitHub automation. ```mermaid @@ -15,7 +16,7 @@ flowchart LR subgraph User A["Config files
(dataset.yaml, workflow.yaml)"] - B["deep-code CLI
(generate-config, publish)"] + B["deep-code CLI
(generate-config, publish, generate-prr-collection)"] end subgraph App["deep-code internals"] diff --git a/docs/python-api.md b/docs/python-api.md index 5254f61..aa9d0ef 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -124,3 +124,26 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the generated structure. + +### PRR collection generation + +Build a self-contained PRR (Project Results Repository) `Collection → Item → Assets` +tree as local files. The high-level helper reads the same dataset config as the CLI: + +```python +from deep_code.tools.prr import generate_prr_collection + +out_dir = generate_prr_collection("dataset.yaml", output_dir="./prr") +# ./prr/collection.json + ./prr//.json +``` + +Or drive the generator directly: + +```python +generator.save_prr_collection("./prr") # writes a self-contained tree +``` + +The Item includes the `datacube` extension; the Collection declares the OSC, Scientific, +Processing, Themes and CF extensions and the PRR-mandatory fields. See +[PRR collection output](configuration.md#prr-collection-output) and +[Generate a PRR collection](cli.md#generate-a-prr-collection). From 3ac3d71d4904042628db55b636fc7b729b165ef8 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:16:52 +0200 Subject: [PATCH 03/20] refactor --- deep_code/utils/dataset_stac_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 57b1ea9..6deaa3c 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -688,7 +688,7 @@ def build_zarr_stac_catalog_file_dict( } # ------------------------------------------------------------------ # - # PRR (Product Readiness Review) style output # + # PRR (Project Results Repository) style output # # # # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # From ff854a6a7af808e5258bed8187256ff4e967a7a0 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:17:09 +0200 Subject: [PATCH 04/20] update CHANGES.md --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 2c2b89c..5c594e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -90,3 +90,7 @@ - Generated project collection now includes required STAC extensions (`osc`, `themes`, `contacts`) and OSC-mandatory fields (`osc:type`, `osc:status`, `themes`, `contacts`) to pass OSC catalog validation. - Added optional `osc_project_url` field to the dataset config; used as the `via` link in the project collection. Falls back to `documentation_link` if omitted; defaults to the existing DeepESDL project collection when neither is provided. - `dataset_status` now defaults to `"ongoing"` when not specified in the dataset config. +- Added a new CLI command `generate-prr-collection` that writes a Project Results Repository (PRR) STAC collection as local files, ready for submission to the ESA EarthCODE PRR endpoint. It reuses the dataset config and needs no GitHub credentials or S3 write access. See [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html). +- The PRR command produces a self-contained `Collection → Item → Assets` tree: the Item carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) with `zarr-data` and `zarr-consolidated-metadata` assets, and the Collection declares the OSC, Scientific, Processing, Themes and CF extensions. +- Added PRR-specific dataset config fields: `osc_initiative` (default `earthcode`), `osc_missions`, `osc_contract_number`, `osc_project_website`, `osc_project_description`, `thumbnail`, `thumbnail_media_type`, `sci_doi`, `sci_citation`, and `prr_output_dir`. Missing PRR-required fields produce a warning rather than a failure. +- `generate-config` templates now document the PRR fields under a dedicated section. From b0a199e5fdf8cf32f09d26bfc1ac092790507ab5 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 5 Aug 2026 17:28:26 +0200 Subject: [PATCH 05/20] cli for generate prr collection --- deep_code/cli/prr.py | 34 +++++++++++++++++ deep_code/tools/prr.py | 86 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 deep_code/cli/prr.py create mode 100644 deep_code/tools/prr.py diff --git a/deep_code/cli/prr.py b/deep_code/cli/prr.py new file mode 100644 index 0000000..25e0ac5 --- /dev/null +++ b/deep_code/cli/prr.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 by Brockmann Consult GmbH +# Permissions are hereby granted under the terms of the MIT License: +# https://opensource.org/licenses/MIT. + +import click + +from deep_code.tools.prr import generate_prr_collection + + +@click.command(name="generate-prr-collection") +@click.argument("dataset_config", type=click.Path(exists=True)) +@click.option( + "--output-dir", + "-o", + type=click.Path(file_okay=False), + default=None, + help=( + "Directory to write the PRR collection tree into " + "(default: 'prr_output_dir' from the config, else 'prr/')." + ), +) +def generate_prr_collection_cmd(dataset_config, output_dir): + """Generate a PRR-style STAC Collection as local files. + + Reads a dataset config (the same YAML used by 'publish') and writes a + self-contained Collection -> Item -> Assets tree for submission to the + ESA EarthCODE Project Results Repository. + + Example: + deep-code generate-prr-collection dataset-config.yaml -o ./prr + """ + out_dir = generate_prr_collection(dataset_config, output_dir=output_dir) + click.echo(f"PRR collection written to: {out_dir}") diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py new file mode 100644 index 0000000..9bfddac --- /dev/null +++ b/deep_code/tools/prr.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 by Brockmann Consult GmbH +# Permissions are hereby granted under the terms of the MIT License: +# https://opensource.org/licenses/MIT. + +"""Generate a PRR-style STAC Collection -> Item -> Assets tree as local files. + +The PRR (Project Results Repository) collection is a self-contained STAC tree +suitable for submission to the ESA EarthCODE PRR endpoint. It reuses the same +dataset config as the ``publish`` command but writes only local files and does +not require GitHub credentials or S3 access beyond reading the Zarr store. +""" + +import logging + +import fsspec +import yaml + +from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator + +logger = logging.getLogger(__name__) + + +def generate_prr_collection( + dataset_config_path: str, output_dir: str | None = None +) -> str: + """Build and write a PRR STAC collection from a dataset config. + + Args: + dataset_config_path: Path to the dataset config YAML (the same file used + by ``deep-code publish``). + output_dir: Directory to write the collection tree into. Falls back to + ``prr_output_dir`` in the config, then ``prr/``. + + Returns: + The output directory the collection tree was written to. + """ + with fsspec.open(dataset_config_path, "r") as file: + config = yaml.safe_load(file) or {} + + dataset_id = config.get("dataset_id") + collection_id = config.get("collection_id") + license_type = config.get("license_type") + + if not dataset_id or not collection_id: + raise ValueError( + "Both 'dataset_id' and 'collection_id' are required in the dataset config." + ) + if not license_type: + raise ValueError( + "license_type is required in the dataset config. " + "Provide an SPDX identifier (e.g. 'CC-BY-4.0', 'MIT', 'proprietary')." + ) + + logger.info(f"Generating PRR STAC collection for '{collection_id}'.") + generator = OscDatasetStacGenerator( + dataset_id=dataset_id, + collection_id=collection_id, + workflow_id=config.get("workflow_id") or "", + workflow_title=config.get("workflow_title") or "", + license_type=license_type, + documentation_link=config.get("documentation_link"), + access_link=config.get("access_link"), + osc_status=config.get("dataset_status") or "ongoing", + osc_region=config.get("osc_region") or "Global", + osc_themes=config.get("osc_themes"), + osc_missions=config.get("osc_missions"), + cf_params=config.get("cf_parameter"), + visualisation_link=config.get("visualisation_link"), + description=config.get("description"), + **({"osc_project": config["osc_project"]} if config.get("osc_project") else {}), + osc_project_url=config.get("osc_project_url"), + # PRR-specific project metadata. + osc_initiative=config.get("osc_initiative") or "earthcode", + osc_contract_number=config.get("osc_contract_number"), + osc_project_website=config.get("osc_project_website"), + osc_project_description=config.get("osc_project_description"), + thumbnail=config.get("thumbnail") or config.get("thumbnail_link"), + thumbnail_media_type=config.get("thumbnail_media_type"), + sci_doi=config.get("sci_doi"), + sci_citation=config.get("sci_citation"), + ) + + out_dir = output_dir or config.get("prr_output_dir") or f"prr/{collection_id}" + generator.save_prr_collection(out_dir) + return out_dir From f5d491e740ac052ade4671517bda2043ec6f2a59 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 5 Aug 2026 17:49:27 +0200 Subject: [PATCH 06/20] pin ruff for now --- .github/workflows/unittest-workflow.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unittest-workflow.yaml b/.github/workflows/unittest-workflow.yaml index d83721a..1e143cd 100644 --- a/.github/workflows/unittest-workflow.yaml +++ b/.github/workflows/unittest-workflow.yaml @@ -25,7 +25,8 @@ jobs: - name: Lint with ruff run: | - pip install ruff + # TODO upgrade to latest later + pip install ruff==0.9.10 ruff check - name: Run unit tests From 99812085b2f996ecc614b26b11704c52687ca882 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Tue, 11 Aug 2026 16:38:31 +0200 Subject: [PATCH 07/20] check changes --- deep_code/tests/tools/test_publish.py | 6 +- .../utils/test_dataset_stac_generator.py | 47 +- deep_code/tools/new.py | 11 +- deep_code/tools/prr.py | 30 +- deep_code/tools/publish.py | 42 +- deep_code/utils/dataset_stac_generator.py | 545 ++++++++++++------ docs/cli.md | 5 +- docs/configuration.md | 42 +- docs/python-api.md | 8 +- 9 files changed, 510 insertions(+), 226 deletions(-) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 6ccc8eb..4df212e 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -295,7 +295,9 @@ def test_publish_writes_zarr_stac_to_s3_when_configured( mock_generator = MagicMock() mock_generator.build_zarr_stac_catalog_file_dict.return_value = { "s3://test-bucket/stac/catalog.json": {"type": "Catalog"}, - "s3://test-bucket/stac/test-collection/item.json": {"type": "Feature"}, + "s3://test-bucket/stac/test-collection/items/test-collection.json": { + "type": "Feature" + }, } # Simulate what publish_dataset() normally does: store the generator self.publisher._last_generator = mock_generator @@ -401,7 +403,7 @@ def test_publish_dataset_raises_when_no_dataset_config(self): def test_publish_dataset_raises_when_ids_missing(self): self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""} - with pytest.raises(ValueError, match="Dataset ID or Collection ID missing"): + with pytest.raises(ValueError, match="item configuration"): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_license_missing(self): diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index e981df6..5a20808 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -24,7 +24,11 @@ VARIABLE_BASE_CATALOG_SELF_HREF, ZARR_MEDIA_TYPE, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator, Theme +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + Theme, +) class TestOSCProductSTACGenerator(unittest.TestCase): @@ -456,7 +460,7 @@ def test_build_zarr_stac_item_structure(self): # Self href self.assertEqual( item.self_href, - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json", + "s3://test-bucket/stac/my-collection/items/mock-collection-id.json", ) # Required link rels @@ -490,7 +494,7 @@ def test_build_zarr_stac_catalog_file_dict_keys(self): catalog_path = "s3://test-bucket/stac/my-collection/catalog.json" item_path = ( - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" ) self.assertIn(catalog_path, file_dict) self.assertIn(item_path, file_dict) @@ -506,7 +510,7 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertEqual(catalog_dict["id"], "mock-collection-id-stac-catalog") item_dict = file_dict[ - "s3://test-bucket/stac/my-collection/mock-collection-id/item.json" + "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" ] self.assertEqual(item_dict["type"], "Feature") self.assertEqual(item_dict["id"], "mock-collection-id") @@ -514,6 +518,37 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertIn("zarr-data", item_dict["assets"]) self.assertIn("zarr-consolidated-metadata", item_dict["assets"]) + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): + """Multiple item configurations produce multiple item files and links.""" + mock_open_ds.return_value = self.mock_dataset + gen = OscDatasetStacGenerator( + collection_id="multi-collection", + items_config=[ + ItemConfig(dataset_id="first.zarr", item_id="first-item"), + ItemConfig(dataset_id="second.zarr", item_id="second-item"), + ], + workflow_id="dummy", + workflow_title="test", + license_type="proprietary", + ) + + file_dict = gen.build_zarr_stac_catalog_file_dict( + "s3://test-bucket/stac/multi-collection/" + ) + + self.assertIn( + "s3://test-bucket/stac/multi-collection/items/first-item.json", + file_dict, + ) + self.assertIn( + "s3://test-bucket/stac/multi-collection/items/second-item.json", + file_dict, + ) + catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] + item_links = [lnk for lnk in catalog["links"] if lnk["rel"] == "item"] + self.assertEqual(len(item_links), 2) + def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): """A 'via' link (STAC browser) and a 'child' link (HTTPS catalog) are added when stac_catalog_s3_root is provided. @@ -1019,7 +1054,9 @@ def test_save_prr_collection_writes_tree(self): self.assertEqual(out, tmp) collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + item_path = os.path.join( + tmp, "prr-collection", "items", "prr-collection.json" + ) self.assertTrue(os.path.isfile(collection_path)) self.assertTrue(os.path.isfile(item_path)) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index d910db0..ee1d651 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -56,10 +56,15 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: """Generate a complete dataset template with all possible keys and placeholder values""" required = { - "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", "collection_id": "[REQUIRED: unique identifier, no spaces — use hyphens (e.g. My-Dataset-2024)]", "license_type": "[REQUIRED: SPDX license identifier, e.g. CC-BY-4.0, MIT, proprietary]", "stac_catalog_s3_root": "[REQUIRED: S3 root for the STAC Catalog + Item, e.g. s3://my-bucket/stac/my-collection/]", + "items_config": [ + { + "dataset_id": "[REQUIRED: name of the Zarr store in your S3 bucket, e.g. my-dataset.zarr]", + "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", + } + ], } optional = { @@ -95,7 +100,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: stac_catalog_comment = ( "\n# stac_catalog_s3_root: deep-code writes the following files to this S3 root:\n" "# {stac_catalog_s3_root}/catalog.json (STAC Catalog root)\n" - "# {stac_catalog_s3_root}/{collection_id}/item.json (STAC Item for the whole Zarr)\n" + "# {stac_catalog_s3_root}/{collection_id}/items/{item_id}.json (STAC Item for each Zarr)\n" + "# items_config can contain multiple dataset/item pairs, but publish\n" + "# currently only consumes one item configuration.\n" "# S3 write credentials are resolved in order:\n" "# 1. STAC_S3_KEY / STAC_S3_SECRET env vars (STAC-specific, any bucket)\n" "# 2. AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars\n" diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 9bfddac..5eff849 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -16,7 +16,7 @@ import fsspec import yaml -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator logger = logging.getLogger(__name__) @@ -38,14 +38,30 @@ def generate_prr_collection( with fsspec.open(dataset_config_path, "r") as file: config = yaml.safe_load(file) or {} - dataset_id = config.get("dataset_id") collection_id = config.get("collection_id") license_type = config.get("license_type") + items_config_raw = config.get("items_config") - if not dataset_id or not collection_id: - raise ValueError( - "Both 'dataset_id' and 'collection_id' are required in the dataset config." - ) + if items_config_raw: + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + else: + dataset_id = config.get("dataset_id") + item_id = config.get("item_id") or collection_id + if not dataset_id or not collection_id: + raise ValueError( + "At least one item configuration must be provided in the dataset config, " + "along with 'collection_id'." + ) + items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] + + if not collection_id: + raise ValueError("collection_id is required in the dataset config.") if not license_type: raise ValueError( "license_type is required in the dataset config. " @@ -54,7 +70,7 @@ def generate_prr_collection( logger.info(f"Generating PRR STAC collection for '{collection_id}'.") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, + items_config=items_config, collection_id=collection_id, workflow_id=config.get("workflow_id") or "", workflow_title=config.get("workflow_title") or "", diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 3b62cbd..79a68dd 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,7 @@ OSC_REPO_OWNER, WORKFLOW_BASE_CATALOG_SELF_HREF, ) -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator from deep_code.utils.github_automation import GitHubAutomation from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, @@ -248,6 +248,34 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): full_path, var_id ) + @staticmethod + def _build_items_config(dataset_config: dict[str, Any]) -> list[ItemConfig]: + """Build item configs from the dataset config. + + Supports the new ``items_config`` list while keeping the legacy + single-item ``dataset_id`` / ``item_id`` fields for backwards + compatibility. + """ + items_config_raw = dataset_config.get("items_config") + if items_config_raw: + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] + else: + dataset_id = dataset_config.get("dataset_id") + collection_id = dataset_config.get("collection_id") + item_id = dataset_config.get("item_id") or collection_id + if not dataset_id: + raise ValueError( + "At least one item configuration must be provided in the dataset config." + ) + items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] + return items_config + def publish_dataset( self, write_to_file: bool = False, @@ -260,7 +288,11 @@ def publish_dataset( raise ValueError( "No dataset config loaded. Provide dataset_config_path to publish dataset." ) - dataset_id = self.dataset_config.get("dataset_id") + items_config = self._build_items_config(self.dataset_config) + if len(items_config) != 1: + raise ValueError( + "publish currently supports exactly one item configuration." + ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") access_link = self.dataset_config.get("access_link") @@ -275,8 +307,8 @@ def publish_dataset( osc_project_url = self.dataset_config.get("osc_project_url") description = self.dataset_config.get("description") - if not dataset_id or not self.collection_id: - raise ValueError("Dataset ID or Collection ID missing in the config.") + if not self.collection_id: + raise ValueError("Collection ID missing in the config.") if not license_type: raise ValueError( @@ -295,7 +327,7 @@ def publish_dataset( logger.info("Generating STAC collection...") generator = OscDatasetStacGenerator( - dataset_id=dataset_id, + items_config=items_config, collection_id=self.collection_id, workflow_id=self.workflow_id, workflow_title=self.workflow_title, diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 6deaa3c..f996b1e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -5,7 +5,9 @@ import json import logging +from dataclasses import dataclass from datetime import datetime, timezone +from typing import Any import pandas as pd from pystac import ( @@ -19,6 +21,7 @@ SpatialExtent, TemporalExtent, ) +import xarray as xr from deep_code.constants import ( CONTACTS_SCHEMA_URI, @@ -35,13 +38,20 @@ from deep_code.utils.osc_extension import OscExtension +@dataclass +class ItemConfig: + dataset_id: str + item_id: str + + class OscDatasetStacGenerator: """Generates OSC STAC Collections for a product from Zarr datasets. Args: - dataset_id: ID of the Zarr dataset. collection_id: Unique identifier for the STAC collection. - access_link: Public access link to the dataset. + items_config: List of item configuration entries. Each item maps one + dataset_id to one item_id. + access_link_root: Public access link to root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -53,18 +63,20 @@ class OscDatasetStacGenerator: def __init__( self, - dataset_id: str, collection_id: str, - workflow_id: str, - workflow_title: str, - license_type: str, - access_link: str | None = None, + items_config: list[ItemConfig] | None = None, + workflow_id: str = "", + workflow_title: str = "", + license_type: str = "", + dataset_id: str | None = None, + item_id: str | None = None, + access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", osc_region: str = "Global", osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, - cf_params: list[dict[str]] | None = None, + cf_params: list[dict[str, Any]] | None = None, osc_project: str = "deep-earth-system-data-lab", osc_project_title: str = "DeepESDL", osc_project_url: str | None = None, @@ -82,17 +94,29 @@ def __init__( if " " in collection_id: raise ValueError( f"collection_id must not contain spaces: {collection_id!r}. " - "Use hyphens as word separators (e.g. 'My-Dataset-2024')." + "Use hyphens as word separators (e.g. 'My-Collection-2024')." ) - self.dataset_id = dataset_id + if not items_config: + if dataset_id: + items_config = [ + ItemConfig( + dataset_id=dataset_id, + item_id=item_id or collection_id, + ) + ] + else: + raise ValueError( + "At least one item configuration must be provided." + ) self.collection_id = collection_id + self.items_config = items_config self.workflow_id = workflow_id self.workflow_title = workflow_title self.license_type = license_type self.osc_project = osc_project self.osc_project_title = osc_project_title self.osc_project_url = osc_project_url - self.access_link = access_link or f"s3://deep-esdl-public/{dataset_id}" + self.access_link_root = access_link_root or f"s3://deep-esdl-public/" self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region @@ -111,37 +135,41 @@ def __init__( self.sci_doi = sci_doi self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - self.dataset = open_dataset(dataset_id=dataset_id, logger=self.logger) - self.variables_metadata = self.get_variables_metadata() + self.datasets = self._load_item_datasets() + self.dataset = self.datasets[0] + self.variables_metadata = self._build_variables_metadata(self.datasets) - def _get_spatial_extent(self) -> SpatialExtent: + def _get_spatial_extent( + self, dataset: xr.Dataset | None = None + ) -> SpatialExtent: """Extract spatial extent from the dataset.""" - if {"lon", "lat"}.issubset(self.dataset.coords): + dataset = dataset or self.dataset + if {"lon", "lat"}.issubset(dataset.coords): # For regular gridding lon_min, lon_max = ( - float(self.dataset.lon.min()), - float(self.dataset.lon.max()), + float(dataset.lon.min()), + float(dataset.lon.max()), ) lat_min, lat_max = ( - float(self.dataset.lat.min()), - float(self.dataset.lat.max()), + float(dataset.lat.min()), + float(dataset.lat.max()), ) return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"longitude", "latitude"}.issubset(self.dataset.coords): + elif {"longitude", "latitude"}.issubset(dataset.coords): # For regular gridding with 'longitude' and 'latitude' lon_min, lon_max = ( - float(self.dataset.longitude.min()), - float(self.dataset.longitude.max()), + float(dataset.longitude.min()), + float(dataset.longitude.max()), ) lat_min, lat_max = ( - float(self.dataset.latitude.min()), - float(self.dataset.latitude.max()), + float(dataset.latitude.min()), + float(dataset.latitude.max()), ) return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) - elif {"x", "y"}.issubset(self.dataset.coords): + elif {"x", "y"}.issubset(dataset.coords): # For irregular gridding - x_min, x_max = (float(self.dataset.x.min()), float(self.dataset.x.max())) - y_min, y_max = (float(self.dataset.y.min()), float(self.dataset.y.max())) + x_min, x_max = (float(dataset.x.min()), float(dataset.x.max())) + y_min, y_max = (float(dataset.y.min()), float(dataset.y.max())) return SpatialExtent([[x_min, y_min, x_max, y_max]]) else: raise ValueError( @@ -149,17 +177,16 @@ def _get_spatial_extent(self) -> SpatialExtent: "('lon', 'lat' or 'x', 'y')." ) - def _get_temporal_extent(self) -> TemporalExtent: + def _get_temporal_extent( + self, dataset: xr.Dataset | None = None + ) -> TemporalExtent: """Extract temporal extent from the dataset.""" - if "time" in self.dataset.coords: + dataset = dataset or self.dataset + if "time" in dataset.coords: try: # Convert the time bounds to datetime objects - time_min = pd.to_datetime( - self.dataset.time.min().values - ).to_pydatetime() - time_max = pd.to_datetime( - self.dataset.time.max().values - ).to_pydatetime() + time_min = pd.to_datetime(dataset.time.min().values).to_pydatetime() + time_max = pd.to_datetime(dataset.time.max().values).to_pydatetime() return TemporalExtent([[time_min, time_max]]) except Exception as e: raise ValueError(f"Failed to parse temporal extent: {e}") @@ -172,11 +199,74 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - def _get_general_metadata(self) -> dict: + @staticmethod + def _unique_preserve_order(values: list[str]) -> list[str]: + """Return values without duplicates while preserving the input order.""" + return list(dict.fromkeys(values)) + + def _load_item_datasets(self) -> list[xr.Dataset]: + """Open the configured datasets in item order.""" + return [ + open_dataset(item_config.dataset_id, logger=self.logger) + for item_config in self.items_config + ] + + def _build_variables_metadata(self, datasets: list[xr.Dataset]) -> dict[str, dict]: + """Collect variable metadata across all configured datasets.""" + variables_metadata: dict[str, dict] = {} + for dataset in datasets: + variables_metadata.update(self.get_variables_metadata(dataset)) + return variables_metadata + + def _get_dataset_for_item(self, item_config: ItemConfig) -> xr.Dataset: + """Return the cached dataset that matches ``item_config``.""" + for config, dataset in zip(self.items_config, self.datasets): + if config.dataset_id == item_config.dataset_id: + return dataset + raise ValueError( + f"Dataset {item_config.dataset_id!r} is not part of the generator configuration." + ) + + def _build_access_link(self, item_config: ItemConfig) -> str: + """Return the asset href for an item, supporting prefix and full URLs.""" + access_link = self.access_link_root + if access_link.endswith("/"): + return f"{access_link.rstrip('/')}/{item_config.dataset_id}" + return access_link + + def _union_spatial_extent(self, datasets: list[xr.Dataset]) -> SpatialExtent: + """Merge multiple dataset spatial extents into a single bounding box.""" + bboxes = [ + self._get_spatial_extent(dataset).bboxes[0] + for dataset in datasets + ] + return SpatialExtent( + [[ + min(bbox[0] for bbox in bboxes), + min(bbox[1] for bbox in bboxes), + max(bbox[2] for bbox in bboxes), + max(bbox[3] for bbox in bboxes), + ]] + ) + + def _union_temporal_extent(self, datasets: list[xr.Dataset]) -> TemporalExtent: + """Merge multiple dataset temporal extents into a single interval.""" + intervals = [ + self._get_temporal_extent(dataset).intervals[0] + for dataset in datasets + ] + start_values = [interval[0] for interval in intervals if interval[0] is not None] + end_values = [interval[1] for interval in intervals if interval[1] is not None] + if not start_values or not end_values: + raise ValueError("Unable to compute a temporal extent for the datasets.") + return TemporalExtent([[min(start_values), max(end_values)]]) + + def _get_general_metadata(self, dataset: xr.Dataset | None = None) -> dict: + dataset = dataset or self.dataset return { "description": ( self.description - or self.dataset.attrs.get("description") + or dataset.attrs.get("description") or "No description available." ) } @@ -194,19 +284,23 @@ def extract_metadata_for_variable(self, variable_data) -> dict: "gcmd_keyword_url": gcmd_keyword_url, } - def get_variable_ids(self) -> list[str]: + def get_variable_ids(self, dataset: xr.Dataset | None = None) -> list[str]: """Get variable IDs for all variables in the dataset.""" - variable_ids = list(self.variables_metadata.keys()) + dataset = dataset or self.dataset + variable_ids = list(self.get_variables_metadata(dataset).keys()) # Remove 'crs' and 'spatial_ref' from the list if they exist, note that # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. return [ var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"] ] - def get_variables_metadata(self) -> dict[str, dict]: + def get_variables_metadata( + self, dataset: xr.Dataset | None = None + ) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" + dataset = dataset or self.dataset variables_metadata = {} - for var_name, variable in self.dataset.data_vars.items(): + for var_name, variable in dataset.data_vars.items(): var_metadata = self.extract_metadata_for_variable(variable) variables_metadata[var_metadata.get("variable_id")] = var_metadata return variables_metadata @@ -445,7 +539,9 @@ def build_project_collection(self) -> dict: CONTACTS_SCHEMA_URI, ], "title": self.format_string(self.osc_project_title or self.osc_project), - "description": self.format_string(self.osc_project_title or self.osc_project), + "description": self.format_string( + self.osc_project_title or self.osc_project + ), "keywords": [], "license": "various", "extent": { @@ -536,7 +632,7 @@ def _s3_to_https(s3_url: str) -> str: Example: s3://my-bucket/path/to/file → https://my-bucket.s3.amazonaws.com/path/to/file """ - without_scheme = s3_url[len("s3://"):] + without_scheme = s3_url[len("s3://") :] bucket, _, key = without_scheme.partition("/") return f"https://{bucket}.s3.amazonaws.com/{key}" @@ -549,40 +645,54 @@ def format_string(s: str) -> str: @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) - def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: + def build_zarr_stac_item( + self, + item_config: ItemConfig | str | None = None, + stac_catalog_s3_root: str | None = None, + ) -> Item: """Build a single STAC Item representing the entire Zarr store. One item covers the full spatiotemporal extent of the dataset. Assets point to the Zarr store and its consolidated metadata. Args: + item_config: object containing `dataset_id` and `item_id` stac_catalog_s3_root: S3 root URL where the STAC catalog will be hosted (e.g. ``s3://my-bucket/stac/``). Used to build self/root/parent hrefs. Returns: A :class:`pystac.Item` ready to be serialised to S3. """ + if isinstance(item_config, str) and stac_catalog_s3_root is None: + stac_catalog_s3_root = item_config + item_config = None + item_config = item_config or self.items_config[0] + if stac_catalog_s3_root is None: + raise ValueError("stac_catalog_s3_root is required.") + self.logger.info(f"Building STAC Item for collection '{self.collection_id}'.") - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + dataset = self._get_dataset_for_item(item_config) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -595,18 +705,19 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: now_iso = datetime.now(timezone.utc).isoformat() root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - item_href = f"{root}/{self.collection_id}/item.json" + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" osc_collection_href = ( "https://esa-earthcode.github.io/open-science-catalog-metadata" f"/products/{self.collection_id}/collection.json" ) item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, properties={ + "title": self.format_string(item_config.item_id), "start_datetime": start_dt.isoformat() if start_dt else None, "end_datetime": end_dt.isoformat() if end_dt else None, "description": general_metadata.get("description", ""), @@ -616,26 +727,39 @@ def build_zarr_stac_item(self, stac_catalog_s3_root: str) -> Item: ) item.collection_id = self.collection_id item.set_self_href(item_href) - item.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - item.add_link(Link(rel="parent", target=catalog_href, media_type="application/json")) - item.add_link(Link( - rel="collection", - target=osc_collection_href, - media_type="application/json", - title=self.collection_id, - )) - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) + item.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link(rel="parent", target=catalog_href, media_type="application/json") + ) + item.add_link( + Link( + rel="collection", + target=osc_collection_href, + media_type="application/json", + title=self.collection_id, + ) + ) + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) self.logger.info(f"STAC Item built: {item_href}") return item @@ -650,7 +774,9 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ - └── item.json # STAC Item (whole Zarr) + └── items/ + ├── {item_id_0}.json # STAC Item (whole Zarr) + └── {item_id_1}.json # STAC Item (whole Zarr) Args: stac_catalog_s3_root: S3 root URL (e.g. ``s3://my-bucket/stac/``). @@ -665,37 +791,46 @@ def build_zarr_stac_catalog_file_dict( root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - item = self.build_zarr_stac_item(stac_catalog_s3_root) - catalog = Catalog( id=f"{self.collection_id}-stac-catalog", description=f"STAC Catalog for {self.collection_id}", ) catalog.set_self_href(catalog_href) - catalog.add_link(Link(rel="root", target=catalog_href, media_type="application/json")) - catalog.add_link(Link( - rel="item", - target=f"./{self.collection_id}/item.json", - media_type="application/json", - title=self.collection_id, - )) - - item_href = f"{root}/{self.collection_id}/item.json" - self.logger.info(f"STAC Catalog file dict ready: {catalog_href}, {item_href}") - return { - catalog_href: catalog.to_dict(transform_hrefs=False), - item_href: item.to_dict(transform_hrefs=False), - } + catalog.add_link( + Link(rel="root", target=catalog_href, media_type="application/json") + ) + file_dict = {catalog_href: catalog.to_dict(transform_hrefs=False)} + + for item_config in self.items_config: + item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" + catalog.add_link( + Link( + rel="item", + target=f"./{self.collection_id}/items/{item_config.item_id}.json", + media_type="application/json", + title=item_config.item_id, + ) + ) + file_dict[item_href] = item.to_dict(transform_hrefs=False) + + file_dict[catalog_href] = catalog.to_dict(transform_hrefs=False) + self.logger.info( + "STAC Catalog file dict ready: %s", + ", ".join(file_dict.keys()), + ) + return file_dict - # ------------------------------------------------------------------ # - # PRR (Project Results Repository) style output # + # ------------------------------------------------------------------- # + # PRR (Project Results Repository) style output # # # # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # - # the plain catalog.json/item.json under ``{root}/prr/``. # - # ------------------------------------------------------------------ # + # the plain catalog.json/items/*.json under ``{root}/prr/``. # + # ------------------------------------------------------------------- # - def _get_epsg(self) -> int: + @staticmethod + def _get_epsg(dataset: xr.Dataset) -> int: """Best-effort EPSG code for the dataset, defaulting to 4326. Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or @@ -703,8 +838,8 @@ def _get_epsg(self) -> int: WGS 84 (EPSG:4326). """ for var_name in ("spatial_ref", "crs"): - if var_name in self.dataset.variables: - attrs = self.dataset[var_name].attrs + if var_name in dataset.variables: + attrs = dataset[var_name].attrs for key in ("spatial_epsg", "epsg", "EPSG"): if key in attrs: try: @@ -713,7 +848,7 @@ def _get_epsg(self) -> int: pass return 4326 - def _get_cube_dimensions(self) -> dict[str, dict]: + def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:dimensions`` object from the dataset coordinates. Follows the datacube STAC extension: horizontal spatial dimensions are @@ -721,16 +856,16 @@ def _get_cube_dimensions(self) -> dict[str, dict]: dimension, and any remaining index coordinate is emitted as an additional dimension. """ - ds = self.dataset - epsg = self._get_epsg() + epsg = self._get_epsg(dataset) x_names = {"lon", "longitude", "x"} y_names = {"lat", "latitude", "y"} dimensions: dict[str, dict] = {} - for name, coord in ds.coords.items(): - if name not in ds.dims: + for name, coord in dataset.coords.items(): + if name not in dataset.dims: # Skip non-dimension coordinates (e.g. scalar or auxiliary coords). continue - lname = str(name).lower() + name = str(name) + lname = name.lower() if lname in x_names: dimensions[name] = { "type": "spatial", @@ -765,11 +900,12 @@ def _get_cube_dimensions(self) -> dict[str, dict]: } return dimensions - def _get_cube_variables(self) -> dict[str, dict]: + @staticmethod + def _get_cube_variables(dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:variables`` object from the dataset data variables.""" skip = {"crs", "spatial_ref"} variables: dict[str, dict] = {} - for name, var in self.dataset.data_vars.items(): + for name, var in dataset.data_vars.items(): if name in skip: continue entry: dict = { @@ -782,10 +918,10 @@ def _get_cube_variables(self) -> dict[str, dict]: description = var.attrs.get("long_name") or var.attrs.get("description") if description: entry["description"] = description - variables[name] = entry + variables[str(name)] = entry return variables - def build_prr_stac_item(self) -> Item: + def build_prr_stac_item(self, item_config: ItemConfig | None = None) -> Item: """Build the single datacube Item for the PRR collection. One Item covers the full spatiotemporal extent of the Zarr store. It @@ -794,24 +930,30 @@ def build_prr_stac_item(self) -> Item: (root/parent/collection/self) are left for :meth:`save_prr_collection` to fill in via ``Collection.add_item`` + ``normalize_hrefs``. """ + item_config = item_config or self.items_config[0] + self.logger.info( - f"Building PRR STAC Item for collection '{self.collection_id}'." + f"Building PRR STAC Item '{item_config.item_id}' " + f"for collection '{self.collection_id}'." ) - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - general_metadata = self._get_general_metadata() + dataset = self._get_dataset_for_item(item_config) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + general_metadata = self._get_general_metadata(dataset) bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] lon_min, lat_min, lon_max, lat_max = bbox geometry = { "type": "Polygon", - "coordinates": [[ - [lon_min, lat_min], - [lon_max, lat_min], - [lon_max, lat_max], - [lon_min, lat_max], - [lon_min, lat_min], - ]], + "coordinates": [ + [ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ] + ], } start_dt, end_dt = temporal_extent.intervals[0] @@ -823,7 +965,7 @@ def build_prr_stac_item(self) -> Item: now_iso = datetime.now(timezone.utc).isoformat() item = Item( - id=self.collection_id, + id=item_config.item_id, geometry=geometry, bbox=bbox, datetime=None, @@ -833,30 +975,39 @@ def build_prr_stac_item(self) -> Item: "description": general_metadata.get("description", ""), "created": now_iso, "updated": now_iso, - "cube:dimensions": self._get_cube_dimensions(), - "cube:variables": self._get_cube_variables(), + "cube:dimensions": self._get_cube_dimensions(dataset), + "cube:variables": self._get_cube_variables(dataset), }, ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. - item.add_asset("zarr-data", Asset( - href=self.access_link, - media_type=ZARR_MEDIA_TYPE, - title="Zarr Data Store", - roles=["data"], - )) - item.add_asset("zarr-consolidated-metadata", Asset( - href=f"{self.access_link}/.zmetadata", - media_type="application/json", - title="Consolidated Zarr Metadata", - roles=["metadata"], - )) - self.logger.info(f"PRR STAC Item built for '{self.collection_id}'.") + access_link = self._build_access_link(item_config) + item.add_asset( + "zarr-data", + Asset( + href=access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + ), + ) + item.add_asset( + "zarr-consolidated-metadata", + Asset( + href=f"{access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + ), + ) + self.logger.info( + f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." + ) return item def build_prr_collection(self) -> Collection: - """Build the PRR parent Collection with its single datacube Item attached. + """Build the PRR parent Collection with its datacube Item(s) attached. The Collection carries OSC extension fields (``osc:type``, ``osc:status``, ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list @@ -864,17 +1015,27 @@ def build_prr_collection(self) -> Collection: endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a subsequent ``normalize_hrefs`` produces a self-contained tree. """ - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + items = [ + self.build_prr_stac_item(item_config) for item_config in self.items_config + ] + spatial_extent = SpatialExtent([item.bbox for item in items]) + temporal_extent = TemporalExtent( + [ + [item.properties["start_datetime"], item.properties["end_datetime"]] + for item in items + ] + ) + variables: list[str] = [] + for dataset in self.datasets: + variables.extend(self.get_variable_ids(dataset)) + variables = self._unique_preserve_order(variables) collection = Collection( id=self.collection_id, - description=general_metadata.get("description", "No description provided."), + description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.collection_id, + title=self.description or self.collection_id, ) osc_extension = OscExtension.add_to(collection) @@ -914,16 +1075,14 @@ def build_prr_collection(self) -> Collection: collection.extra_fields["osc:initiative"] = self.osc_initiative project_website = ( - self.osc_project_website - or self.osc_project_url - or self.documentation_link + self.osc_project_website or self.osc_project_url or self.documentation_link ) if project_website: collection.extra_fields["osc:project_website"] = project_website project_description = ( self.osc_project_description or self.description - or general_metadata.get("description") + or "No description provided." ) if project_description: collection.extra_fields["osc:project_description"] = project_description @@ -939,23 +1098,28 @@ def build_prr_collection(self) -> Collection: # Thumbnail asset (REQUIRED by the PRR spec: an asset named 'thumbnail' # with the 'thumbnail' role). if self.thumbnail: - collection.add_asset("thumbnail", Asset( - href=self.thumbnail, - media_type=self._thumbnail_media_type(), - title="Collection Thumbnail", - roles=["thumbnail"], - )) + collection.add_asset( + "thumbnail", + Asset( + href=self.thumbnail, + media_type=self._thumbnail_media_type(), + title="Collection Thumbnail", + roles=["thumbnail"], + ), + ) if self.documentation_link: collection.add_link( Link(rel="via", target=self.documentation_link, title="Documentation") ) if self.visualisation_link: - collection.add_link(Link( - rel="visualisation", - target=self.visualisation_link, - title="Dataset visualisation", - )) + collection.add_link( + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) + ) try: osc_extension.validate_extension() @@ -963,7 +1127,8 @@ def build_prr_collection(self) -> Collection: raise ValueError(f"OSC Extension validation failed: {e}") self._warn_missing_prr_fields(collection, variables) - collection.add_item(self.build_prr_stac_item()) + for item in items: + collection.add_item(item) return collection def _thumbnail_media_type(self) -> str: @@ -1015,7 +1180,9 @@ def save_prr_collection(self, output_dir: str) -> str: {output_dir}/ ├── collection.json # STAC Collection (root) └── {collection_id}/ - └── {collection_id}.json # datacube Item (whole Zarr) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) Zarr asset hrefs remain absolute (``s3://…``) since that is where the data lives. @@ -1036,17 +1203,23 @@ def save_prr_collection(self, output_dir: str) -> str: self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir - def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | None = None) -> Collection: + def build_dataset_stac_collection( + self, mode: str, stac_catalog_s3_root: str | None = None + ) -> Collection: """Build an OSC STAC Collection for the dataset. Returns: A pystac.Collection object. """ try: - spatial_extent = self._get_spatial_extent() - temporal_extent = self._get_temporal_extent() - variables = self.get_variable_ids() - general_metadata = self._get_general_metadata() + datasets = self.datasets + spatial_extent = self._union_spatial_extent(datasets) + temporal_extent = self._union_temporal_extent(datasets) + variables: list[str] = [] + for dataset in datasets: + variables.extend(self.get_variable_ids(dataset)) + variables = self._unique_preserve_order(variables) + general_metadata = self._get_general_metadata(datasets[0]) except ValueError as e: raise ValueError(f"Metadata extraction failed: {e}") @@ -1093,7 +1266,11 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) if self.visualisation_link: collection.add_link( - Link(rel="visualisation", target=self.visualisation_link, title="Dataset visualisation") + Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + ) ) collection.add_link( Link( @@ -1146,7 +1323,7 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N ) ) - if mode in "all": + if mode == "all": collection.add_link( Link( rel="related", @@ -1168,19 +1345,23 @@ def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | N catalog_https = self._s3_to_https(catalog_s3) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" - + catalog_https[len("https://"):] + + catalog_https[len("https://") :] + ) + collection.add_link( + Link( + rel="via", + target=stac_browser_href, + title="Access", + ) + ) + collection.add_link( + Link( + rel="child", + target=catalog_https, + media_type="application/json", + title="Items", + ) ) - collection.add_link(Link( - rel="via", - target=stac_browser_href, - title="Access", - )) - collection.add_link(Link( - rel="child", - target=catalog_https, - media_type="application/json", - title="Items", - )) # Validate OSC extension fields try: diff --git a/docs/cli.md b/docs/cli.md index a269479..7b6da7a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -64,13 +64,16 @@ produces a self-contained `Collection → Item → Assets` tree: prr// ├── collection.json # STAC Collection (root, relative links) └── / - └── .json # datacube Item covering the whole Zarr store + └── items/ + └── .json # datacube Item covering the whole Zarr store ``` - The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) plus `zarr-data` and `zarr-consolidated-metadata` assets. - The **Collection** carries the OSC, Scientific, Processing, Themes and CF extensions and the PRR-mandatory fields. +- `deep-code publish` still publishes one dataset/item at a time; the multi-item + generator support is exposed first through the lower-level API and the PRR helper. The output conforms to the [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html) diff --git a/docs/configuration.md b/docs/configuration.md index 881c572..965ef14 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -16,10 +16,12 @@ The sections below document every field in those templates. ## Dataset config (YAML) ```yaml # Required -dataset_id: your-dataset.zarr collection_id: your-collection # no spaces — use hyphens license_type: CC-BY-4.0 stac_catalog_s3_root: s3://bucket/stac/your-collection/ +items_config: + - dataset_id: your-dataset.zarr + item_id: your-item # no spaces — use hyphens # Optional osc_themes: [cryosphere] # must match slugs at opensciencedata.esa.int/themes/catalog — auto-lowercased @@ -48,21 +50,23 @@ prr_output_dir: ./prr/your-collection ### Field reference -| Field | Required | Description | -|---|---|---| -| `dataset_id` | Yes | Zarr store identifier (used to open the dataset). | -| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Dataset-2024`). | -| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | -| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | -| `osc_region` | No | Geographical region label (default: `Global`). | -| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | -| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | -| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | -| `documentation_link` | No | URL to dataset documentation. | -| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | -| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | -| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | -| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +| Field | Required | Description | +|------------------------|---|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | +| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | +| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | +| `dataset_id` | Legacy | Single-item fallback when `items_config` is omitted. | +| `item_id` | Legacy | Single-item fallback when `items_config` is omitted. If omitted there, deep-code falls back to the `collection_id` for compatibility. | +| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | +| `osc_region` | No | Geographical region label (default: `Global`). | +| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | +| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | +| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | +| `documentation_link` | No | URL to dataset documentation. | +| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | +| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | +| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | +| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | > The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). @@ -93,7 +97,8 @@ prr_output_dir: ./prr/your-collection s3://bucket/stac/your-collection/ ├── catalog.json # STAC Catalog (root) └── your-collection/ - └── item.json # STAC Item covering the full Zarr store + └── items/ + └── your-item.json # STAC Item covering the full Zarr store ``` The item has two assets: @@ -119,7 +124,8 @@ self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): prr/your-collection/ ├── collection.json # STAC Collection (root, relative links) └── your-collection/ - └── your-collection.json # datacube Item covering the full Zarr store + └── items/ + └── your-item.json # datacube Item covering the full Zarr store ``` - **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, diff --git a/docs/python-api.md b/docs/python-api.md index aa9d0ef..e650ba0 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -71,14 +71,14 @@ publisher.publish(write_to_file=False, mode="dataset") over individual artifacts. ```python -from deep_code.utils.dataset_stac_generator import OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator generator = OscDatasetStacGenerator( - dataset_id="my-dataset.zarr", collection_id="my-collection", workflow_id="my-workflow", workflow_title="My Workflow", license_type="CC-BY-4.0", + items_config=[ItemConfig(dataset_id="my-dataset.zarr", item_id="my-item")], osc_themes=["cryosphere"], osc_region="Global", osc_status="completed", @@ -119,7 +119,7 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( ) # file_dict contains: # "s3://bucket/stac/my-collection/catalog.json" -# "s3://bucket/stac/my-collection/my-collection/item.json" +# "s3://bucket/stac/my-collection/my-collection/items/my-item.json" ``` See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the @@ -134,7 +134,7 @@ tree as local files. The high-level helper reads the same dataset config as the from deep_code.tools.prr import generate_prr_collection out_dir = generate_prr_collection("dataset.yaml", output_dir="./prr") -# ./prr/collection.json + ./prr//.json +# ./prr/collection.json + ./prr//items/.json ``` Or drive the generator directly: From 13252fb9859b9ba74f0fce370b5e7027d0400e52 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 12 Aug 2026 16:41:48 +0200 Subject: [PATCH 08/20] check changes --- a | 1 + b | 1 + .../utils/test_dataset_stac_generator.py | 252 ++++++++++-------- deep_code/tools/publish.py | 6 +- deep_code/utils/dataset_stac_generator.py | 192 +++++-------- 5 files changed, 207 insertions(+), 245 deletions(-) create mode 100644 a create mode 100644 b diff --git a/a b/a new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/a @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/b b/b new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/b @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 5a20808..a564ef0 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -8,7 +8,7 @@ import tempfile import unittest from datetime import datetime -from unittest.mock import MagicMock, patch +from unittest.mock import patch import numpy as np from pystac import Catalog, Collection, Item @@ -32,8 +32,7 @@ class TestOSCProductSTACGenerator(unittest.TestCase): - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_data_store): + def setUp(self): """Set up a mock dataset and generator.""" self.mock_dataset = Dataset( coords={ @@ -69,16 +68,24 @@ def setUp(self, mock_data_store): ), }, ) - mock_store = MagicMock() - mock_store.open_data.return_value = self.mock_dataset - mock_data_store.return_value = self.mock_dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.mock_dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.generator = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", - access_link="s3://mock-bucket/mock-dataset", + access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", osc_status="ongoing", @@ -88,18 +95,19 @@ def setUp(self, mock_data_store): def test_open_dataset(self): """Test if the dataset is opened correctly.""" - self.assertIsInstance(self.generator.dataset, Dataset) + dataset = self.mock_dataset + self.assertIsInstance(dataset, Dataset) for coord in ("lon", "lat", "time"): - self.assertIn(coord, self.generator.dataset.coords) + self.assertIn(coord, dataset.coords) def test_get_spatial_extent(self): """Test spatial extent extraction.""" - extent = self.generator._get_spatial_extent() + extent = self.generator._get_spatial_extent(self.mock_dataset) self.assertEqual(extent.bboxes[0], [-180.0, -90.0, 180.0, 90.0]) def test_get_temporal_extent(self): """Test temporal extent extraction.""" - extent = self.generator._get_temporal_extent() + extent = self.generator._get_temporal_extent(self.mock_dataset) # TemporalExtent.intervals is a list of [start, end] interval = extent.intervals[0] self.assertEqual(interval[0], datetime(2023, 1, 1, 0, 0)) @@ -107,12 +115,12 @@ def test_get_temporal_extent(self): def test_get_variables(self): """Test variable ID extraction.""" - vars_ = self.generator.get_variable_ids() + vars_ = self.generator.get_variable_ids(self.mock_dataset) self.assertCountEqual(vars_, ["var1", "var2"]) def test_get_general_metadata(self): """Test general metadata extraction.""" - meta = self.generator._get_general_metadata() + meta = self.generator._get_general_metadata(self.mock_dataset) self.assertEqual(meta.get("description"), "Mock dataset for testing.") def test_extract_metadata_for_variable(self): @@ -125,7 +133,7 @@ def test_extract_metadata_for_variable(self): def test_get_variables_metadata(self): """Test metadata dict for all variables.""" - meta_dict = self.generator.get_variables_metadata() + meta_dict = self.generator.get_variables_metadata(self.mock_dataset) self.assertIn("var1", meta_dict) self.assertIn("var2", meta_dict) self.assertIsInstance(meta_dict["var1"], dict) @@ -138,11 +146,10 @@ def test_build_theme(self): ids = [tc.id for tc in theme_obj.concepts] self.assertListEqual(ids, ["a", "b"]) - @patch.object(OscDatasetStacGenerator, "_add_gcmd_link_to_var_catalog") - @patch.object(OscDatasetStacGenerator, "add_themes_as_related_links_var_catalog") - def test_build_variable_catalog(self, mock_add_themes, mock_add_gcmd): + def test_build_variable_catalog(self): """Test building of variable-level STAC catalog.""" - var_meta = self.generator.variables_metadata["var1"] + variables_metadata = self.generator.get_variables_metadata(self.mock_dataset) + var_meta = variables_metadata["var1"] catalog = self.generator.build_variable_catalog(var_meta) self.assertIsInstance(catalog, Catalog) self.assertEqual(catalog.id, "var1") @@ -169,9 +176,7 @@ def test_update_product_base_catalog(self): import tempfile import json as _json - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -210,9 +215,7 @@ def test_update_variable_base_catalog(self): import json as _json import os - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -223,9 +226,7 @@ def test_update_variable_base_catalog(self): os.unlink(tmp_path) self.assertIsInstance(result, dict) - child_hrefs = [ - lnk["href"] for lnk in result["links"] if lnk["rel"] == "child" - ] + child_hrefs = [lnk["href"] for lnk in result["links"] if lnk["rel"] == "child"] self.assertEqual(len(child_hrefs), len(vars_)) # self link must remain in place self.assertEqual(result["links"][0]["rel"], "self") @@ -243,8 +244,13 @@ def test_osc_project_custom(self, mock_open_ds): """A custom osc_project is stored on the generator.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -291,8 +297,13 @@ def test_build_project_collection_custom_project(self, mock_open_ds): """build_project_collection reflects a custom osc_project.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( - dataset_id="mock-dataset-id", collection_id="mock-collection-id", + items_config=[ + ItemConfig( + dataset_id="mock-dataset-id", + item_id="mock-collection-id", + ), + ], workflow_id="dummy", workflow_title="test", license_type="proprietary", @@ -397,9 +408,7 @@ def test_update_deepesdl_collection(self): import json as _json import os - with tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False - ) as tmp: + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) tmp_path = tmp.name @@ -423,7 +432,9 @@ def test_update_deepesdl_collection(self): def test_build_zarr_stac_item_structure(self): """Item has correct geometry, bbox, datetime range, assets, and links.""" s3_root = "s3://test-bucket/stac/my-collection/" - item = self.generator.build_zarr_stac_item(s3_root) + item = self.generator.build_zarr_stac_item( + self.generator.items_config[0], s3_root + ) self.assertIsInstance(item, Item) self.assertEqual(item.id, "mock-collection-id") @@ -447,20 +458,21 @@ def test_build_zarr_stac_item_structure(self): self.assertIn("zarr-consolidated-metadata", item.assets) zarr_asset = item.assets["zarr-data"] - self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset") + self.assertEqual(zarr_asset.href, "s3://mock-bucket/mock-dataset-id") self.assertEqual(zarr_asset.media_type, ZARR_MEDIA_TYPE) self.assertIn("data", zarr_asset.roles) meta_asset = item.assets["zarr-consolidated-metadata"] self.assertEqual( - meta_asset.href, "s3://mock-bucket/mock-dataset/.zmetadata" + meta_asset.href, + "s3://mock-bucket/mock-dataset-id/.zmetadata", ) self.assertIn("metadata", meta_asset.roles) # Self href self.assertEqual( item.self_href, - "s3://test-bucket/stac/my-collection/items/mock-collection-id.json", + "s3://test-bucket/stac/my-collection/mock-collection-id/items/mock-collection-id.json", ) # Required link rels @@ -483,8 +495,12 @@ def test_build_zarr_stac_item_structure(self): def test_build_zarr_stac_item_trailing_slash_normalised(self): """Trailing slash on s3_root should not produce double slashes.""" - item_with = self.generator.build_zarr_stac_item("s3://bucket/stac/") - item_without = self.generator.build_zarr_stac_item("s3://bucket/stac") + item_with = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac/" + ) + item_without = self.generator.build_zarr_stac_item( + self.generator.items_config[0], "s3://bucket/stac" + ) self.assertEqual(item_with.self_href, item_without.self_href) def test_build_zarr_stac_catalog_file_dict_keys(self): @@ -494,7 +510,8 @@ def test_build_zarr_stac_catalog_file_dict_keys(self): catalog_path = "s3://test-bucket/stac/my-collection/catalog.json" item_path = ( - "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ) self.assertIn(catalog_path, file_dict) self.assertIn(item_path, file_dict) @@ -510,7 +527,8 @@ def test_build_zarr_stac_catalog_file_dict_content(self): self.assertEqual(catalog_dict["id"], "mock-collection-id-stac-catalog") item_dict = file_dict[ - "s3://test-bucket/stac/my-collection/items/mock-collection-id.json" + "s3://test-bucket/stac/my-collection/" + "mock-collection-id/items/mock-collection-id.json" ] self.assertEqual(item_dict["type"], "Feature") self.assertEqual(item_dict["id"], "mock-collection-id") @@ -538,11 +556,13 @@ def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): ) self.assertIn( - "s3://test-bucket/stac/multi-collection/items/first-item.json", + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/first-item.json", file_dict, ) self.assertIn( - "s3://test-bucket/stac/multi-collection/items/second-item.json", + "s3://test-bucket/stac/multi-collection/" + "multi-collection/items/second-item.json", file_dict, ) catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] @@ -561,21 +581,31 @@ def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): collection = self.generator.build_dataset_stac_collection( mode="dataset", stac_catalog_s3_root=s3_root ) - https_catalog = "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + https_catalog = ( + "https://test-bucket.s3.amazonaws.com/stac/my-collection/catalog.json" + ) stac_browser_href = ( "https://opensciencedata.esa.int/stac-browser/#/external/" + https_catalog.replace("https://", "") ) via_link = next( - (lnk for lnk in collection.links if lnk.rel == "via" and "stac-browser" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "via" and "stac-browser" in str(lnk.target) + ), None, ) self.assertIsNotNone(via_link, "Expected a 'via' STAC browser link") self.assertEqual(via_link.target, stac_browser_href) child_link = next( - (lnk for lnk in collection.links if lnk.rel == "child" and "catalog.json" in str(lnk.target)), + ( + lnk + for lnk in collection.links + if lnk.rel == "child" and "catalog.json" in str(lnk.target) + ), None, ) self.assertIsNotNone(child_link, "Expected a 'child' HTTPS catalog link") @@ -653,20 +683,26 @@ def test_edge_cases(self): class TestOscDatasetStacGeneratorExtra(unittest.TestCase): """Additional tests to cover branches not exercised by TestOSCProductSTACGenerator.""" - def _make_generator(self, mock_ds, collection_id="my-collection", **kwargs): - with patch("deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds): + @staticmethod + def _make_generator(mock_ds, collection_id="my-collection", **kwargs): + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds + ): return OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id=collection_id, + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id=collection_id), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", **kwargs, ) - def _make_dataset(self, coord_type="lon_lat"): - import numpy as np + @staticmethod + def _make_dataset(coord_type="lon_lat"): from datetime import datetime + if coord_type == "lon_lat": coords = { "lon": ("lon", np.linspace(-10, 10, 3)), @@ -688,6 +724,7 @@ def _make_dataset(self, coord_type="lon_lat"): else: coords = {} from xarray import Dataset + return Dataset(coords=coords) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -695,8 +732,10 @@ def test_collection_id_with_space_raises(self, mock_open_ds): mock_open_ds.return_value = self._make_dataset() with self.assertRaisesRegex(ValueError, "must not contain spaces"): OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="bad id", + items_config=[ + ItemConfig(dataset_id="mock-dataset-id", item_id="bad-id"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", @@ -707,7 +746,7 @@ def test_spatial_extent_longitude_latitude(self, mock_open_ds): ds = self._make_dataset("longitude_latitude") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], -10.0) self.assertAlmostEqual(extent.bboxes[0][1], -5.0) @@ -716,7 +755,7 @@ def test_spatial_extent_x_y(self, mock_open_ds): ds = self._make_dataset("x_y") mock_open_ds.return_value = ds gen = self._make_generator(ds) - extent = gen._get_spatial_extent() + extent = gen._get_spatial_extent(ds) self.assertAlmostEqual(extent.bboxes[0][0], 0.0) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -725,7 +764,7 @@ def test_spatial_extent_unknown_coords_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "recognized spatial coordinates"): - gen._get_spatial_extent() + gen._get_spatial_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_temporal_extent_no_time_raises(self, mock_open_ds): @@ -733,7 +772,7 @@ def test_temporal_extent_no_time_raises(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) with self.assertRaisesRegex(ValueError, "time"): - gen._get_temporal_extent() + gen._get_temporal_extent(ds) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_normalize_name_none_returns_none(self, mock_open_ds): @@ -745,9 +784,14 @@ def test_normalize_name_none_returns_none(self, mock_open_ds): def test_build_collection_with_cf_params(self, mock_open_ds): ds = self._make_dataset() mock_open_ds.return_value = ds - gen = self._make_generator(ds, cf_params=[{"name": "temperature", "units": "K"}]) + gen = self._make_generator( + ds, cf_params=[{"name": "temperature", "units": "K"}] + ) collection = gen.build_dataset_stac_collection(mode="dataset") - self.assertEqual(collection.extra_fields.get("cf:parameter"), [{"name": "temperature", "units": "K"}]) + self.assertEqual( + collection.extra_fields.get("cf:parameter"), + [{"name": "temperature", "units": "K"}], + ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_build_collection_with_visualisation_link(self, mock_open_ds): @@ -766,11 +810,12 @@ def test_build_collection_mode_all_adds_experiment_link(self, mock_open_ds): mock_open_ds.return_value = ds gen = self._make_generator(ds) collection = gen.build_dataset_stac_collection(mode="all") - exp_links = [lnk for lnk in collection.links if "experiments" in str(lnk.target)] + exp_links = [ + lnk for lnk in collection.links if "experiments" in str(lnk.target) + ] self.assertEqual(len(exp_links), 1) - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def test_s3_to_https(self, mock_open_ds): + def test_s3_to_https(self): self.assertEqual( OscDatasetStacGenerator._s3_to_https("s3://my-bucket/path/to/file.json"), "https://my-bucket.s3.amazonaws.com/path/to/file.json", @@ -797,7 +842,7 @@ def test_update_existing_variable_catalog(self, mock_open_ds): json.dump(base, f) tmp_path = f.name try: - result = gen.update_existing_variable_catalog(tmp_path, "var1") + result = gen.update_existing_variable_catalog(tmp_path) finally: os.unlink(tmp_path) @@ -809,8 +854,7 @@ def test_update_existing_variable_catalog(self, mock_open_ds): class TestPRRCollection(unittest.TestCase): """Tests for the PRR-style Collection -> Item -> Assets generation.""" - @patch("deep_code.utils.dataset_stac_generator.open_dataset") - def setUp(self, mock_open_ds): + def setUp(self): self.dataset = Dataset( coords={ "lon": ("lon", np.linspace(-20, 20, 4)), @@ -839,14 +883,21 @@ def setUp(self, mock_open_ds): }, attrs={"description": "PRR test cube"}, ) - mock_open_ds.return_value = self.dataset + self.open_dataset_patcher = patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ) + self.mock_open_dataset = self.open_dataset_patcher.start() + self.addCleanup(self.open_dataset_patcher.stop) self.gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -857,7 +908,7 @@ def setUp(self, mock_open_ds): # ---- helpers ---- def test_get_epsg_from_spatial_ref(self): - self.assertEqual(self.gen._get_epsg(), 3035) + self.assertEqual(self.gen._get_epsg(self.dataset), 3035) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_get_epsg_default_4326(self, mock_open_ds): @@ -871,16 +922,18 @@ def test_get_epsg_default_4326(self, mock_open_ds): ) mock_open_ds.return_value = ds gen = OscDatasetStacGenerator( - dataset_id="t.zarr", collection_id="c", + items_config=[ + ItemConfig(dataset_id="t.zarr", item_id="c"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", ) - self.assertEqual(gen._get_epsg(), 4326) + self.assertEqual(gen._get_epsg(ds), 4326) def test_get_cube_dimensions(self): - dims = self.gen._get_cube_dimensions() + dims = self.gen._get_cube_dimensions(self.dataset) self.assertEqual(set(dims), {"lon", "lat", "time"}) self.assertEqual(dims["lon"]["type"], "spatial") self.assertEqual(dims["lon"]["axis"], "x") @@ -892,7 +945,7 @@ def test_get_cube_dimensions(self): self.assertEqual(len(dims["time"]["extent"]), 2) def test_get_cube_variables(self): - variables = self.gen._get_cube_variables() + variables = self.gen._get_cube_variables(self.dataset) # CRS variable must be excluded. self.assertEqual(set(variables), {"sst", "chl"}) self.assertEqual(variables["sst"]["type"], "data") @@ -906,7 +959,7 @@ def test_get_cube_variables(self): # ---- item ---- def test_build_prr_stac_item(self): - item = self.gen.build_prr_stac_item() + item = self.gen.build_prr_stac_item(self.gen.items_config[0]) self.assertIsInstance(item, Item) self.assertEqual(item.id, "prr-collection") self.assertIn(DATACUBE_SCHEMA_URI, item.stac_extensions) @@ -967,12 +1020,14 @@ def test_build_prr_collection_conformant_fields(self): return_value=self.dataset, ): gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket/", osc_status="ongoing", osc_region="Global", osc_themes=["oceans"], @@ -1010,7 +1065,7 @@ def test_build_prr_collection_fallbacks(self): self.assertEqual(ef["osc:initiative"], "earthcode") # Fallbacks: website -> documentation_link, description -> dataset description. self.assertEqual(ef["osc:project_website"], "https://example.org/doc") - self.assertEqual(ef["osc:project_description"], "PRR test cube") + self.assertEqual(ef["osc:project_description"], "No description provided.") # No thumbnail / contract number configured -> absent. self.assertNotIn("thumbnail", coll.assets) self.assertNotIn("osc:contract-number", ef) @@ -1036,12 +1091,14 @@ def test_build_prr_collection_cf_params_override(self): def test_build_prr_collection_no_themes(self, mock_open_ds): mock_open_ds.return_value = self.dataset gen = OscDatasetStacGenerator( - dataset_id="test.zarr", collection_id="prr-collection", + items_config=[ + ItemConfig(dataset_id="test.zarr", item_id="prr-collection"), + ], workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", - access_link="s3://bucket/test.zarr", + access_link_root="s3://bucket", ) coll = gen.build_prr_collection() self.assertNotIn("themes", coll.extra_fields) @@ -1050,41 +1107,10 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): def test_save_prr_collection_writes_tree(self): with tempfile.TemporaryDirectory() as tmp: - out = self.gen.save_prr_collection(tmp) - self.assertEqual(out, tmp) - - collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join( - tmp, "prr-collection", "items", "prr-collection.json" - ) - self.assertTrue(os.path.isfile(collection_path)) - self.assertTrue(os.path.isfile(item_path)) - - # Files are plain-JSON serialisable (no leftover Python objects). - with open(collection_path) as f: - coll_dict = json.load(f) - with open(item_path) as f: - item_dict = json.load(f) - - self.assertEqual(coll_dict["type"], "Collection") - self.assertEqual(item_dict["type"], "Feature") - - # Structural links are relative; the Item link points at the child. - item_link = next( - lnk for lnk in coll_dict["links"] if lnk["rel"] == "item" - ) - self.assertFalse(item_link["href"].startswith("s3://")) - self.assertTrue(item_link["href"].endswith(".json")) - - # Asset hrefs stay absolute (the data lives on S3). - self.assertEqual( - item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" - ) + with self.assertRaisesRegex(AttributeError, "tzinfo"): + self.gen.save_prr_collection(tmp) def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: - self.gen.save_prr_collection(tmp) - coll = Collection.from_file(os.path.join(tmp, "collection.json")) - items = list(coll.get_items()) - self.assertEqual(len(items), 1) - self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) + with self.assertRaisesRegex(AttributeError, "tzinfo"): + self.gen.save_prr_collection(tmp) diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 79a68dd..099e17d 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -245,7 +245,7 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): / var_file_path ) file_dict[var_file_path] = generator.update_existing_variable_catalog( - full_path, var_id + full_path ) @staticmethod @@ -384,7 +384,9 @@ def publish_dataset( file_dict[project_collection_path] = generator.build_project_collection() # Add child link in the projects base catalog self._update_and_add_to_file_dict( - file_dict, "projects/catalog.json", generator.update_project_base_catalog + file_dict, + "projects/catalog.json", + generator.update_project_base_catalog, ) else: self._update_and_add_to_file_dict( diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index f996b1e..d93d561 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -9,6 +9,7 @@ from datetime import datetime, timezone from typing import Any +import numpy as np import pandas as pd from pystac import ( Asset, @@ -50,8 +51,8 @@ class OscDatasetStacGenerator: Args: collection_id: Unique identifier for the STAC collection. items_config: List of item configuration entries. Each item maps one - dataset_id to one item_id. - access_link_root: Public access link to root of the datasets. + dataset_id to one item_id + access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). osc_region: Geographical region associated with the dataset. @@ -64,12 +65,10 @@ class OscDatasetStacGenerator: def __init__( self, collection_id: str, - items_config: list[ItemConfig] | None = None, - workflow_id: str = "", - workflow_title: str = "", - license_type: str = "", - dataset_id: str | None = None, - item_id: str | None = None, + items_config: list[ItemConfig], + workflow_id: str, + workflow_title: str, + license_type: str, access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", @@ -96,18 +95,6 @@ def __init__( f"collection_id must not contain spaces: {collection_id!r}. " "Use hyphens as word separators (e.g. 'My-Collection-2024')." ) - if not items_config: - if dataset_id: - items_config = [ - ItemConfig( - dataset_id=dataset_id, - item_id=item_id or collection_id, - ) - ] - else: - raise ValueError( - "At least one item configuration must be provided." - ) self.collection_id = collection_id self.items_config = items_config self.workflow_id = workflow_id @@ -116,7 +103,7 @@ def __init__( self.osc_project = osc_project self.osc_project_title = osc_project_title self.osc_project_url = osc_project_url - self.access_link_root = access_link_root or f"s3://deep-esdl-public/" + self.access_link_root = access_link_root or "s3://deep-esdl-public/" self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region @@ -135,15 +122,10 @@ def __init__( self.sci_doi = sci_doi self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - self.datasets = self._load_item_datasets() - self.dataset = self.datasets[0] - self.variables_metadata = self._build_variables_metadata(self.datasets) - def _get_spatial_extent( - self, dataset: xr.Dataset | None = None - ) -> SpatialExtent: + @staticmethod + def _get_spatial_extent(dataset: xr.Dataset) -> SpatialExtent: """Extract spatial extent from the dataset.""" - dataset = dataset or self.dataset if {"lon", "lat"}.issubset(dataset.coords): # For regular gridding lon_min, lon_max = ( @@ -177,11 +159,10 @@ def _get_spatial_extent( "('lon', 'lat' or 'x', 'y')." ) - def _get_temporal_extent( - self, dataset: xr.Dataset | None = None - ) -> TemporalExtent: + @staticmethod + def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: """Extract temporal extent from the dataset.""" - dataset = dataset or self.dataset + dataset = dataset if "time" in dataset.coords: try: # Convert the time bounds to datetime objects @@ -199,70 +180,40 @@ def _normalize_name(name: str | None) -> str | None: return name.replace(" ", "-").replace("_", "-").lower() return None - @staticmethod - def _unique_preserve_order(values: list[str]) -> list[str]: - """Return values without duplicates while preserving the input order.""" - return list(dict.fromkeys(values)) - - def _load_item_datasets(self) -> list[xr.Dataset]: - """Open the configured datasets in item order.""" - return [ - open_dataset(item_config.dataset_id, logger=self.logger) - for item_config in self.items_config - ] - - def _build_variables_metadata(self, datasets: list[xr.Dataset]) -> dict[str, dict]: - """Collect variable metadata across all configured datasets.""" - variables_metadata: dict[str, dict] = {} - for dataset in datasets: - variables_metadata.update(self.get_variables_metadata(dataset)) - return variables_metadata - - def _get_dataset_for_item(self, item_config: ItemConfig) -> xr.Dataset: - """Return the cached dataset that matches ``item_config``.""" - for config, dataset in zip(self.items_config, self.datasets): - if config.dataset_id == item_config.dataset_id: - return dataset - raise ValueError( - f"Dataset {item_config.dataset_id!r} is not part of the generator configuration." - ) - def _build_access_link(self, item_config: ItemConfig) -> str: """Return the asset href for an item, supporting prefix and full URLs.""" - access_link = self.access_link_root - if access_link.endswith("/"): - return f"{access_link.rstrip('/')}/{item_config.dataset_id}" - return access_link + root = self.access_link_root + if root.endswith("/"): + root = root.rstrip("/") + return f"{root}/{item_config.dataset_id}" - def _union_spatial_extent(self, datasets: list[xr.Dataset]) -> SpatialExtent: + @staticmethod + def _union_spatial_extent(items: list[Item]) -> SpatialExtent: """Merge multiple dataset spatial extents into a single bounding box.""" - bboxes = [ - self._get_spatial_extent(dataset).bboxes[0] - for dataset in datasets - ] + bboxes = [item.bbox for item in items] return SpatialExtent( - [[ - min(bbox[0] for bbox in bboxes), - min(bbox[1] for bbox in bboxes), - max(bbox[2] for bbox in bboxes), - max(bbox[3] for bbox in bboxes), - ]] + [ + [ + min(bbox[0] for bbox in bboxes), + min(bbox[1] for bbox in bboxes), + max(bbox[2] for bbox in bboxes), + max(bbox[3] for bbox in bboxes), + ] + ] ) - def _union_temporal_extent(self, datasets: list[xr.Dataset]) -> TemporalExtent: + @staticmethod + def _union_temporal_extent(items: list[Item]) -> TemporalExtent: """Merge multiple dataset temporal extents into a single interval.""" - intervals = [ - self._get_temporal_extent(dataset).intervals[0] - for dataset in datasets - ] - start_values = [interval[0] for interval in intervals if interval[0] is not None] - end_values = [interval[1] for interval in intervals if interval[1] is not None] - if not start_values or not end_values: - raise ValueError("Unable to compute a temporal extent for the datasets.") - return TemporalExtent([[min(start_values), max(end_values)]]) - - def _get_general_metadata(self, dataset: xr.Dataset | None = None) -> dict: - dataset = dataset or self.dataset + intervals = np.array( + [ + [item.properties["start_datetime"], item.properties["end_datetime"]] + for item in items + ] + ).astype(np.datetime64) + return TemporalExtent([[min(intervals[:, 0]), max(intervals[:, 1])]]) + + def _get_general_metadata(self, dataset: xr.Dataset) -> dict: return { "description": ( self.description @@ -284,9 +235,8 @@ def extract_metadata_for_variable(self, variable_data) -> dict: "gcmd_keyword_url": gcmd_keyword_url, } - def get_variable_ids(self, dataset: xr.Dataset | None = None) -> list[str]: + def get_variable_ids(self, dataset: xr.Dataset) -> list[str]: """Get variable IDs for all variables in the dataset.""" - dataset = dataset or self.dataset variable_ids = list(self.get_variables_metadata(dataset).keys()) # Remove 'crs' and 'spatial_ref' from the list if they exist, note that # spatial_ref will be normalized to spatial-ref in variable_ids and skipped. @@ -294,11 +244,8 @@ def get_variable_ids(self, dataset: xr.Dataset | None = None) -> list[str]: var_id for var_id in variable_ids if var_id not in ["crs", "spatial-ref"] ] - def get_variables_metadata( - self, dataset: xr.Dataset | None = None - ) -> dict[str, dict]: + def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" - dataset = dataset or self.dataset variables_metadata = {} for var_name, variable in dataset.data_vars.items(): var_metadata = self.extract_metadata_for_variable(variable) @@ -598,7 +545,7 @@ def update_deepesdl_collection(self, deepesdl_collection_full_path) -> dict: ) return data - def update_existing_variable_catalog(self, var_file_path, var_id) -> dict: + def update_existing_variable_catalog(self, var_file_path) -> dict: """Append child and theme links to an existing variable catalog.""" with open(var_file_path, encoding="utf-8") as f: data = json.load(f) @@ -651,8 +598,8 @@ def build_theme(osc_themes: list[str]) -> Theme: def build_zarr_stac_item( self, - item_config: ItemConfig | str | None = None, - stac_catalog_s3_root: str | None = None, + item_config: ItemConfig, + stac_catalog_s3_root: str, ) -> Item: """Build a single STAC Item representing the entire Zarr store. @@ -667,15 +614,11 @@ def build_zarr_stac_item( Returns: A :class:`pystac.Item` ready to be serialised to S3. """ - if isinstance(item_config, str) and stac_catalog_s3_root is None: - stac_catalog_s3_root = item_config - item_config = None - item_config = item_config or self.items_config[0] - if stac_catalog_s3_root is None: - raise ValueError("stac_catalog_s3_root is required.") - - self.logger.info(f"Building STAC Item for collection '{self.collection_id}'.") - dataset = self._get_dataset_for_item(item_config) + self.logger.info( + f"Building STAC Item {item_config.item_id} " + f"for collection '{self.collection_id}'." + ) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) spatial_extent = self._get_spatial_extent(dataset) temporal_extent = self._get_temporal_extent(dataset) general_metadata = self._get_general_metadata(dataset) @@ -921,7 +864,7 @@ def _get_cube_variables(dataset: xr.Dataset) -> dict[str, dict]: variables[str(name)] = entry return variables - def build_prr_stac_item(self, item_config: ItemConfig | None = None) -> Item: + def build_prr_stac_item(self, item_config: ItemConfig) -> Item: """Build the single datacube Item for the PRR collection. One Item covers the full spatiotemporal extent of the Zarr store. It @@ -930,13 +873,11 @@ def build_prr_stac_item(self, item_config: ItemConfig | None = None) -> Item: (root/parent/collection/self) are left for :meth:`save_prr_collection` to fill in via ``Collection.add_item`` + ``normalize_hrefs``. """ - item_config = item_config or self.items_config[0] - self.logger.info( f"Building PRR STAC Item '{item_config.item_id}' " f"for collection '{self.collection_id}'." ) - dataset = self._get_dataset_for_item(item_config) + dataset = open_dataset(item_config.dataset_id, logger=self.logger) spatial_extent = self._get_spatial_extent(dataset) temporal_extent = self._get_temporal_extent(dataset) general_metadata = self._get_general_metadata(dataset) @@ -1018,24 +959,17 @@ def build_prr_collection(self) -> Collection: items = [ self.build_prr_stac_item(item_config) for item_config in self.items_config ] - spatial_extent = SpatialExtent([item.bbox for item in items]) - temporal_extent = TemporalExtent( - [ - [item.properties["start_datetime"], item.properties["end_datetime"]] - for item in items - ] - ) - variables: list[str] = [] - for dataset in self.datasets: - variables.extend(self.get_variable_ids(dataset)) - variables = self._unique_preserve_order(variables) + spatial_extent = self._union_spatial_extent(items) + temporal_extent = self._union_temporal_extent(items) + dataset_ref = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + variables = self.get_variable_ids(dataset_ref) collection = Collection( id=self.collection_id, description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.description or self.collection_id, + title=self.collection_id, ) osc_extension = OscExtension.add_to(collection) @@ -1212,14 +1146,12 @@ def build_dataset_stac_collection( A pystac.Collection object. """ try: - datasets = self.datasets - spatial_extent = self._union_spatial_extent(datasets) - temporal_extent = self._union_temporal_extent(datasets) - variables: list[str] = [] - for dataset in datasets: - variables.extend(self.get_variable_ids(dataset)) - variables = self._unique_preserve_order(variables) - general_metadata = self._get_general_metadata(datasets[0]) + assert len(self.items_config) == 1 + dataset = open_dataset(self.items_config[0].dataset_id, logger=self.logger) + spatial_extent = self._get_spatial_extent(dataset) + temporal_extent = self._get_temporal_extent(dataset) + variables = self.get_variable_ids(dataset) + general_metadata = self._get_general_metadata(dataset) except ValueError as e: raise ValueError(f"Metadata extraction failed: {e}") From f6e01ccc893378a7fd9cb10813053027b268b8cf Mon Sep 17 00:00:00 2001 From: konstntokas Date: Wed, 12 Aug 2026 16:54:38 +0200 Subject: [PATCH 09/20] mini error found --- a | 1 - b | 1 - deep_code/tools/new.py | 42 ++++++++++++++++++++++++++++++-------- deep_code/tools/prr.py | 2 +- deep_code/tools/publish.py | 4 ++-- 5 files changed, 36 insertions(+), 14 deletions(-) delete mode 100644 a delete mode 100644 b diff --git a/a b/a deleted file mode 100644 index 9e26dfe..0000000 --- a/a +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/b b/b deleted file mode 100644 index 9e26dfe..0000000 --- a/b +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index ee1d651..a17f2a2 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -48,8 +48,14 @@ def generate_workflow_template(output_path: Optional[str] = None) -> str: with open(output_path, "w") as f: f.write("# Workflow Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") - f.write(yaml.dump(workflow_template, sort_keys=False, width=1000, - default_flow_style=False)) + f.write( + yaml.dump( + workflow_template, + sort_keys=False, + width=1000, + default_flow_style=False, + ) + ) @staticmethod def generate_dataset_template(output_path: Optional[str] = None) -> str: @@ -69,7 +75,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: optional = { "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection. Defaults to the existing DeepESDL project collection]", - "osc_themes": ["[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]"], + "osc_themes": [ + "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" + ], "osc_region": "[OPTIONAL: geographical coverage, e.g. Global]", "dataset_status": "[OPTIONAL: ongoing | completed | planned (default: ongoing)]", "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", @@ -77,8 +85,10 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", - "access_link": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public/{dataset_id}]", - "cf_parameter": [{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}], + "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", + "cf_parameter": [ + {"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"} + ], } # Fields used only by `deep-code generate-prr-collection` to build a @@ -90,7 +100,9 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "osc_contract_number": "[PRR-REQUIRED: ESA contract identifier, e.g. 4000114410/15/NL/BW]", "osc_project_website": "[PRR-REQUIRED: project website URL — falls back to osc_project_url / documentation_link]", "osc_project_description": "[PRR-REQUIRED: multi-line project description — falls back to description]", - "osc_missions": ["[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]"], + "osc_missions": [ + "[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]" + ], "thumbnail": "[PRR-REQUIRED: URL to a collection thumbnail image (jpeg/png/webp)]", "thumbnail_media_type": "[OPTIONAL: thumbnail MIME type — guessed from the URL suffix if omitted]", "sci_doi": "[OPTIONAL: dataset DOI, e.g. 10.1000/xyz123 (not a DOI link)]", @@ -114,11 +126,23 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: f.write("# Dataset Configuration Template\n") f.write("# Replace all [PLACEHOLDER] values with your actual data\n\n") f.write("# --- REQUIRED fields ---\n") - f.write(yaml.dump(required, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + required, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write("\n# --- OPTIONAL fields ---\n") - f.write(yaml.dump(optional, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + optional, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write( "\n# --- PRR fields (for `deep-code generate-prr-collection`) ---\n" ) - f.write(yaml.dump(prr, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + yaml.dump( + prr, sort_keys=False, width=1000, default_flow_style=False + ) + ) f.write(stac_catalog_comment) diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 5eff849..fff44aa 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -76,7 +76,7 @@ def generate_prr_collection( workflow_title=config.get("workflow_title") or "", license_type=license_type, documentation_link=config.get("documentation_link"), - access_link=config.get("access_link"), + access_link_root=config.get("access_link_root"), osc_status=config.get("dataset_status") or "ongoing", osc_region=config.get("osc_region") or "Global", osc_themes=config.get("osc_themes"), diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index 099e17d..c0db153 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -295,7 +295,7 @@ def publish_dataset( ) self.collection_id = self.dataset_config.get("collection_id") documentation_link = self.dataset_config.get("documentation_link") - access_link = self.dataset_config.get("access_link") + access_link_root = self.dataset_config.get("access_link_root") dataset_status = self.dataset_config.get("dataset_status") or "ongoing" osc_region = self.dataset_config.get("osc_region") osc_themes = self.dataset_config.get("osc_themes") @@ -333,7 +333,7 @@ def publish_dataset( workflow_title=self.workflow_title, license_type=license_type, documentation_link=documentation_link, - access_link=access_link, + access_link_root=access_link_root, osc_status=dataset_status, osc_region=osc_region, osc_themes=osc_themes, From 7e9f98da514701d8819c88e36c0d48864944d471 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 08:08:05 +0200 Subject: [PATCH 10/20] error in time extent --- .../utils/test_dataset_stac_generator.py | 35 ++++++++++++++++--- deep_code/utils/dataset_stac_generator.py | 7 ++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index a564ef0..b9a5bae 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -1107,10 +1107,37 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): def test_save_prr_collection_writes_tree(self): with tempfile.TemporaryDirectory() as tmp: - with self.assertRaisesRegex(AttributeError, "tzinfo"): - self.gen.save_prr_collection(tmp) + out = self.gen.save_prr_collection(tmp) + self.assertEqual(out, tmp) + + collection_path = os.path.join(tmp, "collection.json") + item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + self.assertTrue(os.path.isfile(collection_path)) + self.assertTrue(os.path.isfile(item_path)) + + # Files are plain-JSON serialisable (no leftover Python objects). + with open(collection_path) as f: + coll_dict = json.load(f) + with open(item_path) as f: + item_dict = json.load(f) + + self.assertEqual(coll_dict["type"], "Collection") + self.assertEqual(item_dict["type"], "Feature") + + # Structural links are relative; the Item link points at the child. + item_link = next(lnk for lnk in coll_dict["links"] if lnk["rel"] == "item") + self.assertFalse(item_link["href"].startswith("s3://")) + self.assertTrue(item_link["href"].endswith(".json")) + + # Asset hrefs stay absolute (the data lives on S3). + self.assertEqual( + item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" + ) def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: - with self.assertRaisesRegex(AttributeError, "tzinfo"): - self.gen.save_prr_collection(tmp) + self.gen.save_prr_collection(tmp) + coll = Collection.from_file(os.path.join(tmp, "collection.json")) + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index d93d561..0a95fda 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -207,10 +207,13 @@ def _union_temporal_extent(items: list[Item]) -> TemporalExtent: """Merge multiple dataset temporal extents into a single interval.""" intervals = np.array( [ - [item.properties["start_datetime"], item.properties["end_datetime"]] + [ + datetime.fromisoformat(item.properties["start_datetime"]), + datetime.fromisoformat(item.properties["end_datetime"]), + ] for item in items ] - ).astype(np.datetime64) + ) return TemporalExtent([[min(intervals[:, 0]), max(intervals[:, 1])]]) def _get_general_metadata(self, dataset: xr.Dataset) -> dict: From 79d371496b9ae4792623932517ad5b5e4577b6ef Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 09:52:54 +0200 Subject: [PATCH 11/20] relative self href --- deep_code/tools/prr.py | 30 ++++++++--------------- deep_code/utils/dataset_stac_generator.py | 19 ++++++++------ 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index fff44aa..9824677 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -41,25 +41,6 @@ def generate_prr_collection( collection_id = config.get("collection_id") license_type = config.get("license_type") items_config_raw = config.get("items_config") - - if items_config_raw: - items_config = [ - ItemConfig( - dataset_id=item_config["dataset_id"], - item_id=item_config["item_id"], - ) - for item_config in items_config_raw - ] - else: - dataset_id = config.get("dataset_id") - item_id = config.get("item_id") or collection_id - if not dataset_id or not collection_id: - raise ValueError( - "At least one item configuration must be provided in the dataset config, " - "along with 'collection_id'." - ) - items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] - if not collection_id: raise ValueError("collection_id is required in the dataset config.") if not license_type: @@ -67,11 +48,20 @@ def generate_prr_collection( "license_type is required in the dataset config. " "Provide an SPDX identifier (e.g. 'CC-BY-4.0', 'MIT', 'proprietary')." ) + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] logger.info(f"Generating PRR STAC collection for '{collection_id}'.") generator = OscDatasetStacGenerator( - items_config=items_config, collection_id=collection_id, + items_config=items_config, workflow_id=config.get("workflow_id") or "", workflow_title=config.get("workflow_title") or "", license_type=license_type, diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 0a95fda..c2d78f9 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -720,6 +720,7 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ + └── collection.json └── items/ ├── {item_id_0}.json # STAC Item (whole Zarr) └── {item_id_1}.json # STAC Item (whole Zarr) @@ -767,13 +768,13 @@ def build_zarr_stac_catalog_file_dict( ) return file_dict - # ------------------------------------------------------------------- # - # PRR (Project Results Repository) style output # - # # - # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # - # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # - # the plain catalog.json/items/*.json under ``{root}/prr/``. # - # ------------------------------------------------------------------- # + # --------------------------------------------------------------------- # + # PRR (Project Results Repository) style output # + # # + # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # + # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # + # the plain {collection_id}/items/{item_id}.json under ``{root}/prr/``. # + # --------------------------------------------------------------------- # @staticmethod def _get_epsg(dataset: xr.Dataset) -> int: @@ -945,6 +946,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: roles=["metadata"], ), ) + item.set_self_href(f"./{self.collection_id}/items/{item_config.item_id}.json") self.logger.info( f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." ) @@ -974,6 +976,7 @@ def build_prr_collection(self) -> Collection: license=self.license_type, title=self.collection_id, ) + collection.set_self_href(f"./{self.collection_id}/collection.json") osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project @@ -1115,8 +1118,8 @@ def save_prr_collection(self, output_dir: str) -> str: ready to inspect or submit to the ESA EarthCODE PRR endpoint:: {output_dir}/ - ├── collection.json # STAC Collection (root) └── {collection_id}/ + └── collection.json # STAC Collection (root) └── items └── {item_id_0}.json # datacube Item (whole Zarr) └── {item_id_1}.json # datacube Item (whole Zarr) From 604f57d81cadc84c6c07080c25470ee1f3b3a20b Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 10:01:26 +0200 Subject: [PATCH 12/20] remove normalize_href --- deep_code/utils/dataset_stac_generator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index c2d78f9..fb85a4e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -958,8 +958,8 @@ def build_prr_collection(self) -> Collection: The Collection carries OSC extension fields (``osc:type``, ``osc:status``, ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list and ``processing:datetime`` — aligning with the ESA EarthCODE PRR - endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a - subsequent ``normalize_hrefs`` produces a self-contained tree. + endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child, so + it produces a self-contained tree. """ items = [ self.build_prr_stac_item(item_config) for item_config in self.items_config @@ -1138,7 +1138,6 @@ def save_prr_collection(self, output_dir: str) -> str: f"'{output_dir}'." ) collection = self.build_prr_collection() - collection.normalize_hrefs(output_dir) collection.save(catalog_type=CatalogType.SELF_CONTAINED) self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir From 904d37846086358bbabd1f9b154e7d7aa59064d6 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 10:17:37 +0200 Subject: [PATCH 13/20] add absolute hrefs --- deep_code/utils/dataset_stac_generator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index fb85a4e..68e2912 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -946,7 +946,7 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: roles=["metadata"], ), ) - item.set_self_href(f"./{self.collection_id}/items/{item_config.item_id}.json") + self.logger.info( f"PRR STAC Item '{item_config.item_id}' built for '{self.collection_id}'." ) @@ -976,7 +976,6 @@ def build_prr_collection(self) -> Collection: license=self.license_type, title=self.collection_id, ) - collection.set_self_href(f"./{self.collection_id}/collection.json") osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project @@ -1138,6 +1137,15 @@ def save_prr_collection(self, output_dir: str) -> str: f"'{output_dir}'." ) collection = self.build_prr_collection() + + # Set absolute HREFs for writing. + collection_dir = f"{output_dir}/{self.collection_id}" + items_dir = f"{collection_dir}/items" + collection.set_self_href(f"{collection_dir}/collection.json") + for item in collection.get_items(): + item.set_self_href(f"{items_dir}/{item.id}.json") + + # Write the collection and its children. collection.save(catalog_type=CatalogType.SELF_CONTAINED) self.logger.info(f"PRR STAC collection written to '{output_dir}'.") return output_dir From 8de42242bf87f1949441ae70192e2b604ad2a355 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 11:31:57 +0200 Subject: [PATCH 14/20] link changed --- a | 1 + b | 1 + deep_code/tools/prr.py | 3 +- deep_code/utils/dataset_stac_generator.py | 108 ++++++++++++++++------ 4 files changed, 84 insertions(+), 29 deletions(-) create mode 100644 a create mode 100644 b diff --git a/a b/a new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/a @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/b b/b new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/b @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 9824677..a264e54 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -74,7 +74,8 @@ def generate_prr_collection( cf_params=config.get("cf_parameter"), visualisation_link=config.get("visualisation_link"), description=config.get("description"), - **({"osc_project": config["osc_project"]} if config.get("osc_project") else {}), + osc_project=config.get("osc_project"), + osc_project_title=config.get("osc_project_title"), osc_project_url=config.get("osc_project_url"), # PRR-specific project metadata. osc_initiative=config.get("osc_initiative") or "earthcode", diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 68e2912..1ead07e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd +import pyproj from pystac import ( Asset, Catalog, @@ -123,42 +124,36 @@ def __init__( self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) - @staticmethod - def _get_spatial_extent(dataset: xr.Dataset) -> SpatialExtent: - """Extract spatial extent from the dataset.""" + def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: + """Extract the spatial extent and return it in EPSG:4326.""" + + epsg = self._get_epsg(dataset) + if {"lon", "lat"}.issubset(dataset.coords): - # For regular gridding - lon_min, lon_max = ( - float(dataset.lon.min()), - float(dataset.lon.max()), - ) - lat_min, lat_max = ( - float(dataset.lat.min()), - float(dataset.lat.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) + x_name, y_name = "lon", "lat" elif {"longitude", "latitude"}.issubset(dataset.coords): - # For regular gridding with 'longitude' and 'latitude' - lon_min, lon_max = ( - float(dataset.longitude.min()), - float(dataset.longitude.max()), - ) - lat_min, lat_max = ( - float(dataset.latitude.min()), - float(dataset.latitude.max()), - ) - return SpatialExtent([[lon_min, lat_min, lon_max, lat_max]]) + x_name, y_name = "longitude", "latitude" elif {"x", "y"}.issubset(dataset.coords): - # For irregular gridding - x_min, x_max = (float(dataset.x.min()), float(dataset.x.max())) - y_min, y_max = (float(dataset.y.min()), float(dataset.y.max())) - return SpatialExtent([[x_min, y_min, x_max, y_max]]) + x_name, y_name = "x", "y" else: raise ValueError( "Dataset does not have recognized spatial coordinates " - "('lon', 'lat' or 'x', 'y')." + "('lon', 'lat'), ('longitude', 'latitude'), or ('x', 'y')." ) + x_min = float(dataset[x_name].min()) + x_max = float(dataset[x_name].max()) + y_min = float(dataset[y_name].min()) + y_max = float(dataset[y_name].max()) + + epsg = self._get_epsg(dataset) + + if epsg != 4326: + transformer = pyproj.Transformer.from_crs(epsg, 4326, always_xy=True) + bbox = transformer.transform_bounds(x_min, y_min, x_max, y_max) + + return SpatialExtent([bbox]) + @staticmethod def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: """Extract temporal extent from the dataset.""" @@ -924,6 +919,34 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: "cube:variables": self._get_cube_variables(dataset), }, ) + item.add_link( + Link( + rel="collection", + target="../collection.json", + media_type="application/json", + ) + ) + item.add_link( + Link( + rel="parent", + target="../collection.json", + media_type="application/json", + ) + ) + item.add_link( + Link( + rel="root", + target="../../", + media_type="application/json", + ) + ) + item.add_link( + Link( + rel="self", + target=f"./{item_config.item_id}.json", + media_type="application/geo+json", + ) + ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. @@ -976,6 +999,35 @@ def build_prr_collection(self) -> Collection: license=self.license_type, title=self.collection_id, ) + collection.stac_version = "1.0.0" + collection.add_link( + Link( + rel="parent", + target="../", + media_type="application/json", + ) + ) + collection.add_link( + Link( + rel="root", + target="../", + media_type="application/json", + ) + ) + collection.add_link( + Link( + rel="self", + target="./collection.json", + media_type="application/json", + ) + ) + collection.add_link( + Link( + rel="items", + target="./items", + media_type="application/geo+json", + ) + ) osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project From 9352ade660bafe8b840e15ab20e30772eb107f0d Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 11:45:40 +0200 Subject: [PATCH 15/20] links missiong --- .../utils/test_dataset_stac_generator.py | 10 ++- deep_code/utils/dataset_stac_generator.py | 62 ++----------------- 2 files changed, 11 insertions(+), 61 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index b9a5bae..04a5a00 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -1110,8 +1110,10 @@ def test_save_prr_collection_writes_tree(self): out = self.gen.save_prr_collection(tmp) self.assertEqual(out, tmp) - collection_path = os.path.join(tmp, "collection.json") - item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + collection_path = os.path.join(tmp, "prr-collection", "collection.json") + item_path = os.path.join( + tmp, "prr-collection", "items", "prr-collection.json" + ) self.assertTrue(os.path.isfile(collection_path)) self.assertTrue(os.path.isfile(item_path)) @@ -1137,7 +1139,9 @@ def test_save_prr_collection_writes_tree(self): def test_save_prr_collection_readable_by_pystac(self): with tempfile.TemporaryDirectory() as tmp: self.gen.save_prr_collection(tmp) - coll = Collection.from_file(os.path.join(tmp, "collection.json")) + coll = Collection.from_file( + os.path.join(tmp, "prr-collection", "collection.json") + ) items = list(coll.get_items()) self.assertEqual(len(items), 1) self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 1ead07e..b0f070e 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -150,9 +150,11 @@ def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: if epsg != 4326: transformer = pyproj.Transformer.from_crs(epsg, 4326, always_xy=True) - bbox = transformer.transform_bounds(x_min, y_min, x_max, y_max) + x_min, y_min, x_max, y_max = transformer.transform_bounds( + x_min, y_min, x_max, y_max + ) - return SpatialExtent([bbox]) + return SpatialExtent([[x_min, y_min, x_max, y_max]]) @staticmethod def _get_temporal_extent(dataset: xr.Dataset) -> TemporalExtent: @@ -919,34 +921,6 @@ def build_prr_stac_item(self, item_config: ItemConfig) -> Item: "cube:variables": self._get_cube_variables(dataset), }, ) - item.add_link( - Link( - rel="collection", - target="../collection.json", - media_type="application/json", - ) - ) - item.add_link( - Link( - rel="parent", - target="../collection.json", - media_type="application/json", - ) - ) - item.add_link( - Link( - rel="root", - target="../../", - media_type="application/json", - ) - ) - item.add_link( - Link( - rel="self", - target=f"./{item_config.item_id}.json", - media_type="application/geo+json", - ) - ) item.stac_extensions.append(DATACUBE_SCHEMA_URI) # Asset hrefs stay absolute (the Zarr lives on S3); only the structural # links become relative when the tree is normalised locally. @@ -1000,34 +974,6 @@ def build_prr_collection(self) -> Collection: title=self.collection_id, ) collection.stac_version = "1.0.0" - collection.add_link( - Link( - rel="parent", - target="../", - media_type="application/json", - ) - ) - collection.add_link( - Link( - rel="root", - target="../", - media_type="application/json", - ) - ) - collection.add_link( - Link( - rel="self", - target="./collection.json", - media_type="application/json", - ) - ) - collection.add_link( - Link( - rel="items", - target="./items", - media_type="application/geo+json", - ) - ) osc_extension = OscExtension.add_to(collection) osc_extension.osc_project = self.osc_project From a1cf6cf963e09f10778a0038d894e8808adb6419 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 12:17:16 +0200 Subject: [PATCH 16/20] allow cf spatial ref --- .../utils/test_dataset_stac_generator.py | 6 ++-- deep_code/utils/dataset_stac_generator.py | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 04a5a00..097ed4a 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -907,8 +907,8 @@ def setUp(self): # ---- helpers ---- - def test_get_epsg_from_spatial_ref(self): - self.assertEqual(self.gen._get_epsg(self.dataset), 3035) + def test_get_crs_from_spatial_ref(self): + self.assertEqual(self.gen._get_crs(self.dataset).to_epsg(), 3035) @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_get_epsg_default_4326(self, mock_open_ds): @@ -930,7 +930,7 @@ def test_get_epsg_default_4326(self, mock_open_ds): workflow_title="WF", license_type="CC-BY-4.0", ) - self.assertEqual(gen._get_epsg(ds), 4326) + self.assertEqual(gen._get_crs(ds).to_epsg(), 4326) def test_get_cube_dimensions(self): dims = self.gen._get_cube_dimensions(self.dataset) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index b0f070e..558371f 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -127,8 +127,6 @@ def __init__( def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: """Extract the spatial extent and return it in EPSG:4326.""" - epsg = self._get_epsg(dataset) - if {"lon", "lat"}.issubset(dataset.coords): x_name, y_name = "lon", "lat" elif {"longitude", "latitude"}.issubset(dataset.coords): @@ -146,10 +144,10 @@ def _get_spatial_extent(self, dataset: xr.Dataset) -> SpatialExtent: y_min = float(dataset[y_name].min()) y_max = float(dataset[y_name].max()) - epsg = self._get_epsg(dataset) + crs = self._get_crs(dataset) - if epsg != 4326: - transformer = pyproj.Transformer.from_crs(epsg, 4326, always_xy=True) + if crs.to_epsg() != 4326: + transformer = pyproj.Transformer.from_crs(crs, 4326, always_xy=True) x_min, y_min, x_max, y_max = transformer.transform_bounds( x_min, y_min, x_max, y_max ) @@ -774,7 +772,7 @@ def build_zarr_stac_catalog_file_dict( # --------------------------------------------------------------------- # @staticmethod - def _get_epsg(dataset: xr.Dataset) -> int: + def _get_crs(dataset: xr.Dataset) -> pyproj.CRS: """Best-effort EPSG code for the dataset, defaulting to 4326. Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or @@ -784,13 +782,16 @@ def _get_epsg(dataset: xr.Dataset) -> int: for var_name in ("spatial_ref", "crs"): if var_name in dataset.variables: attrs = dataset[var_name].attrs - for key in ("spatial_epsg", "epsg", "EPSG"): - if key in attrs: - try: - return int(attrs[key]) - except (TypeError, ValueError): - pass - return 4326 + try: + return pyproj.CRS.from_cf(attrs) + except pyproj.exceptions.CRSError: + for key in ("spatial_epsg", "epsg", "EPSG"): + if key in attrs: + try: + return pyproj.CRS.from_epsg(attrs[key]) + except (TypeError, ValueError): + pass + return pyproj.CRS.from_epsg(4326) def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: """Build the ``cube:dimensions`` object from the dataset coordinates. @@ -800,7 +801,7 @@ def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: dimension, and any remaining index coordinate is emitted as an additional dimension. """ - epsg = self._get_epsg(dataset) + crs = self._get_crs(dataset) x_names = {"lon", "longitude", "x"} y_names = {"lat", "latitude", "y"} dimensions: dict[str, dict] = {} @@ -815,14 +816,14 @@ def _get_cube_dimensions(self, dataset: xr.Dataset) -> dict[str, dict]: "type": "spatial", "axis": "x", "extent": [float(coord.min()), float(coord.max())], - "reference_system": epsg, + "reference_system": crs.to_epsg(), } elif lname in y_names: dimensions[name] = { "type": "spatial", "axis": "y", "extent": [float(coord.min()), float(coord.max())], - "reference_system": epsg, + "reference_system": crs.to_epsg(), } elif lname == "time": time_min = pd.to_datetime(coord.min().values).to_pydatetime() From 334eca55dcdf053bb2966509ab48cf296c2cea80 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 14:50:05 +0200 Subject: [PATCH 17/20] ready for review --- a | 1 - b | 1 - deep_code/tests/tools/test_publish.py | 86 +++++++++++++------ .../utils/test_dataset_stac_generator.py | 8 +- deep_code/tools/new.py | 4 +- deep_code/tools/prr.py | 6 ++ deep_code/tools/publish.py | 58 ++++++------- deep_code/utils/dataset_stac_generator.py | 42 ++++----- deep_code/utils/ogc_api_record.py | 4 +- 9 files changed, 118 insertions(+), 92 deletions(-) delete mode 100644 a delete mode 100644 b diff --git a/a b/a deleted file mode 100644 index 9e26dfe..0000000 --- a/a +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/b b/b deleted file mode 100644 index 9e26dfe..0000000 --- a/b +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 4df212e..9a1cd0f 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -11,6 +11,7 @@ from deep_code.tools.publish import Publisher from deep_code.utils.ogc_api_record import LinksBuilder +from deep_code.utils.dataset_stac_generator import ItemConfig class TestPublisher(unittest.TestCase): @@ -24,7 +25,7 @@ def setUp(self, mock_github_publisher, mock_fsspec_open): # Mock dataset and workflow config files self.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.workflow_config = { "properties": {"title": "Test Workflow"}, @@ -94,7 +95,7 @@ def test_read_config_files(self): # Mock dataset and workflow config files dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } workflow_config = { "properties": {"title": "Test Workflow"}, @@ -138,7 +139,7 @@ def test_publish_mode_routing(self, mock_wf, mock_ds, mock_s3): self.publisher.dataset_config = { "stac_catalog_s3_root": "s3://bucket/stac/", "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], } self.publisher.gh_publisher.publish_files.return_value = "PR_URL" @@ -198,7 +199,6 @@ def test_publish_builds_pr_params(self, mock_wf, mock_ds, mock_s3): assert "dataset: col" in kwargs["pr_title"] assert "workflow/experiment: wf" in kwargs["pr_title"] - # ------------------------------------------------------------------ # S3 credential resolution # ------------------------------------------------------------------ @@ -262,9 +262,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): "s3://bucket/catalog.json": {"type": "Catalog", "id": "test"}, "s3://bucket/col/item.json": {"type": "Feature", "id": "item"}, } - self.publisher._write_stac_catalog_to_s3( - file_dict, {"key": "k", "secret": "s"} - ) + self.publisher._write_stac_catalog_to_s3(file_dict, {"key": "k", "secret": "s"}) self.assertEqual(mock_fsspec_open.call_count, 2) mock_fsspec_open.assert_any_call( @@ -283,9 +281,7 @@ def test_write_stac_catalog_to_s3(self, mock_fsspec_open): def test_publish_writes_zarr_stac_to_s3_when_configured( self, mock_publish_ds, mock_fsspec_open ): - self.publisher.dataset_config["stac_catalog_s3_root"] = ( - "s3://test-bucket/stac/" - ) + self.publisher.dataset_config["stac_catalog_s3_root"] = "s3://test-bucket/stac/" mock_ctx = MagicMock() mock_ctx.__enter__ = MagicMock(return_value=MagicMock()) @@ -332,8 +328,10 @@ def test_publish_dataset_creates_project_collection_when_missing( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -342,9 +340,14 @@ def test_publish_dataset_creates_project_collection_when_missing( # Project collection is missing; all other file_exists calls return True self.publisher.gh_publisher.github_automation.file_exists.return_value = False - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - file_dict = self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + file_dict = self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_called_once() self.assertIn("projects/test-project/collection.json", file_dict) @@ -367,8 +370,10 @@ def test_publish_dataset_updates_project_collection_when_exists( MockGenerator.return_value = mock_gen self.publisher.dataset_config = { - "dataset_id": "test-dataset", "collection_id": "test-collection", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } @@ -377,9 +382,14 @@ def test_publish_dataset_updates_project_collection_when_exists( # Project collection already exists self.publisher.gh_publisher.github_automation.file_exists.return_value = True - with patch.object(self.publisher, "_update_and_add_to_file_dict") as mock_update, \ - patch.object(self.publisher, "_update_variable_catalogs"): - self.publisher.publish_dataset(write_to_file=False) + with patch("deep_code.tools.publish.open_dataset", return_value=object()): + with ( + patch.object( + self.publisher, "_update_and_add_to_file_dict" + ) as mock_update, + patch.object(self.publisher, "_update_variable_catalogs"), + ): + self.publisher.publish_dataset(write_to_file=False) mock_gen.build_project_collection.assert_not_called() @@ -390,7 +400,9 @@ def test_publish_dataset_updates_project_collection_when_exists( def test_publish_dataset_raises_when_stac_root_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", "license_type": "CC-BY-4.0", } with pytest.raises(ValueError, match="stac_catalog_s3_root"): @@ -402,14 +414,24 @@ def test_publish_dataset_raises_when_no_dataset_config(self): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_ids_missing(self): - self.publisher.dataset_config = {"collection_id": "", "dataset_id": ""} - with pytest.raises(ValueError, match="item configuration"): + self.publisher.dataset_config = { + "collection_id": "", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "license_type": "CC-BY-4.0", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", + } + with pytest.raises(ValueError, match="collection_id missing"): self.publisher.publish_dataset(write_to_file=False) def test_publish_dataset_raises_when_license_missing(self): self.publisher.dataset_config = { "collection_id": "test-collection", - "dataset_id": "test-dataset", + "items_config": [{"dataset_id": "test-dataset", "item_id": "test-item"}], + "osc_project": "test-project", + "osc_project_url": "https://example.com/projects/test-project", + "stac_catalog_s3_root": "s3://bucket/stac/test-collection/", } with pytest.raises(ValueError, match="license_type is required"): self.publisher.publish_dataset(write_to_file=False) @@ -433,14 +455,18 @@ def test_update_and_add_to_file_dict(self): file_dict = {} self.publisher.gh_publisher.github_automation.local_clone_dir = "/tmp" update_method = MagicMock(return_value={"key": "value"}) - self.publisher._update_and_add_to_file_dict(file_dict, "some/catalog.json", update_method) + self.publisher._update_and_add_to_file_dict( + file_dict, "some/catalog.json", update_method + ) update_method.assert_called_once() assert any("some/catalog.json" in str(k) for k in file_dict) def test_update_variable_catalogs_creates_new_when_missing(self): mock_gen = MagicMock() mock_gen.variables_metadata = {"var1": {"variable_id": "var1"}} - mock_gen.build_variable_catalog.return_value.to_dict.return_value = {"id": "var1"} + mock_gen.build_variable_catalog.return_value.to_dict.return_value = { + "id": "var1" + } self.publisher.gh_publisher.github_automation.file_exists.return_value = False file_dict = {} @@ -509,8 +535,12 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF @patch("deep_code.tools.publish.WorkflowAsOgcRecord") @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") - def test_generate_workflow_records_mode_all(self, MockRG, MockLinks, MockWF, MockExp): - mock_rg, mock_props, mock_wf_record, mock_exp_record = self._setup_workflow_mocks() + def test_generate_workflow_records_mode_all( + self, MockRG, MockLinks, MockWF, MockExp + ): + mock_rg, mock_props, mock_wf_record, mock_exp_record = ( + self._setup_workflow_mocks() + ) MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record MockExp.return_value = mock_exp_record @@ -585,8 +615,8 @@ class TestParseGithubNotebookUrl: ], ) def test_valid_urls(self, url, repo_url, repo_name, branch, file_path): - got_repo_url, got_repo_name, got_branch, got_file_path = LinksBuilder._parse_github_notebook_url( - url + got_repo_url, got_repo_name, got_branch, got_file_path = ( + LinksBuilder._parse_github_notebook_url(url) ) assert got_repo_url == repo_url assert got_repo_name == repo_name diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 097ed4a..9c9cd42 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -472,7 +472,7 @@ def test_build_zarr_stac_item_structure(self): # Self href self.assertEqual( item.self_href, - "s3://test-bucket/stac/my-collection/mock-collection-id/items/mock-collection-id.json", + "s3://test-bucket/stac/my-collection/mock-collection-id/item.json", ) # Required link rels @@ -538,7 +538,7 @@ def test_build_zarr_stac_catalog_file_dict_content(self): @patch("deep_code.utils.dataset_stac_generator.open_dataset") def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): - """Multiple item configurations produce multiple item files and links.""" + """Only the first item configuration is emitted in the STAC file dict.""" mock_open_ds.return_value = self.mock_dataset gen = OscDatasetStacGenerator( collection_id="multi-collection", @@ -560,14 +560,14 @@ def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): "multi-collection/items/first-item.json", file_dict, ) - self.assertIn( + self.assertNotIn( "s3://test-bucket/stac/multi-collection/" "multi-collection/items/second-item.json", file_dict, ) catalog = file_dict["s3://test-bucket/stac/multi-collection/catalog.json"] item_links = [lnk for lnk in catalog["links"] if lnk["rel"] == "item"] - self.assertEqual(len(item_links), 2) + self.assertEqual(len(item_links), 1) def test_build_dataset_stac_collection_adds_s3_catalog_via_link(self): """A 'via' link (STAC browser) and a 'child' link (HTTPS catalog) are added diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index a17f2a2..7264fb6 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -71,10 +71,11 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", } ], + "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", + "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", } optional = { - "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection. Defaults to the existing DeepESDL project collection]", "osc_themes": [ "[OPTIONAL: OSC theme slug, e.g. land, ocean, atmosphere — auto-lowercased]" ], @@ -83,7 +84,6 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "description": "[OPTIONAL: human-readable description of the dataset. Overrides the description attribute in the Zarr store if set]", "documentation_link": "[OPTIONAL: link to documentation, publication, or handbook]", "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", - "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", "cf_parameter": [ diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index a264e54..93b9e45 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -39,10 +39,16 @@ def generate_prr_collection( config = yaml.safe_load(file) or {} collection_id = config.get("collection_id") + osc_project = config.get("osc_project") + osc_project_url = config.get("osc_project_url") license_type = config.get("license_type") items_config_raw = config.get("items_config") if not collection_id: raise ValueError("collection_id is required in the dataset config.") + if not osc_project: + raise ValueError("osc_project is required in the dataset config.") + if not osc_project_url: + raise ValueError("osc_project_url is required in the dataset config.") if not license_type: raise ValueError( "license_type is required in the dataset config. " diff --git a/deep_code/tools/publish.py b/deep_code/tools/publish.py index c0db153..399f102 100644 --- a/deep_code/tools/publish.py +++ b/deep_code/tools/publish.py @@ -22,7 +22,11 @@ OSC_REPO_OWNER, WORKFLOW_BASE_CATALOG_SELF_HREF, ) -from deep_code.utils.dataset_stac_generator import ItemConfig, OscDatasetStacGenerator +from deep_code.utils.dataset_stac_generator import ( + ItemConfig, + OscDatasetStacGenerator, + open_dataset, +) from deep_code.utils.github_automation import GitHubAutomation from deep_code.utils.ogc_api_record import ( ExperimentAsOgcRecord, @@ -160,6 +164,8 @@ def __init__( # Values that may be set from configs self.collection_id: str | None = None + self.osc_project: str | None = None + self.osc_project_url: str | None = None self.workflow_title: str | None = None self.workflow_id: str | None = None @@ -250,30 +256,17 @@ def _update_variable_catalogs(self, generator, file_dict, variable_ids): @staticmethod def _build_items_config(dataset_config: dict[str, Any]) -> list[ItemConfig]: - """Build item configs from the dataset config. - - Supports the new ``items_config`` list while keeping the legacy - single-item ``dataset_id`` / ``item_id`` fields for backwards - compatibility. - """ + """Build item configs from the dataset config.""" items_config_raw = dataset_config.get("items_config") - if items_config_raw: - items_config = [ - ItemConfig( - dataset_id=item_config["dataset_id"], - item_id=item_config["item_id"], - ) - for item_config in items_config_raw - ] - else: - dataset_id = dataset_config.get("dataset_id") - collection_id = dataset_config.get("collection_id") - item_id = dataset_config.get("item_id") or collection_id - if not dataset_id: - raise ValueError( - "At least one item configuration must be provided in the dataset config." - ) - items_config = [ItemConfig(dataset_id=dataset_id, item_id=item_id)] + if not items_config_raw: + raise ValueError("items_config is required in the dataset config.") + items_config = [ + ItemConfig( + dataset_id=item_config["dataset_id"], + item_id=item_config["item_id"], + ) + for item_config in items_config_raw + ] return items_config def publish_dataset( @@ -302,13 +295,17 @@ def publish_dataset( cf_params = self.dataset_config.get("cf_parameter") license_type = self.dataset_config.get("license_type") visualisation_link = self.dataset_config.get("visualisation_link") - osc_project = self.dataset_config.get("osc_project") + self.osc_project = self.dataset_config.get("osc_project") osc_project_title = self.dataset_config.get("osc_project_title") - osc_project_url = self.dataset_config.get("osc_project_url") + self.osc_project_url = self.dataset_config.get("osc_project_url") description = self.dataset_config.get("description") if not self.collection_id: - raise ValueError("Collection ID missing in the config.") + raise ValueError("collection_id missing in the config.") + if not self.osc_project: + raise ValueError("osc_project missing in the config.") + if not self.osc_project_url: + raise ValueError("osc_project missing in the config.") if not license_type: raise ValueError( @@ -339,15 +336,16 @@ def publish_dataset( osc_themes=osc_themes, cf_params=cf_params, visualisation_link=visualisation_link, - **({"osc_project": osc_project} if osc_project else {}), + osc_project=self.osc_project, osc_project_title=osc_project_title, - osc_project_url=osc_project_url, + osc_project_url=self.osc_project_url, description=description, ) # Store so publish() can reuse it for zarr STAC catalog generation self._last_generator = generator - variable_ids = generator.get_variable_ids() + dataset = open_dataset(generator.items_config[0].dataset_id) + variable_ids = generator.get_variable_ids(dataset) ds_collection = generator.build_dataset_stac_collection( mode=mode, stac_catalog_s3_root=stac_catalog_s3_root ) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 558371f..a2c7a4c 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -646,7 +646,7 @@ def build_zarr_stac_item( now_iso = datetime.now(timezone.utc).isoformat() root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" + item_href = f"{root}/{self.collection_id}/item.json" osc_collection_href = ( "https://esa-earthcode.github.io/open-science-catalog-metadata" f"/products/{self.collection_id}/collection.json" @@ -715,10 +715,7 @@ def build_zarr_stac_catalog_file_dict( {stac_catalog_s3_root}/ ├── catalog.json # STAC Catalog (root) └── {collection_id}/ - └── collection.json - └── items/ - ├── {item_id_0}.json # STAC Item (whole Zarr) - └── {item_id_1}.json # STAC Item (whole Zarr) + └── item.json # STAC Item (whole Zarr) Args: stac_catalog_s3_root: S3 root URL (e.g. ``s3://my-bucket/stac/``). @@ -730,9 +727,9 @@ def build_zarr_stac_catalog_file_dict( f"Building STAC Catalog file dict for collection '{self.collection_id}' " f"at root '{stac_catalog_s3_root}'." ) + root = stac_catalog_s3_root.rstrip("/") catalog_href = f"{root}/catalog.json" - catalog = Catalog( id=f"{self.collection_id}-stac-catalog", description=f"STAC Catalog for {self.collection_id}", @@ -741,27 +738,24 @@ def build_zarr_stac_catalog_file_dict( catalog.add_link( Link(rel="root", target=catalog_href, media_type="application/json") ) - file_dict = {catalog_href: catalog.to_dict(transform_hrefs=False)} - for item_config in self.items_config: - item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) - item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" - catalog.add_link( - Link( - rel="item", - target=f"./{self.collection_id}/items/{item_config.item_id}.json", - media_type="application/json", - title=item_config.item_id, - ) + item_config = self.items_config[0] + item = self.build_zarr_stac_item(item_config, stac_catalog_s3_root) + catalog.add_link( + Link( + rel="item", + target=f"./{self.collection_id}/items/{item_config.item_id}.json", + media_type="application/json", + title=item_config.item_id, ) - file_dict[item_href] = item.to_dict(transform_hrefs=False) - - file_dict[catalog_href] = catalog.to_dict(transform_hrefs=False) - self.logger.info( - "STAC Catalog file dict ready: %s", - ", ".join(file_dict.keys()), ) - return file_dict + item_href = f"{root}/{self.collection_id}/items/{item_config.item_id}.json" + + self.logger.info(f"STAC Catalog file dict ready: {catalog_href}, {item_href}") + return { + catalog_href: catalog.to_dict(transform_hrefs=False), + item_href: item.to_dict(transform_hrefs=False), + } # --------------------------------------------------------------------- # # PRR (Project Results Repository) style output # diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 4a46fb9..9b5c97c 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -110,7 +110,7 @@ def to_dict(self, value_name: str | None = None) -> dict[str, JsonValue]: class LinksBuilder: - def __init__(self, themes: list[str], jupyter_kernel_info: dict[str]): + def __init__(self, themes: list[str], jupyter_kernel_info: dict[str, Any]): self.themes = themes self.jupyter_kernel_info = jupyter_kernel_info self.theme_links = [] @@ -233,7 +233,7 @@ def make_related_link_for_opening_jnb_from_github( jupyter_notebook_url: str, title: str = "Open notebook on the DeepESDL platform", branch_override: str | None = None, - ) -> dict[str, str]: + ) -> list[dict[str, str]]: return [ { "rel": "related", From 9528fcd82aeaf540ba9f5bd11e97dd117de00aa4 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Thu, 13 Aug 2026 15:18:57 +0200 Subject: [PATCH 18/20] ruff --- deep_code/cli/generate_config.py | 4 +- deep_code/cli/main.py | 1 - deep_code/tests/tools/test_publish.py | 1 - .../utils/test_dataset_stac_generator.py | 6 +-- .../tests/utils/test_github_automation.py | 13 +++-- deep_code/tools/new.py | 5 +- deep_code/tools/prr.py | 2 +- deep_code/tools/test.py | 2 +- deep_code/utils/dataset_stac_generator.py | 8 +-- deep_code/utils/helper.py | 18 ++++--- deep_code/utils/ogc_api_record.py | 12 ++--- deep_code/utils/ogc_record_generator.py | 18 +++---- docs/cli.md | 14 ++--- docs/configuration.md | 54 ++++++++++--------- docs/examples.md | 2 +- docs/python-api.md | 2 +- 16 files changed, 83 insertions(+), 79 deletions(-) diff --git a/deep_code/cli/generate_config.py b/deep_code/cli/generate_config.py index ce1c286..ba63d0d 100644 --- a/deep_code/cli/generate_config.py +++ b/deep_code/cli/generate_config.py @@ -18,5 +18,5 @@ help="Output directory for templates", ) def generate_config(output_dir): - TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow_config.yaml") - TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset_config.yaml") + TemplateGenerator.generate_workflow_template(f"{output_dir}/workflow.yaml") + TemplateGenerator.generate_dataset_template(f"{output_dir}/dataset.yaml") diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index ac306f5..dfc8295 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -15,7 +15,6 @@ @click.group() def main(): """Deep Code CLI.""" - pass main.add_command(publish) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index 9a1cd0f..a411779 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -11,7 +11,6 @@ from deep_code.tools.publish import Publisher from deep_code.utils.ogc_api_record import LinksBuilder -from deep_code.utils.dataset_stac_generator import ItemConfig class TestPublisher(unittest.TestCase): diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 9c9cd42..ed5dabc 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -173,8 +173,8 @@ def test_update_product_base_catalog(self): } ], } - import tempfile import json as _json + import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) @@ -211,9 +211,9 @@ def test_update_variable_base_catalog(self): } ], } - import tempfile import json as _json import os + import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) @@ -404,9 +404,9 @@ def test_update_deepesdl_collection(self): } ], } - import tempfile import json as _json import os + import tempfile with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as tmp: _json.dump(base, tmp) diff --git a/deep_code/tests/utils/test_github_automation.py b/deep_code/tests/utils/test_github_automation.py index 671adb3..25ca33b 100644 --- a/deep_code/tests/utils/test_github_automation.py +++ b/deep_code/tests/utils/test_github_automation.py @@ -47,6 +47,7 @@ def test_clone_sync_repository_new(self, mock_run): """ No .git directory → we clone and then ensure upstream remote gets added. """ + # Simulate: "git remote -v" returns nothing so we add 'upstream' def run_side_effect(args, cwd, check, capture_output=False, text=True): if args[:3] == ["git", "remote", "-v"]: @@ -55,8 +56,9 @@ def run_side_effect(args, cwd, check, capture_output=False, text=True): mock_run.side_effect = run_side_effect - with patch.object(Path, "mkdir") as _mk, patch( - "pathlib.Path.exists", side_effect=lambda p=None: False + with ( + patch.object(Path, "mkdir") as _mk, + patch("pathlib.Path.exists", side_effect=lambda p=None: False), ): self.gha.clone_sync_repository() @@ -239,9 +241,10 @@ def test_create_branch(self, mock_run): @patch("subprocess.run") def test_add_file(self, mock_run): mock_run.return_value = make_cp() - with patch.object(Path, "mkdir") as _mk, patch.object( - Path, "write_text" - ) as _wt: + with ( + patch.object(Path, "mkdir") as _mk, + patch.object(Path, "write_text") as _wt, + ): # Ensure .git exists with patch( "pathlib.Path.exists", diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 7264fb6..d39fd07 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -4,14 +4,13 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. -from typing import Optional import yaml class TemplateGenerator: @staticmethod - def generate_workflow_template(output_path: Optional[str] = None) -> str: + def generate_workflow_template(output_path: str | None = None) -> str: """Generate a complete template with all possible keys and placeholder values""" workflow_template = { @@ -58,7 +57,7 @@ def generate_workflow_template(output_path: Optional[str] = None) -> str: ) @staticmethod - def generate_dataset_template(output_path: Optional[str] = None) -> str: + def generate_dataset_template(output_path: str | None = None) -> str: """Generate a complete dataset template with all possible keys and placeholder values""" required = { diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 93b9e45..07c82c3 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -94,6 +94,6 @@ def generate_prr_collection( sci_citation=config.get("sci_citation"), ) - out_dir = output_dir or config.get("prr_output_dir") or f"prr/{collection_id}" + out_dir = output_dir or config.get("prr_output_dir") or "prr" generator.save_prr_collection(out_dir) return out_dir diff --git a/deep_code/tools/test.py b/deep_code/tools/test.py index 5bdf092..b42e21f 100644 --- a/deep_code/tools/test.py +++ b/deep_code/tools/test.py @@ -1,2 +1,2 @@ -""" Execute the application package of a published experiment on a subset of input data +"""Execute the application package of a published experiment on a subset of input data to verify the reproducibility is achieved""" diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index a2c7a4c..51f4433 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -12,6 +12,7 @@ import numpy as np import pandas as pd import pyproj +import xarray as xr from pystac import ( Asset, Catalog, @@ -23,7 +24,6 @@ SpatialExtent, TemporalExtent, ) -import xarray as xr from deep_code.constants import ( CONTACTS_SCHEMA_URI, @@ -70,6 +70,7 @@ def __init__( workflow_id: str, workflow_title: str, license_type: str, + osc_project: str, access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", @@ -77,8 +78,7 @@ def __init__( osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, cf_params: list[dict[str, Any]] | None = None, - osc_project: str = "deep-earth-system-data-lab", - osc_project_title: str = "DeepESDL", + osc_project_title: str = None, osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, @@ -102,7 +102,7 @@ def __init__( self.workflow_title = workflow_title self.license_type = license_type self.osc_project = osc_project - self.osc_project_title = osc_project_title + self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url self.access_link_root = access_link_root or "s3://deep-esdl-public/" self.documentation_link = documentation_link diff --git a/deep_code/utils/helper.py b/deep_code/utils/helper.py index 9452b81..a27a786 100644 --- a/deep_code/utils/helper.py +++ b/deep_code/utils/helper.py @@ -1,6 +1,5 @@ import logging import os -from typing import Optional import xarray as xr from xcube.core.store import new_data_store @@ -25,8 +24,8 @@ def serialize(obj): def open_dataset( dataset_id: str, root: str = "deep-esdl-public", - storage_configs: Optional[list[dict]] = None, - logger: Optional[logging.Logger] = None, + storage_configs: list[dict] | None = None, + logger: logging.Logger | None = None, ) -> xr.Dataset: """Open an xarray dataset from a specified store. @@ -63,10 +62,15 @@ def open_dataset( "root": os.environ.get("S3_USER_STORAGE_BUCKET", root), "storage_options": { "anon": False, - **({ - "key": os.environ["S3_USER_STORAGE_KEY"], - "secret": os.environ["S3_USER_STORAGE_SECRET"], - } if os.environ.get("S3_USER_STORAGE_KEY") and os.environ.get("S3_USER_STORAGE_SECRET") else {}), + **( + { + "key": os.environ["S3_USER_STORAGE_KEY"], + "secret": os.environ["S3_USER_STORAGE_SECRET"], + } + if os.environ.get("S3_USER_STORAGE_KEY") + and os.environ.get("S3_USER_STORAGE_SECRET") + else {} + ), }, }, }, diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 9b5c97c..23eecf7 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any from urllib.parse import quote, urlencode, urlparse from xrlint.util.constructible import MappingConstructible @@ -152,7 +152,7 @@ def build_child_link_to_related_experiment( } ] - def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, Any]]: + def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> list[dict[str, Any]]: return [ { "rel": "application", @@ -171,7 +171,7 @@ def build_link_to_jnb(self, workflow_title, jupyter_nb_url) -> List[Dict[str, An ] @staticmethod - def _parse_github_notebook_url(url: str) -> Tuple[str, str, str, str]: + def _parse_github_notebook_url(url: str) -> tuple[str, str, str, str]: """ Returns (repo_url, repo_name, branch, file_path_in_repo) from a GitHub URL. @@ -257,8 +257,8 @@ def __init__( links: list[dict], linkTemplates: list = [], conformsTo: list[str] = None, - geometry: Optional[Any] = None, - themes: Optional[Any] = None, + geometry: Any | None = None, + themes: Any | None = None, ): if conformsTo is None: conformsTo = [ @@ -332,7 +332,7 @@ def __init__( links: list[dict], linkTemplates=None, conformsTo: list[str] = None, - geometry: Optional[Any] = None, + geometry: Any | None = None, ): if linkTemplates is None: linkTemplates = [] diff --git a/deep_code/utils/ogc_record_generator.py b/deep_code/utils/ogc_record_generator.py index 83bd08a..e300a60 100644 --- a/deep_code/utils/ogc_record_generator.py +++ b/deep_code/utils/ogc_record_generator.py @@ -16,26 +16,24 @@ class OSCWorkflowOGCApiRecordGenerator: - """Generates OGC API record for a workflow - """ + """Generates OGC API record for a workflow""" @staticmethod def build_contact_objects(contacts_list: list[dict]) -> list[Contact]: """Build a list of Contact objects from a list of contact dictionaries. - Uses the inherited MappingConstructible logic to parse each dict. + Uses the inherited MappingConstructible logic to parse each dict. - Args: - contacts_list: A list of dictionaries, each containing contact information. + Args: + contacts_list: A list of dictionaries, each containing contact information. - Returns: - A list of Contact instances. - """ + Returns: + A list of Contact instances. + """ return [Contact.from_value(cdict) for cdict in contacts_list] @staticmethod def build_theme(osc_themes: list[str]) -> Theme: - """Convert each string into a ThemeConcept - """ + """Convert each string into a ThemeConcept""" concepts = [ThemeConcept(id=theme_str) for theme_str in osc_themes] return Theme(concepts=concepts, scheme=OSC_THEME_SCHEME) diff --git a/docs/cli.md b/docs/cli.md index 7b6da7a..6c5c89b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -61,11 +61,12 @@ needs no GitHub credentials or S3 write access (it only reads the Zarr store). I produces a self-contained `Collection → Item → Assets` tree: ``` -prr// -├── collection.json # STAC Collection (root, relative links) -└── / - └── items/ - └── .json # datacube Item covering the whole Zarr store +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) ``` - The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` @@ -83,5 +84,4 @@ still runs but logs a warning listing what is needed for full conformance. See Options: -- `--output-dir/-o`: directory to write the tree into. Defaults to `prr_output_dir` - from the config, then `prr/`. +- `--output-dir/-o`: directory to write the tree into. Defaults to `prr`. diff --git a/docs/configuration.md b/docs/configuration.md index 965ef14..cd4020d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,7 +7,7 @@ deep-code generate-config # writes to current directory deep-code generate-config -o ./configs # custom output folder ``` -This creates `dataset_config.yaml` and `workflow_config.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). +This creates `dataset.yaml` and `workflow.yaml` with all supported fields and placeholder values. Fill them in, then run [`deep-code publish`](cli.md#publish-metadata). The sections below document every field in those templates. @@ -22,6 +22,8 @@ stac_catalog_s3_root: s3://bucket/stac/your-collection/ items_config: - dataset_id: your-dataset.zarr item_id: your-item # no spaces — use hyphens +osc_project: osc-project-name +osc_project_url: osc-project-url # Optional osc_themes: [cryosphere] # must match slugs at opensciencedata.esa.int/themes/catalog — auto-lowercased @@ -29,8 +31,7 @@ osc_region: global dataset_status: completed # ongoing | completed | planned (default: ongoing) documentation_link: https://example.com/docs visualisation_link: https://example.com/viewer # URL to a visualisation of the dataset -osc_project: deep-earth-system-data-lab # defaults to deep-earth-system-data-lab -access_link: s3://bucket/your-dataset.zarr # defaults to s3://deep-esdl-public/{dataset_id} +access_link_root: s3://bucket/ # defaults to s3://deep-esdl-public # CF parameter overrides (list of {name, units, ...} dicts) cf_parameter: @@ -50,23 +51,23 @@ prr_output_dir: ./prr/your-collection ### Field reference -| Field | Required | Description | -|------------------------|---|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | -| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | -| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | -| `dataset_id` | Legacy | Single-item fallback when `items_config` is omitted. | -| `item_id` | Legacy | Single-item fallback when `items_config` is omitted. If omitted there, deep-code falls back to the `collection_id` for compatibility. | -| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | -| `osc_region` | No | Geographical region label (default: `Global`). | -| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | -| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | -| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | -| `documentation_link` | No | URL to dataset documentation. | -| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | -| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | -| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | -| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +| Field | Required | Description | +|------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `collection_id` | Yes | Unique ID for the STAC collection in the OSC catalog. **Must not contain spaces** — use hyphens as word separators (e.g. `My-Collection-2024`). | +| `license_type` | Yes | SPDX license identifier (e.g. `CC-BY-4.0`). Publishing fails if this field is absent. | +| `items_config` | Yes | List of `{dataset_id, item_id}` entries. Use one entry for `publish` today; the generator can emit multiple items when more are provided. | +| `osc_project` | Yes | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). | +| `osc_project_url` | Yes | OSC project url used to link to project. | +| `osc_themes` | No | List of OSC theme slugs (e.g. `[cryosphere, oceans]`). Values are automatically lowercased so `Land` and `land` are equivalent. | +| `osc_region` | No | Geographical region label (default: `Global`). | +| `dataset_status` | No | One of `ongoing`, `completed`, or `planned` (default: `ongoing`). | +| `access_link` | No | Public S3 URL of the Zarr store. Defaults to `s3://deep-esdl-public/{dataset_id}`. | +| `description` | No | Human-readable description of the dataset. Overrides the `description` attribute in the Zarr store; falls back to `"No description available."` if neither is set. | +| `documentation_link` | No | URL to dataset documentation. | +| `visualisation_link` | No | URL to a visualisation of the dataset (e.g. xcube Viewer, WMS). Added as a `visualisation` link with title `"Dataset visualisation"`. | +| `osc_project` | No | OSC project ID this dataset belongs to (e.g. `deep-earth-system-data-lab`). Defaults to `deep-earth-system-data-lab`. | +| `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | +| `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | > The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). @@ -121,11 +122,12 @@ then the boto3 default chain (IAM role, `~/.aws/credentials`). self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): ``` -prr/your-collection/ -├── collection.json # STAC Collection (root, relative links) -└── your-collection/ - └── items/ - └── your-item.json # datacube Item covering the full Zarr store +prr/ +└── {collection_id}/ + └── collection.json # STAC Collection (root) + └── items + └── {item_id_0}.json # datacube Item (whole Zarr) + └── {item_id_1}.json # datacube Item (whole Zarr) ``` - **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, @@ -191,4 +193,4 @@ links: | `contact` | No | List of contact objects with `name`, `organization`, and `links`. | | `links` | No | Additional OGC API record links (e.g. `related`, `describedby`). | -More templates and examples live in `dataset_config.yaml`, `workflow_config.yaml`, and `example-config/`. +More templates and examples live in `dataset.yaml`, `workflow.yaml`, and `example-config/`. diff --git a/docs/examples.md b/docs/examples.md index bf3a596..c1eac29 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,5 +1,5 @@ # Examples -- Templates: `dataset_config.yaml`, `workflow_config.yaml` +- Templates: `dataset.yaml`, `workflow.yaml` - Example configs: `examples/example-config/` - Notebooks on publishing: `examples/notebooks` diff --git a/docs/python-api.md b/docs/python-api.md index e650ba0..2203e93 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -119,7 +119,7 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( ) # file_dict contains: # "s3://bucket/stac/my-collection/catalog.json" -# "s3://bucket/stac/my-collection/my-collection/items/my-item.json" +# "s3://bucket/stac/my-collection/my-collection/item.json" ``` See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the From 7edfc00d8c9de9373c0b2d62ec83608aff1bb259 Mon Sep 17 00:00:00 2001 From: konstntokas Date: Fri, 14 Aug 2026 08:23:37 +0200 Subject: [PATCH 19/20] CI fixed --- deep_code/tests/tools/test_publish.py | 4 ++-- .../tests/utils/test_dataset_stac_generator.py | 8 ++++++++ deep_code/tools/new.py | 4 ++-- deep_code/utils/dataset_stac_generator.py | 4 ++-- deep_code/utils/ogc_api_record.py | 18 ++++++++++-------- examples/notebooks/publish_to_EarthCODE.ipynb | 10 +++++----- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/deep_code/tests/tools/test_publish.py b/deep_code/tests/tools/test_publish.py index a411779..4048b09 100644 --- a/deep_code/tests/tools/test_publish.py +++ b/deep_code/tests/tools/test_publish.py @@ -513,7 +513,7 @@ def _setup_workflow_mocks(self): @patch("deep_code.tools.publish.LinksBuilder") @patch("deep_code.tools.publish.OSCWorkflowOGCApiRecordGenerator") def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF): - mock_rg, mock_props, mock_wf_record, _ = self._setup_workflow_mocks() + mock_rg, _mock_props, mock_wf_record, _ = self._setup_workflow_mocks() MockRG.return_value = mock_rg MockWF.return_value = mock_wf_record @@ -537,7 +537,7 @@ def test_generate_workflow_records_mode_workflow(self, MockRG, MockLinks, MockWF def test_generate_workflow_records_mode_all( self, MockRG, MockLinks, MockWF, MockExp ): - mock_rg, mock_props, mock_wf_record, mock_exp_record = ( + mock_rg, _mock_props, mock_wf_record, mock_exp_record = ( self._setup_workflow_mocks() ) MockRG.return_value = mock_rg diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index ed5dabc..9b0a2d7 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -88,6 +88,7 @@ def setUp(self): access_link_root="s3://mock-bucket/", documentation_link="https://example.com/docs", license_type="proprietary", + osc_project="deep-earth-system-data-lab", osc_status="ongoing", osc_region="Global", osc_themes=["climate", "environment"], @@ -549,6 +550,7 @@ def test_build_zarr_stac_catalog_file_dict_multiple_items(self, mock_open_ds): workflow_id="dummy", workflow_title="test", license_type="proprietary", + osc_project="deep-earth-system-data-lab", ) file_dict = gen.build_zarr_stac_catalog_file_dict( @@ -688,6 +690,7 @@ def _make_generator(mock_ds, collection_id="my-collection", **kwargs): with patch( "deep_code.utils.dataset_stac_generator.open_dataset", return_value=mock_ds ): + kwargs.setdefault("osc_project", "deep-earth-system-data-lab") return OscDatasetStacGenerator( collection_id=collection_id, items_config=[ @@ -739,6 +742,7 @@ def test_collection_id_with_space_raises(self, mock_open_ds): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) @patch("deep_code.utils.dataset_stac_generator.open_dataset") @@ -897,6 +901,7 @@ def setUp(self): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", access_link_root="s3://bucket", osc_status="ongoing", osc_region="Global", @@ -929,6 +934,7 @@ def test_get_epsg_default_4326(self, mock_open_ds): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", ) self.assertEqual(gen._get_crs(ds).to_epsg(), 4326) @@ -1032,6 +1038,7 @@ def test_build_prr_collection_conformant_fields(self): osc_region="Global", osc_themes=["oceans"], osc_missions=["sentinel-3"], + osc_project="deep-earth-system-data-lab", osc_project_description="A detailed project description.", osc_project_website="https://project.example.org", osc_contract_number="4000114410/15/NL/BW", @@ -1098,6 +1105,7 @@ def test_build_prr_collection_no_themes(self, mock_open_ds): workflow_id="wf", workflow_title="WF", license_type="CC-BY-4.0", + osc_project="deep-earth-system-data-lab", access_link_root="s3://bucket", ) coll = gen.build_prr_collection() diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index d39fd07..54bae9b 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -70,8 +70,8 @@ def generate_dataset_template(output_path: str | None = None) -> str: "item_id": "[REQUIRED: unique STAC item id, no spaces — use hyphens]", } ], - "osc_project": "[OPTIONAL: OSC project ID (e.g. deep-earth-system-data-lab). Defaults to deep-earth-system-data-lab]", - "osc_project_url": "[OPTIONAL: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", + "osc_project": "[REQUIRED: OSC project ID (e.g. deep-earth-system-data-lab)]", + "osc_project_url": "[REQUIRED: URL to the project website (e.g. https://deepesdl.eu). Used as the 'via' link in the project collection.]", } optional = { diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 51f4433..9a3df7d 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -78,7 +78,7 @@ def __init__( osc_themes: list[str] | None = None, osc_missions: list[str] | None = None, cf_params: list[dict[str, Any]] | None = None, - osc_project_title: str = None, + osc_project_title: str | None = None, osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, @@ -245,7 +245,7 @@ def get_variable_ids(self, dataset: xr.Dataset) -> list[str]: def get_variables_metadata(self, dataset: xr.Dataset) -> dict[str, dict]: """Extract metadata for all variables in the dataset.""" variables_metadata = {} - for var_name, variable in dataset.data_vars.items(): + for variable in dataset.data_vars.values(): var_metadata = self.extract_metadata_for_variable(variable) variables_metadata[var_metadata.get("variable_id")] = var_metadata return variables_metadata diff --git a/deep_code/utils/ogc_api_record.py b/deep_code/utils/ogc_api_record.py index 23eecf7..9d80152 100644 --- a/deep_code/utils/ogc_api_record.py +++ b/deep_code/utils/ogc_api_record.py @@ -72,13 +72,13 @@ def __init__( description: str, osc_project: str, jupyter_kernel_info: JupyterKernelInfo = None, - osc_workflow: str = None, - updated: str = None, - contacts: list[Contact] = None, - themes: list[Theme] = None, + osc_workflow: str | None = None, + updated: str | None = None, + contacts: list[Contact] | None = None, + themes: list[Theme] | None = None, keywords: list[str] | None = None, formats: list[dict] | None = None, - license: str = None, + license: str | None = None, ): self.created = created self.updated = updated @@ -255,11 +255,13 @@ def __init__( jupyter_notebook_url: str, properties: RecordProperties, links: list[dict], - linkTemplates: list = [], - conformsTo: list[str] = None, + linkTemplates: list | None = None, + conformsTo: list[str] | None = None, geometry: Any | None = None, themes: Any | None = None, ): + if linkTemplates is None: + linkTemplates = [] if conformsTo is None: conformsTo = [ OGC_API_RECORD_SPEC, @@ -331,7 +333,7 @@ def __init__( properties: RecordProperties, links: list[dict], linkTemplates=None, - conformsTo: list[str] = None, + conformsTo: list[str] | None = None, geometry: Any | None = None, ): if linkTemplates is None: diff --git a/examples/notebooks/publish_to_EarthCODE.ipynb b/examples/notebooks/publish_to_EarthCODE.ipynb index eb8d520..2fec356 100644 --- a/examples/notebooks/publish_to_EarthCODE.ipynb +++ b/examples/notebooks/publish_to_EarthCODE.ipynb @@ -234,11 +234,11 @@ "team_store = new_data_store(\n", " \"s3\", \n", " root=S3_USER_STORAGE_BUCKET, \n", - " storage_options=dict(\n", - " anon=False, \n", - " key=S3_USER_STORAGE_KEY, \n", - " secret=S3_USER_STORAGE_SECRET\n", - " )\n", + " storage_options={\n", + " 'anon': False, \n", + " 'key': S3_USER_STORAGE_KEY, \n", + " 'secret': S3_USER_STORAGE_SECRET\n", + " }\n", ")" ] }, From 5ac982fd337cc60d0e103a5779b5a966cf15d63b Mon Sep 17 00:00:00 2001 From: konstntokas Date: Mon, 17 Aug 2026 14:45:59 +0200 Subject: [PATCH 20/20] add collection title --- deep_code/tools/new.py | 1 + deep_code/tools/prr.py | 1 + deep_code/utils/dataset_stac_generator.py | 10 ++++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 54bae9b..5558501 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -85,6 +85,7 @@ def generate_dataset_template(output_path: str | None = None) -> str: "visualisation_link": "[OPTIONAL: URL to a visualisation of the dataset (e.g. xcube Viewer, WMS)]", "osc_project_title": "[OPTIONAL: display title of the OSC project as it appears in the catalog (e.g. DeepESDL). Defaults to a formatted version of osc_project if omitted]", "access_link_root": "[OPTIONAL: public S3 URL of the Zarr store — defaults to s3://deep-esdl-public]", + "collection_title": "[OPTIONAL: title present in the collection and in the STAC browser]", "cf_parameter": [ {"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"} ], diff --git a/deep_code/tools/prr.py b/deep_code/tools/prr.py index 07c82c3..6a65035 100644 --- a/deep_code/tools/prr.py +++ b/deep_code/tools/prr.py @@ -72,6 +72,7 @@ def generate_prr_collection( workflow_title=config.get("workflow_title") or "", license_type=license_type, documentation_link=config.get("documentation_link"), + collection_title=config.get("collection_title"), access_link_root=config.get("access_link_root"), osc_status=config.get("dataset_status") or "ongoing", osc_region=config.get("osc_region") or "Global", diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 9a3df7d..373029b 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -53,6 +53,7 @@ class OscDatasetStacGenerator: collection_id: Unique identifier for the STAC collection. items_config: List of item configuration entries. Each item maps one dataset_id to one item_id + collection_title: Title present in the collection and in the STAC browser access_link_root: Public access link to the root of the datasets. documentation_link: Link to dataset documentation. osc_status: Status of the dataset (e.g., "ongoing"). @@ -71,6 +72,7 @@ def __init__( workflow_title: str, license_type: str, osc_project: str, + collection_title: str | None = None, access_link_root: str | None = None, documentation_link: str | None = None, osc_status: str = "ongoing", @@ -105,10 +107,14 @@ def __init__( self.osc_project_title = osc_project_title or osc_project self.osc_project_url = osc_project_url self.access_link_root = access_link_root or "s3://deep-esdl-public/" + self.collection_title = collection_title or collection_id self.documentation_link = documentation_link self.osc_status = osc_status self.osc_region = osc_region - self.osc_themes = [t.lower() for t in (osc_themes or [])] + if osc_themes is None: + osc_themes = [] + assert isinstance(osc_themes, list) + self.osc_themes = [t.lower() for t in osc_themes] self.osc_missions = osc_missions or [] self.cf_params = cf_params or {} self.visualisation_link = visualisation_link @@ -966,7 +972,7 @@ def build_prr_collection(self) -> Collection: description=self.description or "No description provided.", extent=Extent(spatial=spatial_extent, temporal=temporal_extent), license=self.license_type, - title=self.collection_id, + title=self.collection_title, ) collection.stac_version = "1.0.0"