Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/multi-omics-submission-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ jobs:
REPOSITORY_SERVICES_PATH: ${{ github.workspace }}/MARS/repository-services
ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/ena-complete.json
HOLOFOOD_ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/HoloFoodinISA_v1.0.json
PTX_ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/i_investigation_PTX_Pilot_Daphnia_magna_valid.json

# Credentials from GitHub secrets
WEBIN_USERNAME: ${{ secrets.WEBIN_USERNAME }}
Expand Down Expand Up @@ -100,6 +101,18 @@ jobs:
--credentials-file ./poc_work/credentials.json \
poc_work/holofood/isa.json

- name: Submit PTX ISA JSON
run: |
set -euo pipefail

mars-cli --development submit \
--submit-to-metabolights False \
--file-transfer ftp \
--output "output_ptx_${{ github.run_id }}_${{ github.run_attempt }}" \
--data-folder poc_work/ptx/data \
--credentials-file ./poc_work/credentials.json \
poc_work/ptx/isa.json

- name: Collect repository-service logs
if: always()
working-directory: ${{ env.REPOSITORY_SERVICES_PATH }}
Expand Down
2 changes: 1 addition & 1 deletion MARS
Submodule MARS updated 26 files
+5 −1 docs/src/pages/repository-services/repository-api.md
+2 −2 repository-services/isajson-biosamples/Dockerfile
+1 −1 repository-services/isajson-biosamples/build.gradle
+45 −83 ...sajson-biosamples/src/main/java/com/elixir/biohackaton/ISAToSRA/biosamples/service/BioSamplesSubmitter.java
+5 −10 ...isajson-biosamples/src/main/java/com/elixir/biohackaton/ISAToSRA/biosamples/service/MarsReceiptService.java
+2 −2 repository-services/isajson-ena/Dockerfile
+1 −1 repository-services/isajson-ena/build.gradle
+77 −0 ...sitory-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/sra/service/DataFileComments.java
+24 −68 ...tory-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/sra/service/MarsReceiptService.java
+30 −33 ...y-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/sra/service/SRAAnalysisXmlCreator.java
+46 −36 ...rvices/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/sra/service/WebinExperimentXmlCreator.java
+28 −19 ...tory-services/isajson-ena/src/main/java/com/elixir/biohackaton/ISAToSRA/sra/service/WebinRunXmlCreator.java
+23 −12 repository-services/isajson-ena/src/test/java/com/elixir/biohackaton/ISAToSRA/EnaReceiptToMarsTest.java
+111 −0 ...itory-services/isajson-ena/src/test/java/com/elixir/biohackaton/ISAToSRA/WebinExperimentXmlCreatorTest.java
+115 −0 repository-services/isajson-ena/src/test/java/com/elixir/biohackaton/ISAToSRA/WebinRunXmlCreatorTest.java
+4 −4 test-data/HoloFoodinISA_v1.0.json
+0 −7 test-data/ISA-PTX-TEST/isatab/a_PTX_Pilot_Daphnia_magna_hilic_assay.txt
+0 −7 test-data/ISA-PTX-TEST/isatab/a_PTX_Pilot_Daphnia_magna_lipid_assay.txt
+0 −13 test-data/ISA-PTX-TEST/isatab/a_PTX_Pilot_Daphnia_magna_rnaseq_assay.txt
+0 −94 test-data/ISA-PTX-TEST/isatab/i_investigation_PTX_Pilot_Daphnia_magna.txt
+0 −7 test-data/ISA-PTX-TEST/isatab/s_PTX_Pilot_Daphnia_magna.txt
+9 −9 test-data/ena-complete.json
+3 −3 test-data/ena-minimal-after-biosamples.json
+3 −3 test-data/ena-minimal-no-accession-category.json
+3 −3 test-data/ena-minimal.json
+241 −81 test-data/i_investigation_PTX_Pilot_Daphnia_magna_valid.json
46 changes: 36 additions & 10 deletions scripts/isa_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,17 @@
import hashlib
from typing import Any

# Data-file comments for the attributes of ENA's FILE element, named like those
# attributes (as the MARS ENA adapter reads them), with the spellings that older
# ISA-JSON used. The older spellings are removed when a comment is set, so a template
# that still uses them does not end up with two conflicting values.
DATA_FILE_COMMENT_LEGACY_NAMES = {
"filename": ("file name", "file_name"),
"filetype": ("file type", "file_type"),
"checksum": ("file checksum",),
"checksum_method": ("checksum type",),
}


def _timestamp_suffix() -> str:
"""Unique suffix for this run (used for data file names)."""
Expand Down Expand Up @@ -84,6 +95,19 @@ def _ensure_comment(comments: list[dict[str, Any]], name: str, value: str) -> No
comments.append({"name": name, "value": value})


def _set_data_file_comment(
comments: list[dict[str, Any]], name: str, value: str
) -> None:
"""Set a data-file comment, dropping any comment under an older spelling."""
legacy_names = DATA_FILE_COMMENT_LEGACY_NAMES[name]
comments[:] = [
c
for c in comments
if not (isinstance(c, dict) and c.get("name") in legacy_names)
]
_ensure_comment(comments, name, value)


def _update_datafiles_with_generated_files(
assays: list[dict[str, Any]],
data_dir: Path,
Expand All @@ -103,12 +127,13 @@ def _update_datafiles_with_generated_files(

- Update the dataFiles entry:
* "name" = new file name
* in "comments":
- "file name" -> new file name
- "file type" -> the preserved extension
- "file checksum" -> MD5 of the generated file
* in "comments", named like the attributes of ENA's FILE element:
- "filename" -> new file name
- "filetype" -> "fastq", or the preserved extension
- "checksum" -> MD5 of the generated file
- "checksum_method" -> "MD5"
(existing "accession", "submission date", etc. are kept as-is)
(older spellings such as "file checksum" are removed; existing
"accession", "submission date", etc. are kept as-is)
"""
generated_paths: list[Path] = []
suffix = _timestamp_suffix()
Expand Down Expand Up @@ -147,15 +172,15 @@ def _update_datafiles_with_generated_files(
comments = []
df_json["comments"] = comments

_ensure_comment(comments, "file name", new_name)
file_type = (
"fastq"
if new_name.lower().endswith((".fastq.gz", ".fq.gz"))
else Path(new_name).suffix.lstrip(".") or "data"
)
_ensure_comment(comments, "file type", file_type)
_ensure_comment(comments, "file checksum", md5)
_ensure_comment(comments, "checksum_method", "MD5")
_set_data_file_comment(comments, "filename", new_name)
_set_data_file_comment(comments, "filetype", file_type)
_set_data_file_comment(comments, "checksum", md5)
_set_data_file_comment(comments, "checksum_method", "MD5")

generated_paths.append(file_path)
updated_count += 1
Expand All @@ -176,7 +201,8 @@ def generate_investigation_with_data(
3. For each data file (or the first n_files when limited), generate UNIQUE
files with preserved extensions and update:
- dataFiles[i]["name"]
- dataFiles[i]["comments"] entries for file name, type, checksum, method.
- dataFiles[i]["comments"] filename, filetype, checksum and
checksum_method.
4. Write the resulting ISA-JSON to work_dir / 'isa.json'.

We DO NOT change other identifiers or comments (including 'target_repository').
Expand Down
27 changes: 18 additions & 9 deletions scripts/prepare_poc_submission.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
- Generate an ISA-JSON from each configured template, where:
* dataFiles entries are updated to point to UNIQUE generated files while
preserving their original extensions
* 'file name', 'file type', 'file checksum', 'checksum_method'
* 'filename', 'filetype', 'checksum', 'checksum_method'
comments are updated accordingly (MD5 over the .fastq.gz)
- Create poc_work/credentials.json from environment variables

Set ``HOLOFOOD_ISA_TEMPLATE_PATH`` to prepare the additional HoloFood
submission alongside ``ISA_TEMPLATE_PATH``.
Set ``HOLOFOOD_ISA_TEMPLATE_PATH`` and/or ``PTX_ISA_TEMPLATE_PATH`` to prepare
the additional HoloFood and PTX submissions alongside ``ISA_TEMPLATE_PATH``.

The GitHub Action (or you, locally) will then call mars-cli using:

Expand All @@ -31,6 +31,13 @@

PROJECT_ROOT = Path(__file__).resolve().parents[1]

# Additional submissions, each prepared when its environment variable is set:
# submission name -> (label, environment variable with the ISA template path)
EXTRA_SUBMISSIONS = {
"holofood": ("HoloFood", "HOLOFOOD_ISA_TEMPLATE_PATH"),
"ptx": ("PTX", "PTX_ISA_TEMPLATE_PATH"),
}


def ensure_settings_ini() -> Path:
parent_str = os.environ.get("MARS_SETTINGS_DIR")
Expand Down Expand Up @@ -126,14 +133,16 @@ def main() -> None:
work_dir.mkdir(parents=True, exist_ok=True)

isa_templates = {"multi-omics": resolve_isa_template()}
holofood_template = os.environ.get("HOLOFOOD_ISA_TEMPLATE_PATH")
if holofood_template:
holofood_path = Path(holofood_template)
if not holofood_path.exists():
for submission_name, (label, env_var) in EXTRA_SUBMISSIONS.items():
template = os.environ.get(env_var)
if not template:
continue
template_path = Path(template)
if not template_path.exists():
raise FileNotFoundError(
f"HoloFood ISA template not found at {holofood_path}"
f"{label} ISA template not found at {template_path}"
)
isa_templates["holofood"] = holofood_path
isa_templates[submission_name] = template_path

prepared_submissions = []
for submission_name, isa_template in isa_templates.items():
Expand Down
6 changes: 3 additions & 3 deletions tests/fixtures/isa_jsons/1_after_biosamples.json
Original file line number Diff line number Diff line change
Expand Up @@ -89,15 +89,15 @@
{
"comments": [
{
"name": "file name",
"name": "filename",
"value": "ENA_TEST2.R2.fastq.gz"
},
{
"name": "file type",
"name": "filetype",
"value": "fastq"
},
{
"name": "file checksum",
"name": "checksum",
"value": "69c903251902c1e0b75331f70e531012"
},
{
Expand Down
5 changes: 1 addition & 4 deletions tests/test_isa_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,7 @@
HOLOFOOD_ISA = "MARS/test-data/HoloFoodinISA_v1.0.json"
# Every sampling step is split over two linked processes: one with only the source as
# input, and one with only the sample as output
PTX_ISA = (
"MARS/test-data/ISA-PTX-TEST/isajson/"
"i_investigation_PTX_Pilot_Daphnia_magna_valid.json"
)
PTX_ISA = "MARS/test-data/i_investigation_PTX_Pilot_Daphnia_magna_valid.json"


def _multi_repository_investigation():
Expand Down
Loading