Try out improving study and experiment handling

.github/workflows/multi-omics-submission-test.yaml

@@ -15,9 +15,10 @@ jobs:
       # Where mars-cli will put .mars/
       MARS_SETTINGS_DIR: ${{ github.workspace }}/mars_settings
 
-      # Paths for MARS repo and ISA template
+      # Paths for MARS repo and ISA templates
       REPOSITORY_SERVICES_PATH: ${{ github.workspace }}/MARS/repository-services
-      ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa.json
+      ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa-multi.json
+      HOLOFOOD_ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/HoloFoodinISA_v1.0.json
 
       # Credentials from GitHub secrets
       WEBIN_USERNAME: ${{ secrets.WEBIN_USERNAME }}
@@ -73,30 +74,43 @@ jobs:
           echo "Services did not become healthy in time ❌"
           exit 1
 
-      - name: Prepare PoC - settings, ISA JSON, credentials, dataFiles
+      - name: Prepare PoC - settings, ISA JSONs, credentials, dataFiles
         run: |
           python scripts/prepare_poc_submission.py
 
-      - name: Run mars-cli submit
+      - name: Submit multi-omics ISA JSON
         run: |
           set -euo pipefail
 
-          # Build list of --data-files arguments from generated .fastq.gz files
           DATA_ARGS=()
-          for f in poc_work/data/*.fastq.gz; do
-            echo "Using data file: ./$f"
-            DATA_ARGS+=( --data-files "./$f" )
+          for data_file in poc_work/multi-omics/data/*; do
+            DATA_ARGS+=( --data-files "$data_file" )
           done
 
-          echo "Running mars-cli with data args: $DATA_ARGS"
+          mars-cli --development submit \
+            --submit-to-metabolights False \
+            --file-transfer ftp \
+            --output "output_multi_omics_${{ github.run_id }}_${{ github.run_attempt }}" \
+            "${DATA_ARGS[@]}" \
+            --credentials-file ./poc_work/credentials.json \
+            poc_work/multi-omics/isa.json
+
+      - name: Submit HoloFood ISA JSON
+        run: |
+          set -euo pipefail
+
+          DATA_ARGS=()
+          for data_file in poc_work/holofood/data/*; do
+            DATA_ARGS+=( --data-files "$data_file" )
+          done
 
           mars-cli --development submit \
             --submit-to-metabolights False \
             --file-transfer ftp \
-            --output "output_${{ github.run_id }}_${{ github.run_attempt }}.json" \
+            --output "output_holofood_${{ github.run_id }}_${{ github.run_attempt }}" \
             "${DATA_ARGS[@]}" \
             --credentials-file ./poc_work/credentials.json \
-            ./poc_work/isa.json
+            poc_work/holofood/isa.json
 
       - name: Collect repository-service logs
         if: always()

README.md

@@ -404,5 +404,3 @@ python mars_cli.py --credential-service-name metabolights  --username-credential
 
 [To set up and run the MARS tool locally using Docker, follow these steps](../repository-services/README.md)
 
-
-

mars_lib/isa_json.py

@@ -484,29 +484,26 @@ def update_isa_json(isa_json: IsaJson, repo_response: RepositoryResponse) -> Isa
 
             add_accession_to_data_file_node(updated_node, accession.value)
         else:
-            # Add study accession to study comments
             updated_study = apply_filter(study_filter, investigation.studies)
-
-            study_accession_comment: Comment = Comment(
+            accession_comment = Comment(
                 name=f"{target_repository}_{target_level}_accession",
                 value=accession.value,
             )
-            updated_study.comments.append(study_accession_comment)
-
-            # Add study accession to assay comments
-            updated_assay = next(
-                filter(
-                    lambda assay: is_assay_for_target_repo(assay, target_repository),
-                    updated_study.assays,
-                ),
-                None,
-            )
-            if updated_assay:
-                assay_accession_comment: Comment = Comment(
-                    name=f"{target_repository}_{target_level}_accession",
-                    value=accession.value,
+
+            if target_level == "study":
+                updated_study.comments.append(accession_comment)
+            else:
+                updated_assay = next(
+                    filter(
+                        lambda assay: is_assay_for_target_repo(
+                            assay, target_repository
+                        ),
+                        updated_study.assays,
+                    ),
+                    None,
                 )
-                updated_assay.comments.append(assay_accession_comment)
+                if updated_assay:
+                    updated_assay.comments.append(accession_comment)
     isa_json.investigation = investigation
     return isa_json
 
@@ -525,10 +522,13 @@ def map_data_files_to_repositories(
     for assay in assays:
         target_repo_comment: Comment = detect_target_repo_comment(assay.comments)
         # This is an effect of everything being optional in the Comment model.
-        # Should we decide to make the value mandatory, this guard clause would not be necessary anymore.
+        # Should we decide to make the value mandatory, this guard clause
+        # would not be necessary anymore.
         if target_repo_comment.value is None:
             raise ValueError(
-                f"At least one assay in the ISA-JSON has no '{TARGET_REPO_KEY}' comment. Mapping not possible. Make sure all assays in the ISA-JSON have this comment!"
+                f"At least one assay in the ISA-JSON has no "
+                f"'{TARGET_REPO_KEY}' comment. Mapping not possible. "
+                f"Make sure all assays in the ISA-JSON have this comment!"
             )
         assay_data_files = [df.name for df in assay.dataFiles]
 
@@ -555,7 +555,11 @@ def map_data_files_to_repositories(
 
     [
         print_and_log(
-            msg=f"File '{rf['short_name']}' could not be mapped to any data file in the ISA-JSON. For this reason, it will be skipped during submission!",
+            msg=(
+                f"File '{rf['short_name']}' could not be mapped to any data "
+                f"file in the ISA-JSON. For this reason, it will be skipped "
+                f"during submission!"
+            ),
             level="warning",
         )
         for rf in remaining_files

scripts/isa_generator.py

@@ -14,19 +14,29 @@ def _timestamp_suffix() -> str:
     return datetime.now(UTC).strftime("%Y%m%d%H%M%S") + "_" + uuid.uuid4().hex[:6]
 
 
-def _write_dummy_fastq_gz(path: Path) -> None:
-    """
-    Write a tiny dummy FASTQ dataset into a .fastq.gz file.
-    Content doesn't matter, as long as it's valid-ish FASTQ text.
-    """
+def _write_dummy_data_file(path: Path) -> None:
+    """Write a tiny dummy file, using gzip for compressed FASTQ extensions."""
     path.parent.mkdir(parents=True, exist_ok=True)
-    with gzip.open(path, "wt") as fh:
-        fh.write(
-            "@read1\n"
-            "ACGTACGTACGTACGT\n"
-            "+\n"
-            "FFFFFFFFFFFFFFFF\n"
-        )
+    content = "@read1\nACGTACGTACGTACGT\n+\nFFFFFFFFFFFFFFFF\n"
+    if path.name.lower().endswith((".fastq.gz", ".fq.gz")):
+        with gzip.open(path, "wt") as fh:
+            fh.write(content)
+    else:
+        path.write_text(content)
+
+
+def _add_suffix_before_extension(file_name: str, suffix: str) -> str:
+    """Insert a unique suffix while preserving the original file extension."""
+    lower_name = file_name.lower()
+    for compound_extension in (".fastq.gz", ".fq.gz"):
+        if lower_name.endswith(compound_extension):
+            extension = file_name[-len(compound_extension) :]
+            return f"{file_name[:-len(compound_extension)]}_{suffix}{extension}"
+
+    extension = Path(file_name).suffix
+    if extension:
+        return f"{file_name[:-len(extension)]}_{suffix}{extension}"
+    return f"{file_name}_{suffix}"
 
 
 def _md5_of_file(path: Path) -> str:
@@ -40,34 +50,33 @@ def _md5_of_file(path: Path) -> str:
     return h.hexdigest()
 
 
-def _get_first_assay(isa_obj: dict[str, Any]) -> dict[str, Any] | None:
+def _get_all_assays(isa_obj: dict[str, Any]) -> List[dict[str, Any]]:
     """
-    Navigate to investigation.studies[0].assays[0] (if present).
+    Return all assays found under investigation.studies[*].assays[*].
     """
     inv = isa_obj.get("investigation")
     if inv is None:
         inv = isa_obj
 
     if not isinstance(inv, dict):
-        return None
+        return []
 
     studies = inv.get("studies") or []
-    if not isinstance(studies, list) or not studies:
-        return None
+    if not isinstance(studies, list):
+        return []
 
-    first_study = studies[0]
-    if not isinstance(first_study, dict):
-        return None
+    assays: List[dict[str, Any]] = []
+    for study in studies:
+        if not isinstance(study, dict):
+            continue
 
-    assays = first_study.get("assays") or []
-    if not isinstance(assays, list) or not assays:
-        return None
+        study_assays = study.get("assays") or []
+        if not isinstance(study_assays, list):
+            continue
 
-    first_assay = assays[0]
-    if not isinstance(first_assay, dict):
-        return None
+        assays.extend(assay for assay in study_assays if isinstance(assay, dict))
 
-    return first_assay
+    return assays
 
 
 def _ensure_comment(comments: List[dict[str, Any]], name: str, value: str) -> None:
@@ -83,90 +92,96 @@ def _ensure_comment(comments: List[dict[str, Any]], name: str, value: str) -> No
 
 
 def _update_datafiles_with_generated_files(
-    assay: dict[str, Any],
+    assays: List[dict[str, Any]],
     data_dir: Path,
-    n_files: int,
+    n_files: int | None,
 ) -> List[Path]:
     """
-    For the first assay, update its dataFiles entries with newly generated .fastq.gz files.
+    Update assay dataFiles entries with newly generated files.
 
-    Behaviour per dataFiles[i] (for i < n_files):
+    Behaviour per touched data file:
 
-      - Generate a unique .fastq.gz file based on the existing 'name':
-          e.g. ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_<suffix>.fastq.gz
-        (if name doesn't end with .fastq.gz, just append _<suffix>.fastq.gz)
+      - Generate a unique file based on the existing 'name' while preserving
+        its extension, for example:
+          ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_<suffix>.fastq.gz
+          reads.R1.fq.gz         -> reads.R1_<suffix>.fq.gz
 
       - Write a dummy FASTQ into that file and compute its MD5.
 
-      - Update the dataFiles[i] object:
+      - Update the dataFiles entry:
           * "name" = new file name
           * in "comments":
               - "file name"       -> new file name
-              - "file type"       -> "fastq"
-              - "file checksum"   -> MD5 of the .fastq.gz
+              - "file type"       -> the preserved extension
+              - "file checksum"   -> MD5 of the generated file
               - "checksum_method" -> "MD5"
             (existing "accession", "submission date", etc. are kept as-is)
     """
-    data_files_json = assay.get("dataFiles") or []
-    if not isinstance(data_files_json, list):
-        return []
-
     generated_paths: List[Path] = []
     suffix = _timestamp_suffix()
+    updated_count = 0
 
-    # We only touch up to n_files entries, and only those that look like objects
-    for i, df_json in enumerate(data_files_json):
-        if i >= n_files:
-            break
-        if not isinstance(df_json, dict):
+    for assay in assays:
+        data_files_json = assay.get("dataFiles") or []
+        if not isinstance(data_files_json, list):
             continue
 
-        original_name = df_json.get("name")
-        if not isinstance(original_name, str) or not original_name:
-            continue
+        for df_json in data_files_json:
+            if n_files is not None and updated_count >= n_files:
+                return generated_paths
+            if not isinstance(df_json, dict):
+                continue
+
+            original_name = df_json.get("name")
+            if not isinstance(original_name, str) or not original_name:
+                continue
+
+            # ISA data file names can include a relative directory (HoloFood uses
+            # FILES/RAW_FILES/...). Upload mapping is filename-based, so generated
+            # PoC files and their ISA names must use only the basename.
+            original_basename = original_name.replace("\\", "/").rsplit("/", 1)[-1]
 
-        # Build unique .fastq.gz name
-        if original_name.endswith(".fastq.gz"):
-            base = original_name[:-len(".fastq.gz")]
-            new_name = f"{base}_{suffix}.fastq.gz"
-        else:
-            new_name = f"{original_name}_{suffix}.fastq.gz"
+            new_name = _add_suffix_before_extension(original_basename, suffix)
 
-        file_path = data_dir / new_name
-        _write_dummy_fastq_gz(file_path)
-        md5 = _md5_of_file(file_path)
+            file_path = data_dir / new_name
+            _write_dummy_data_file(file_path)
+            md5 = _md5_of_file(file_path)
 
-        # Update the JSON entry
-        df_json["name"] = new_name
+            df_json["name"] = new_name
 
-        comments = df_json.get("comments")
-        if not isinstance(comments, list):
-            comments = []
-            df_json["comments"] = comments
+            comments = df_json.get("comments")
+            if not isinstance(comments, list):
+                comments = []
+                df_json["comments"] = comments
 
-        _ensure_comment(comments, "file name", new_name)
-        _ensure_comment(comments, "file type", "fastq")
-        _ensure_comment(comments, "file checksum", md5)
-        _ensure_comment(comments, "checksum_method", "MD5")
-        # DO NOT touch 'accession' or 'submission date' if present
+            _ensure_comment(comments, "file name", new_name)
+            file_type = (
+                "fastq"
+                if new_name.lower().endswith((".fastq.gz", ".fq.gz"))
+                else Path(new_name).suffix.lstrip(".") or "data"
+            )
+            _ensure_comment(comments, "file type", file_type)
+            _ensure_comment(comments, "file checksum", md5)
+            _ensure_comment(comments, "checksum_method", "MD5")
 
-        generated_paths.append(file_path)
+            generated_paths.append(file_path)
+            updated_count += 1
 
     return generated_paths
 
 
 def generate_isa_json_with_data(
     work_dir: Path,
     template_path: Path,
-    n_files: int = 2,
+    n_files: int | None = None,
 ) -> Tuple[Path, List[Path]]:
     """
     PoC behaviour:
 
       1. Load ISA-JSON template from template_path.
-      2. Find investigation.studies[0].assays[0].dataFiles.
-      3. For up to n_files entries in dataFiles, generate UNIQUE .fastq.gz files
-         and update:
+      2. Find all investigation.studies[*].assays[*].dataFiles.
+      3. For each data file (or the first n_files when limited), generate UNIQUE
+         files with preserved extensions and update:
            - dataFiles[i]["name"]
            - dataFiles[i]["comments"] entries for file name, type, checksum, method.
       4. Write the resulting ISA-JSON to work_dir / 'isa.json'.
@@ -177,12 +192,12 @@ def generate_isa_json_with_data(
 
     isa_obj = json.loads(template_path.read_text())
 
-    assay = _get_first_assay(isa_obj)
+    assays = _get_all_assays(isa_obj)
     generated_paths: List[Path] = []
-    if assay is not None:
+    if assays:
         data_dir = work_dir / "data"
         generated_paths = _update_datafiles_with_generated_files(
-            assay=assay,
+            assays=assays,
             data_dir=data_dir,
             n_files=n_files,
         )