From d2a031cd7a3633a5f80e83282d557877ed8a367e Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 18:56:11 +0200 Subject: [PATCH 01/41] Try out improving study and experiment handling --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 8d9763e..8e8e5fa 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 8d9763e3501c3243ea5772b1d070e264494200c1 +Subproject commit 8e8e5fac2d9a9e2a35a4d7c4f21f6bfc648d0ecc From 6d50de709b6a1f0b8e6ee5693206aeaa43a74806 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 18:59:18 +0200 Subject: [PATCH 02/41] trigger ci/cd --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 9378a67..cfffc94 100644 --- a/README.md +++ b/README.md @@ -404,5 +404,3 @@ python mars_cli.py --credential-service-name metabolights --username-credential [To set up and run the MARS tool locally using Docker, follow these steps](../repository-services/README.md) - - From 5f00469f9728926f8019dcf5a53420fcfb74c329 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 19:10:10 +0200 Subject: [PATCH 03/41] remove not needed code to prevent double study submission --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 8e8e5fa..e3357d8 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 8e8e5fac2d9a9e2a35a4d7c4f21f6bfc648d0ecc +Subproject commit e3357d8eddd527e4dcc23091bd573cdd1f2d7785 From ce3d9ccb366356d47150a727a9c2e2fc756f04f7 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 19:19:22 +0200 Subject: [PATCH 04/41] DO not add study comments --- mars_lib/isa_json.py | 31 +++++++++++++------------------ tests/test_isa_json.py | 10 +++++----- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/mars_lib/isa_json.py b/mars_lib/isa_json.py index 793376b..1a6d1a4 100644 --- a/mars_lib/isa_json.py +++ b/mars_lib/isa_json.py @@ -484,29 +484,24 @@ def update_isa_json(isa_json: IsaJson, repo_response: RepositoryResponse) -> Isa add_accession_to_data_file_node(updated_node, accession.value) else: - # Add study accession to study comments updated_study = apply_filter(study_filter, investigation.studies) - - study_accession_comment: Comment = Comment( + accession_comment = Comment( name=f"{target_repository}_{target_level}_accession", value=accession.value, ) - updated_study.comments.append(study_accession_comment) - - # Add study accession to assay comments - updated_assay = next( - filter( - lambda assay: is_assay_for_target_repo(assay, target_repository), - updated_study.assays, - ), - None, - ) - if updated_assay: - assay_accession_comment: Comment = Comment( - name=f"{target_repository}_{target_level}_accession", - value=accession.value, + + if target_level == "study": + updated_study.comments.append(accession_comment) + else: + updated_assay = next( + filter( + lambda assay: is_assay_for_target_repo(assay, target_repository), + updated_study.assays, + ), + None, ) - updated_assay.comments.append(assay_accession_comment) + if updated_assay: + updated_assay.comments.append(accession_comment) isa_json.investigation = investigation return isa_json diff --git a/tests/test_isa_json.py b/tests/test_isa_json.py index b358c63..7e293a9 100644 --- a/tests/test_isa_json.py +++ b/tests/test_isa_json.py @@ -245,7 +245,7 @@ def test_update_study_materials_with_accession_categories(): ) -def test_update_study_and_assay_with_ena_study_accession_comment(): +def test_update_assay_only_with_ena_study_accession_comment(): json_path = "tests/fixtures/isa_jsons/1_after_biosamples.json" isa_json = load_isa_json(json_path, False) response_file_path = "tests/fixtures/mars_receipts/ena_success_response.json" @@ -254,10 +254,10 @@ def test_update_study_and_assay_with_ena_study_accession_comment(): updated_isa_json = update_isa_json(isa_json, ena_response) study_comments = updated_isa_json.investigation.studies[0].comments - accession_comment = filter( - lambda x: x.name == "ena_study_accession", study_comments - ) - assert next(accession_comment).value == ena_study_accession_number + accession_comments = [ + comment for comment in study_comments if comment.name == "ena_study_accession" + ] + assert accession_comments == [] ena_assay = next( filter( From 2a5be4308b2ea4732ec5be8c54db257a06bf642a Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 19:23:40 +0200 Subject: [PATCH 05/41] fix test --- tests/test_isa_json.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_isa_json.py b/tests/test_isa_json.py index 7e293a9..13d2b28 100644 --- a/tests/test_isa_json.py +++ b/tests/test_isa_json.py @@ -245,7 +245,7 @@ def test_update_study_materials_with_accession_categories(): ) -def test_update_assay_only_with_ena_study_accession_comment(): +def test_update_study_only_with_ena_study_accession_comment(): json_path = "tests/fixtures/isa_jsons/1_after_biosamples.json" isa_json = load_isa_json(json_path, False) response_file_path = "tests/fixtures/mars_receipts/ena_success_response.json" @@ -254,10 +254,10 @@ def test_update_assay_only_with_ena_study_accession_comment(): updated_isa_json = update_isa_json(isa_json, ena_response) study_comments = updated_isa_json.investigation.studies[0].comments - accession_comments = [ - comment for comment in study_comments if comment.name == "ena_study_accession" - ] - assert accession_comments == [] + accession_comment = filter( + lambda x: x.name == "ena_study_accession", study_comments + ) + assert next(accession_comment).value == ena_study_accession_number ena_assay = next( filter( @@ -267,10 +267,10 @@ def test_update_assay_only_with_ena_study_accession_comment(): None, ) assay_comments = ena_assay.comments - accession_comment = filter( - lambda x: x.name == "ena_study_accession", assay_comments - ) - assert next(accession_comment).value == ena_study_accession_number + accession_comments = [ + comment for comment in assay_comments if comment.name == "ena_study_accession" + ] + assert accession_comments == [] def test_update_datafile_comment_with_accession_comment_present(): From 7d261e28e8cc6a85b0916289cb80ca3d3627ad77 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 19:26:25 +0200 Subject: [PATCH 06/41] make linter content --- mars_lib/isa_json.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mars_lib/isa_json.py b/mars_lib/isa_json.py index 1a6d1a4..53695f5 100644 --- a/mars_lib/isa_json.py +++ b/mars_lib/isa_json.py @@ -495,7 +495,9 @@ def update_isa_json(isa_json: IsaJson, repo_response: RepositoryResponse) -> Isa else: updated_assay = next( filter( - lambda assay: is_assay_for_target_repo(assay, target_repository), + lambda assay: is_assay_for_target_repo( + assay, target_repository + ), updated_study.assays, ), None, @@ -520,10 +522,13 @@ def map_data_files_to_repositories( for assay in assays: target_repo_comment: Comment = detect_target_repo_comment(assay.comments) # This is an effect of everything being optional in the Comment model. - # Should we decide to make the value mandatory, this guard clause would not be necessary anymore. + # Should we decide to make the value mandatory, this guard clause + # would not be necessary anymore. if target_repo_comment.value is None: raise ValueError( - f"At least one assay in the ISA-JSON has no '{TARGET_REPO_KEY}' comment. Mapping not possible. Make sure all assays in the ISA-JSON have this comment!" + f"At least one assay in the ISA-JSON has no " + f"'{TARGET_REPO_KEY}' comment. Mapping not possible. " + f"Make sure all assays in the ISA-JSON have this comment!" ) assay_data_files = [df.name for df in assay.dataFiles] @@ -550,7 +555,11 @@ def map_data_files_to_repositories( [ print_and_log( - msg=f"File '{rf['short_name']}' could not be mapped to any data file in the ISA-JSON. For this reason, it will be skipped during submission!", + msg=( + f"File '{rf['short_name']}' could not be mapped to any data " + f"file in the ISA-JSON. For this reason, it will be skipped " + f"during submission!" + ), level="warning", ) for rf in remaining_files From 67f3bbb70be4c8e84a4a784c6df913f66592f7f2 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 20:49:53 +0200 Subject: [PATCH 07/41] run ci with multi experiment example --- .../multi-omics-submission-test.yaml | 2 +- MARS | 2 +- scripts/isa_generator.py | 118 +++++++++--------- scripts/prepare_poc_submission.py | 1 - 4 files changed, 60 insertions(+), 63 deletions(-) diff --git a/.github/workflows/multi-omics-submission-test.yaml b/.github/workflows/multi-omics-submission-test.yaml index 886686d..7a962d4 100644 --- a/.github/workflows/multi-omics-submission-test.yaml +++ b/.github/workflows/multi-omics-submission-test.yaml @@ -17,7 +17,7 @@ jobs: # Paths for MARS repo and ISA template REPOSITORY_SERVICES_PATH: ${{ github.workspace }}/MARS/repository-services - ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa.json + ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa-multi.json # Credentials from GitHub secrets WEBIN_USERNAME: ${{ secrets.WEBIN_USERNAME }} diff --git a/MARS b/MARS index e3357d8..826d09d 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit e3357d8eddd527e4dcc23091bd573cdd1f2d7785 +Subproject commit 826d09d42d9c10709a278a78b79031e7326569db diff --git a/scripts/isa_generator.py b/scripts/isa_generator.py index 555a79b..cef1a89 100644 --- a/scripts/isa_generator.py +++ b/scripts/isa_generator.py @@ -40,34 +40,33 @@ def _md5_of_file(path: Path) -> str: return h.hexdigest() -def _get_first_assay(isa_obj: dict[str, Any]) -> dict[str, Any] | None: +def _get_all_assays(isa_obj: dict[str, Any]) -> List[dict[str, Any]]: """ - Navigate to investigation.studies[0].assays[0] (if present). + Return all assays found under investigation.studies[*].assays[*]. """ inv = isa_obj.get("investigation") if inv is None: inv = isa_obj if not isinstance(inv, dict): - return None + return [] studies = inv.get("studies") or [] - if not isinstance(studies, list) or not studies: - return None + if not isinstance(studies, list): + return [] - first_study = studies[0] - if not isinstance(first_study, dict): - return None + assays: List[dict[str, Any]] = [] + for study in studies: + if not isinstance(study, dict): + continue - assays = first_study.get("assays") or [] - if not isinstance(assays, list) or not assays: - return None + study_assays = study.get("assays") or [] + if not isinstance(study_assays, list): + continue - first_assay = assays[0] - if not isinstance(first_assay, dict): - return None + assays.extend(assay for assay in study_assays if isinstance(assay, dict)) - return first_assay + return assays def _ensure_comment(comments: List[dict[str, Any]], name: str, value: str) -> None: @@ -83,14 +82,14 @@ def _ensure_comment(comments: List[dict[str, Any]], name: str, value: str) -> No def _update_datafiles_with_generated_files( - assay: dict[str, Any], + assays: List[dict[str, Any]], data_dir: Path, - n_files: int, + n_files: int | None, ) -> List[Path]: """ - For the first assay, update its dataFiles entries with newly generated .fastq.gz files. + Update assay dataFiles entries with newly generated .fastq.gz files. - Behaviour per dataFiles[i] (for i < n_files): + Behaviour per touched data file: - Generate a unique .fastq.gz file based on the existing 'name': e.g. ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_.fastq.gz @@ -98,7 +97,7 @@ def _update_datafiles_with_generated_files( - Write a dummy FASTQ into that file and compute its MD5. - - Update the dataFiles[i] object: + - Update the dataFiles entry: * "name" = new file name * in "comments": - "file name" -> new file name @@ -107,50 +106,49 @@ def _update_datafiles_with_generated_files( - "checksum_method" -> "MD5" (existing "accession", "submission date", etc. are kept as-is) """ - data_files_json = assay.get("dataFiles") or [] - if not isinstance(data_files_json, list): - return [] - generated_paths: List[Path] = [] suffix = _timestamp_suffix() + updated_count = 0 - # We only touch up to n_files entries, and only those that look like objects - for i, df_json in enumerate(data_files_json): - if i >= n_files: - break - if not isinstance(df_json, dict): + for assay in assays: + data_files_json = assay.get("dataFiles") or [] + if not isinstance(data_files_json, list): continue - original_name = df_json.get("name") - if not isinstance(original_name, str) or not original_name: - continue + for df_json in data_files_json: + if n_files is not None and updated_count >= n_files: + return generated_paths + if not isinstance(df_json, dict): + continue + + original_name = df_json.get("name") + if not isinstance(original_name, str) or not original_name: + continue - # Build unique .fastq.gz name - if original_name.endswith(".fastq.gz"): - base = original_name[:-len(".fastq.gz")] - new_name = f"{base}_{suffix}.fastq.gz" - else: - new_name = f"{original_name}_{suffix}.fastq.gz" + if original_name.endswith(".fastq.gz"): + base = original_name[:-len(".fastq.gz")] + new_name = f"{base}_{suffix}.fastq.gz" + else: + new_name = f"{original_name}_{suffix}.fastq.gz" - file_path = data_dir / new_name - _write_dummy_fastq_gz(file_path) - md5 = _md5_of_file(file_path) + file_path = data_dir / new_name + _write_dummy_fastq_gz(file_path) + md5 = _md5_of_file(file_path) - # Update the JSON entry - df_json["name"] = new_name + df_json["name"] = new_name - comments = df_json.get("comments") - if not isinstance(comments, list): - comments = [] - df_json["comments"] = comments + comments = df_json.get("comments") + if not isinstance(comments, list): + comments = [] + df_json["comments"] = comments - _ensure_comment(comments, "file name", new_name) - _ensure_comment(comments, "file type", "fastq") - _ensure_comment(comments, "file checksum", md5) - _ensure_comment(comments, "checksum_method", "MD5") - # DO NOT touch 'accession' or 'submission date' if present + _ensure_comment(comments, "file name", new_name) + _ensure_comment(comments, "file type", "fastq") + _ensure_comment(comments, "file checksum", md5) + _ensure_comment(comments, "checksum_method", "MD5") - generated_paths.append(file_path) + generated_paths.append(file_path) + updated_count += 1 return generated_paths @@ -158,15 +156,15 @@ def _update_datafiles_with_generated_files( def generate_isa_json_with_data( work_dir: Path, template_path: Path, - n_files: int = 2, + n_files: int | None = None, ) -> Tuple[Path, List[Path]]: """ PoC behaviour: 1. Load ISA-JSON template from template_path. - 2. Find investigation.studies[0].assays[0].dataFiles. - 3. For up to n_files entries in dataFiles, generate UNIQUE .fastq.gz files - and update: + 2. Find all investigation.studies[*].assays[*].dataFiles. + 3. For each data file (or the first n_files when limited), generate UNIQUE + .fastq.gz files and update: - dataFiles[i]["name"] - dataFiles[i]["comments"] entries for file name, type, checksum, method. 4. Write the resulting ISA-JSON to work_dir / 'isa.json'. @@ -177,12 +175,12 @@ def generate_isa_json_with_data( isa_obj = json.loads(template_path.read_text()) - assay = _get_first_assay(isa_obj) + assays = _get_all_assays(isa_obj) generated_paths: List[Path] = [] - if assay is not None: + if assays: data_dir = work_dir / "data" generated_paths = _update_datafiles_with_generated_files( - assay=assay, + assays=assays, data_dir=data_dir, n_files=n_files, ) diff --git a/scripts/prepare_poc_submission.py b/scripts/prepare_poc_submission.py index c47456b..23b70f5 100644 --- a/scripts/prepare_poc_submission.py +++ b/scripts/prepare_poc_submission.py @@ -125,7 +125,6 @@ def main() -> None: isa_path, data_files = generate_isa_json_with_data( work_dir=work_dir, template_path=isa_template, - n_files=2, ) cred_path = write_credentials_json(work_dir) From 1409950d5ad6bef3c02c2ac88d711eaff2a8cc54 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 20:53:32 +0200 Subject: [PATCH 08/41] fix illumina model --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 826d09d..7ae7d53 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 826d09d42d9c10709a278a78b79031e7326569db +Subproject commit 7ae7d5397f45e162fcbaf2dfe680a5c11ceffb1e From 245333432977bfbe4121e2711d0f2a3b07e26311 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 21:20:48 +0200 Subject: [PATCH 09/41] add support for paired end reads --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 7ae7d53..7e4caae 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 7ae7d5397f45e162fcbaf2dfe680a5c11ceffb1e +Subproject commit 7e4caaee5f902e7386faa6b932d170d3d4a0defc From b493f681242f24b0684b1f456981c668a5bada99 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 21:33:26 +0200 Subject: [PATCH 10/41] retuen all run accessions in the receipt --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 7e4caae..c389838 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 7e4caaee5f902e7386faa6b932d170d3d4a0defc +Subproject commit c3898381a624f720e147cb9bd693a66bfa648f7e From bf8140e0f1a9fac253af832d64522b45b031ba56 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 21:41:17 +0200 Subject: [PATCH 11/41] add support for multiple experiments/samples --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index c389838..a98589b 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit c3898381a624f720e147cb9bd693a66bfa648f7e +Subproject commit a98589b12e15327fb10ce7bf69419b260e3829ab From a70e6861ff3833e826dd976bbe1bec1c587cbeda Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 21:49:16 +0200 Subject: [PATCH 12/41] add logging --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index a98589b..c01abdd 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit a98589b12e15327fb10ce7bf69419b260e3829ab +Subproject commit c01abddbc7491980cbc955b5a91ef56cdd24e5e6 From 5ffa82b7c64ed97c9f7da40ae07b2ae9de9442e4 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 22:03:36 +0200 Subject: [PATCH 13/41] MORE LOGS! --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index c01abdd..5b938cd 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit c01abddbc7491980cbc955b5a91ef56cdd24e5e6 +Subproject commit 5b938cd944a0b82866f785cda44112de2f0e0f1d From e2fc6047198c5b92d117d9dfff6abb900a911081 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 22:09:44 +0200 Subject: [PATCH 14/41] MORE! --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 5b938cd..c6a3b84 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 5b938cd944a0b82866f785cda44112de2f0e0f1d +Subproject commit c6a3b847a807db8e87d0f8aff85062de1a940f9c From fa88f1177a42c63c23ed259fb66f6bea4851c512 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 22:15:46 +0200 Subject: [PATCH 15/41] final logging --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index c6a3b84..053862f 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit c6a3b847a807db8e87d0f8aff85062de1a940f9c +Subproject commit 053862fa7cdef27f49369e7a4ece597e7644251b From a0492ad092826139090445fbe9f4b55814adb474 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 27 May 2026 22:23:09 +0200 Subject: [PATCH 16/41] fix run accession mapping --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 053862f..96f76a7 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 053862fa7cdef27f49369e7a4ece597e7644251b +Subproject commit 96f76a7fb9e7abbdf241cc8369015dfce2e45cd5 From 93e1d4a22d4fa836d8f5938a7fe892421da6508d Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 5 Jun 2026 11:20:15 +0200 Subject: [PATCH 17/41] read study title and description from study level metadata, not assay --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 96f76a7..d43e4c7 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 96f76a7fb9e7abbdf241cc8369015dfce2e45cd5 +Subproject commit d43e4c751676b35f294a0c03079a3da6351121cb From f9cd415d72c6059bd3f4c6db34111f2cd634e1de Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 5 Jun 2026 11:21:10 +0200 Subject: [PATCH 18/41] read study title and description from study level metadata, not assay --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index d43e4c7..76ccde1 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit d43e4c751676b35f294a0c03079a3da6351121cb +Subproject commit 76ccde1cbc5491a30acfb9e3af41f714da48565f From f1113e2810ff28d4cf41157cdffebb8095226d81 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 5 Jun 2026 12:10:06 +0200 Subject: [PATCH 19/41] read study title and descr from study + remove unused project xml generator --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 76ccde1..e54a397 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 76ccde1cbc5491a30acfb9e3af41f714da48565f +Subproject commit e54a397ae49cd9d9270ed000267bea26ace8e29f From bc0d98451cd13855c767c5abda401c4f730deecb Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 5 Jun 2026 13:14:15 +0200 Subject: [PATCH 20/41] add missing function --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index e54a397..5e719c2 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit e54a397ae49cd9d9270ed000267bea26ace8e29f +Subproject commit 5e719c29e2fb9728e6cc450cb5d91812e952f1d3 From 525565cb875ca7816895a363de539ea45d22afea Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 5 Jun 2026 13:23:59 +0200 Subject: [PATCH 21/41] fix unit tests with new example data --- MARS | 2 +- .../fixtures/mars_receipts/biosamples_success_response.json | 4 ++-- tests/test_isa_json.py | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/MARS b/MARS index 5e719c2..9752f65 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 5e719c29e2fb9728e6cc450cb5d91812e952f1d3 +Subproject commit 9752f656724c9ed53190ac37836e60d42d2f3fd6 diff --git a/tests/fixtures/mars_receipts/biosamples_success_response.json b/tests/fixtures/mars_receipts/biosamples_success_response.json index 1fb4458..d7ef41a 100644 --- a/tests/fixtures/mars_receipts/biosamples_success_response.json +++ b/tests/fixtures/mars_receipts/biosamples_success_response.json @@ -13,7 +13,7 @@ "key": "studies", "where": { "key": "title", - "value": "Arabidopsis thaliana" + "value": "Integrated multi-omics profiling of Arabidopsis thaliana under controlled experimental conditions" } }, { @@ -38,7 +38,7 @@ "key": "studies", "where": { "key": "title", - "value": "Arabidopsis thaliana" + "value": "Integrated multi-omics profiling of Arabidopsis thaliana under controlled experimental conditions" } }, { diff --git a/tests/test_isa_json.py b/tests/test_isa_json.py index 13d2b28..3ef020f 100644 --- a/tests/test_isa_json.py +++ b/tests/test_isa_json.py @@ -35,7 +35,10 @@ def test_load_isa_json(): # Should test the validation process of the ISA JSON file where root has 'investigation' as key. valid_isa_json02 = load_isa_json("MARS/test-data/biosamples-input-isa.json", False) assert len(valid_isa_json02.investigation.studies) == 1 - assert valid_isa_json02.investigation.studies[0].title == "Arabidopsis thaliana" + assert ( + valid_isa_json02.investigation.studies[0].title + == "Integrated multi-omics profiling of Arabidopsis thaliana under controlled experimental conditions" + ) with pytest.raises(ValidationError): load_isa_json("./tests/fixtures/invalid_investigation.json", True) From b581d484deb1b516c637ca103366e79059329991 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 13:58:59 +0200 Subject: [PATCH 22/41] update test file --- MARS | 2 +- .../0_Initial_ISA_JSON_in_model.json | 93 +++---------------- .../isa_jsons/1_after_biosamples.json | 93 +++---------------- tests/fixtures/isa_jsons/2_after_ena.json | 81 +++------------- 4 files changed, 42 insertions(+), 227 deletions(-) diff --git a/MARS b/MARS index 9752f65..057df32 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 9752f656724c9ed53190ac37836e60d42d2f3fd6 +Subproject commit 057df32ff5f176a2a00daa61572aac87187b73da diff --git a/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json b/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json index 0cf74fc..a3a768e 100644 --- a/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json +++ b/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json @@ -82,33 +82,6 @@ "termAccession": "", "termSource": "" } - }, - { - "@id": "#characteristic_category/submission_date_358", - "characteristicType": { - "comments": [], - "annotationValue": "submission date", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/status_359", - "characteristicType": { - "comments": [], - "annotationValue": "status", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/accession_360", - "characteristicType": { - "comments": [], - "annotationValue": "accession", - "termAccession": "", - "termSource": "" - } } ], "dataFiles": [ @@ -158,7 +131,18 @@ { "comments": [], "@id": "#other_material/332", - "characteristics": [], + "characteristics": [ + { + "category": { + "@id": "#characteristic_category/Title_350" + }, + "value": { + "annotationValue": "Arabidopsis leaf amplicon sequencing experiment", + "termAccession": "", + "termSource": "" + } + } + ], "name": "extract 1", "type": "library name", "derivesFrom": [ @@ -173,56 +157,7 @@ { "comments": [], "@id": "#other_material/333", - "characteristics": [ - { - "comments": [], - "category": { - "@id": "#characteristic_category/Title_350" - }, - "value": { - "comments": [], - "annotationValue": "library 1", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/submission_date_358" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/status_359" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/accession_360" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - } - ], + "characteristics": [], "name": "library 1", "type": "library name", "derivesFrom": [ @@ -1062,4 +997,4 @@ "submissionDate": "", "title": "Bob's investigation" } -} \ No newline at end of file +} diff --git a/tests/fixtures/isa_jsons/1_after_biosamples.json b/tests/fixtures/isa_jsons/1_after_biosamples.json index e6d713c..dcb3923 100644 --- a/tests/fixtures/isa_jsons/1_after_biosamples.json +++ b/tests/fixtures/isa_jsons/1_after_biosamples.json @@ -82,33 +82,6 @@ "termAccession": "", "termSource": "" } - }, - { - "@id": "#characteristic_category/submission_date_358", - "characteristicType": { - "comments": [], - "annotationValue": "submission date", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/status_359", - "characteristicType": { - "comments": [], - "annotationValue": "status", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/accession_360", - "characteristicType": { - "comments": [], - "annotationValue": "accession", - "termAccession": "", - "termSource": "" - } } ], "dataFiles": [ @@ -158,7 +131,18 @@ { "comments": [], "@id": "#other_material/332", - "characteristics": [], + "characteristics": [ + { + "category": { + "@id": "#characteristic_category/Title_350" + }, + "value": { + "annotationValue": "Arabidopsis leaf amplicon sequencing experiment", + "termAccession": "", + "termSource": "" + } + } + ], "name": "extract 1", "type": "library name", "derivesFrom": [ @@ -173,56 +157,7 @@ { "comments": [], "@id": "#other_material/333", - "characteristics": [ - { - "comments": [], - "category": { - "@id": "#characteristic_category/Title_350" - }, - "value": { - "comments": [], - "annotationValue": "library 1", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/submission_date_358" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/status_359" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/accession_360" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - } - ], + "characteristics": [], "name": "library 1", "type": "library name", "derivesFrom": [ @@ -1080,4 +1015,4 @@ "submissionDate": "", "title": "Bob's investigation" } -} \ No newline at end of file +} diff --git a/tests/fixtures/isa_jsons/2_after_ena.json b/tests/fixtures/isa_jsons/2_after_ena.json index db856ae..b3a9982 100644 --- a/tests/fixtures/isa_jsons/2_after_ena.json +++ b/tests/fixtures/isa_jsons/2_after_ena.json @@ -87,24 +87,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/submission_date_358", - "characteristicType": { - "comments": [], - "annotationValue": "submission date", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/status_359", - "characteristicType": { - "comments": [], - "annotationValue": "status", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/accession_360", "characteristicType": { @@ -163,6 +145,18 @@ "comments": [], "@id": "#other_material/332", "characteristics": [ + { + "category": { + "@id": "#characteristic_category/Title_350" + }, + "value": { + "annotationValue": "Arabidopsis leaf amplicon sequencing experiment", + "termAccession": "", + "termSource": "", + "comments": [] + }, + "comments": [] + }, { "comments": [], "@id": "#material_attribute_value/accession_360", @@ -197,55 +191,6 @@ "comments": [], "@id": "#other_material/333", "characteristics": [ - { - "comments": [], - "category": { - "@id": "#characteristic_category/Title_350" - }, - "value": { - "comments": [], - "annotationValue": "library 1", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/submission_date_358" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/status_359" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "@id": "#material_attribute_value/accession_360", - "category": { - "@id": "#characteristic_category/accession_360", - "characteristicType": { - "comments": [], - "annotationValue": "accession", - "termAccession": "", - "termSource": "" - } - } - }, { "comments": [], "category": { @@ -1115,4 +1060,4 @@ "submissionDate": "", "title": "Bob's investigation" } -} \ No newline at end of file +} From d9a0bb2397d6b962c8946d94f2ce2fa485500682 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 15:46:07 +0200 Subject: [PATCH 23/41] include betetr experiment builder --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 057df32..5071194 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 057df32ff5f176a2a00daa61572aac87187b73da +Subproject commit 5071194b4d78ba7c877b00eb05b50a1a1d5dd007 From 117c8ba16dadeb67c5eea691f7ec3bc2bb7b73b9 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 16:09:40 +0200 Subject: [PATCH 24/41] apply sample builder changes --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 5071194..b93dd42 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 5071194b4d78ba7c877b00eb05b50a1a1d5dd007 +Subproject commit b93dd42f4851e30630d0ceaaf9fc58a4d2380fee From 1c50036df235b263f2c925fcdbb328cc6ed7e46e Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 16:21:13 +0200 Subject: [PATCH 25/41] apply relation ship samples fix --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index b93dd42..55ee59d 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit b93dd42f4851e30630d0ceaaf9fc58a4d2380fee +Subproject commit 55ee59d67d0171223c4049db71119c27f383fcfa From 3d65f454a00230527c3339fc0fab9485e6d7127b Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 16:30:53 +0200 Subject: [PATCH 26/41] adding some extra tests --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 55ee59d..f20687f 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 55ee59d67d0171223c4049db71119c27f383fcfa +Subproject commit f20687fa3f6c86cfa530b00f016f9a88514aacbd From 2fabb9db180f5d7a7e8864923b8515167e77dc3c Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 16:44:02 +0200 Subject: [PATCH 27/41] adapt tests to richer examples --- MARS | 2 +- .../0_Initial_ISA_JSON_in_model.json | 162 +++++++++++++++--- .../isa_jsons/1_after_biosamples.json | 159 ++++++++++++++--- tests/fixtures/isa_jsons/2_after_ena.json | 159 ++++++++++++++--- 4 files changed, 402 insertions(+), 80 deletions(-) diff --git a/MARS b/MARS index f20687f..a65f10a 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit f20687fa3f6c86cfa530b00f016f9a88514aacbd +Subproject commit a65f10a2f9b5ebccf0ca8dbaaf463ae9a2e10666 diff --git a/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json b/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json index a3a768e..5e553a5 100644 --- a/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json +++ b/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json @@ -459,15 +459,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/cell_type_321", - "characteristicType": { - "comments": [], - "annotationValue": "cell_type", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/dev_stage_322", "characteristicType": { @@ -539,6 +530,38 @@ "termAccession": "", "termSource": "" } + }, + { + "@id": "#characteristic_category/genotype_330", + "characteristicType": { + "annotationValue": "genotype", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/growth_condition_331", + "characteristicType": { + "annotationValue": "growth condition", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/organism_part_332", + "characteristicType": { + "annotationValue": "organism part", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/sample_description_333", + "characteristicType": { + "annotationValue": "sample description", + "termAccession": "", + "termSource": "" + } } ], "description": "Nucleic acid sequencing and metabolomics and proteomics of Arabidopsis thaliana in specific experimental conditions to test a specific hypothesis.\r\n", @@ -599,18 +622,6 @@ "termSource": "" } }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/cell_type_321" - }, - "value": { - "comments": [], - "annotationValue": "na", - "termAccession": "", - "termSource": "" - } - }, { "comments": [], "category": { @@ -618,7 +629,7 @@ }, "value": { "comments": [], - "annotationValue": "budding", + "annotationValue": "vegetative rosette stage", "termAccession": "", "termSource": "" } @@ -630,7 +641,7 @@ }, "value": { "comments": [], - "annotationValue": "01/01/2022", + "annotationValue": "2022-01-01", "termAccession": "", "termSource": "" } @@ -642,7 +653,7 @@ }, "value": { "comments": [], - "annotationValue": "seed", + "annotationValue": "whole plant", "termAccession": "", "termSource": "" } @@ -695,6 +706,26 @@ "termSource": "" } }, + { + "category": { + "@id": "#characteristic_category/genotype_330" + }, + "value": { + "annotationValue": "Col-0", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/growth_condition_331" + }, + "value": { + "annotationValue": "16 h light / 8 h dark growth chamber", + "termAccession": "", + "termSource": "" + } + }, { "comments": [], "category": { @@ -722,7 +753,28 @@ "comments": [], "@id": "#sample/331", "name": "leaf 1", - "characteristics": [], + "characteristics": [ + { + "category": { + "@id": "#characteristic_category/organism_part_332" + }, + "value": { + "annotationValue": "leaf", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/sample_description_333" + }, + "value": { + "annotationValue": "young rosette leaf collected for DNA extraction", + "termAccession": "", + "termSource": "" + } + } + ], "factorValues": [], "derivesFrom": [ { @@ -796,7 +848,38 @@ "derivesFrom": [] } ], - "parameterValues": [], + "parameterValues": [ + { + "category": { + "@id": "#parameter/sample_collection_isolation_source" + }, + "value": { + "annotationValue": "leaf tissue", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_method" + }, + "value": { + "annotationValue": "sterile scalpel excision", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_preservation" + }, + "value": { + "annotationValue": "flash frozen in liquid nitrogen", + "termAccession": "", + "termSource": "" + } + } + ], "performer": "", "previousProcess": { "comments": [], @@ -823,7 +906,32 @@ ], "description": "", "name": "sample collection", - "parameters": [], + "parameters": [ + { + "@id": "#parameter/sample_collection_isolation_source", + "parameterName": { + "annotationValue": "isolation_source", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_method", + "parameterName": { + "annotationValue": "collection method", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_preservation", + "parameterName": { + "annotationValue": "sample preservation", + "termAccession": "", + "termSource": "" + } + } + ], "protocolType": { "comments": [], "annotationValue": "sample collection", diff --git a/tests/fixtures/isa_jsons/1_after_biosamples.json b/tests/fixtures/isa_jsons/1_after_biosamples.json index dcb3923..fb25781 100644 --- a/tests/fixtures/isa_jsons/1_after_biosamples.json +++ b/tests/fixtures/isa_jsons/1_after_biosamples.json @@ -459,15 +459,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/cell_type_321", - "characteristicType": { - "comments": [], - "annotationValue": "cell_type", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/dev_stage_322", "characteristicType": { @@ -539,6 +530,38 @@ "termAccession": "", "termSource": "" } + }, + { + "@id": "#characteristic_category/genotype_330", + "characteristicType": { + "annotationValue": "genotype", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/growth_condition_331", + "characteristicType": { + "annotationValue": "growth condition", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/organism_part_332", + "characteristicType": { + "annotationValue": "organism part", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/sample_description_333", + "characteristicType": { + "annotationValue": "sample description", + "termAccession": "", + "termSource": "" + } } ], "description": "Nucleic acid sequencing and metabolomics and proteomics of Arabidopsis thaliana in specific experimental conditions to test a specific hypothesis.\r\n", @@ -599,18 +622,6 @@ "termSource": "" } }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/cell_type_321" - }, - "value": { - "comments": [], - "annotationValue": "na", - "termAccession": "", - "termSource": "" - } - }, { "comments": [], "category": { @@ -618,7 +629,7 @@ }, "value": { "comments": [], - "annotationValue": "budding", + "annotationValue": "vegetative rosette stage", "termAccession": "", "termSource": "" } @@ -630,7 +641,7 @@ }, "value": { "comments": [], - "annotationValue": "01/01/2022", + "annotationValue": "2022-01-01", "termAccession": "", "termSource": "" } @@ -642,7 +653,7 @@ }, "value": { "comments": [], - "annotationValue": "seed", + "annotationValue": "whole plant", "termAccession": "", "termSource": "" } @@ -695,6 +706,26 @@ "termSource": "" } }, + { + "category": { + "@id": "#characteristic_category/genotype_330" + }, + "value": { + "annotationValue": "Col-0", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/growth_condition_331" + }, + "value": { + "annotationValue": "16 h light / 8 h dark growth chamber", + "termAccession": "", + "termSource": "" + } + }, { "comments": [], "category": { @@ -739,6 +770,26 @@ "@id": "#ontology_annotation/accession_#sample/331", "annotationValue": "SAMEA131504584" } + }, + { + "category": { + "@id": "#characteristic_category/organism_part_332" + }, + "value": { + "annotationValue": "leaf", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/sample_description_333" + }, + "value": { + "annotationValue": "young rosette leaf collected for DNA extraction", + "termAccession": "", + "termSource": "" + } } ], "factorValues": [], @@ -814,7 +865,38 @@ "derivesFrom": [] } ], - "parameterValues": [], + "parameterValues": [ + { + "category": { + "@id": "#parameter/sample_collection_isolation_source" + }, + "value": { + "annotationValue": "leaf tissue", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_method" + }, + "value": { + "annotationValue": "sterile scalpel excision", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_preservation" + }, + "value": { + "annotationValue": "flash frozen in liquid nitrogen", + "termAccession": "", + "termSource": "" + } + } + ], "performer": "", "previousProcess": { "comments": [], @@ -841,7 +923,32 @@ ], "description": "", "name": "sample collection", - "parameters": [], + "parameters": [ + { + "@id": "#parameter/sample_collection_isolation_source", + "parameterName": { + "annotationValue": "isolation_source", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_method", + "parameterName": { + "annotationValue": "collection method", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_preservation", + "parameterName": { + "annotationValue": "sample preservation", + "termAccession": "", + "termSource": "" + } + } + ], "protocolType": { "comments": [], "annotationValue": "sample collection", diff --git a/tests/fixtures/isa_jsons/2_after_ena.json b/tests/fixtures/isa_jsons/2_after_ena.json index b3a9982..77669d1 100644 --- a/tests/fixtures/isa_jsons/2_after_ena.json +++ b/tests/fixtures/isa_jsons/2_after_ena.json @@ -504,15 +504,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/cell_type_321", - "characteristicType": { - "comments": [], - "annotationValue": "cell_type", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/dev_stage_322", "characteristicType": { @@ -584,6 +575,38 @@ "termAccession": "", "termSource": "" } + }, + { + "@id": "#characteristic_category/genotype_330", + "characteristicType": { + "annotationValue": "genotype", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/growth_condition_331", + "characteristicType": { + "annotationValue": "growth condition", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/organism_part_332", + "characteristicType": { + "annotationValue": "organism part", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/sample_description_333", + "characteristicType": { + "annotationValue": "sample description", + "termAccession": "", + "termSource": "" + } } ], "description": "Nucleic acid sequencing and metabolomics and proteomics of Arabidopsis thaliana in specific experimental conditions to test a specific hypothesis.\r\n", @@ -644,18 +667,6 @@ "termSource": "" } }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/cell_type_321" - }, - "value": { - "comments": [], - "annotationValue": "na", - "termAccession": "", - "termSource": "" - } - }, { "comments": [], "category": { @@ -663,7 +674,7 @@ }, "value": { "comments": [], - "annotationValue": "budding", + "annotationValue": "vegetative rosette stage", "termAccession": "", "termSource": "" } @@ -675,7 +686,7 @@ }, "value": { "comments": [], - "annotationValue": "01/01/2022", + "annotationValue": "2022-01-01", "termAccession": "", "termSource": "" } @@ -687,7 +698,7 @@ }, "value": { "comments": [], - "annotationValue": "seed", + "annotationValue": "whole plant", "termAccession": "", "termSource": "" } @@ -740,6 +751,26 @@ "termSource": "" } }, + { + "category": { + "@id": "#characteristic_category/genotype_330" + }, + "value": { + "annotationValue": "Col-0", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/growth_condition_331" + }, + "value": { + "annotationValue": "16 h light / 8 h dark growth chamber", + "termAccession": "", + "termSource": "" + } + }, { "comments": [], "category": { @@ -784,6 +815,26 @@ "@id": "#ontology_annotation/accession_#sample/331", "annotationValue": "SAMEA131504584" } + }, + { + "category": { + "@id": "#characteristic_category/organism_part_332" + }, + "value": { + "annotationValue": "leaf", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/sample_description_333" + }, + "value": { + "annotationValue": "young rosette leaf collected for DNA extraction", + "termAccession": "", + "termSource": "" + } } ], "factorValues": [], @@ -859,7 +910,38 @@ "derivesFrom": [] } ], - "parameterValues": [], + "parameterValues": [ + { + "category": { + "@id": "#parameter/sample_collection_isolation_source" + }, + "value": { + "annotationValue": "leaf tissue", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_method" + }, + "value": { + "annotationValue": "sterile scalpel excision", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_preservation" + }, + "value": { + "annotationValue": "flash frozen in liquid nitrogen", + "termAccession": "", + "termSource": "" + } + } + ], "performer": "", "previousProcess": { "comments": [], @@ -886,7 +968,32 @@ ], "description": "", "name": "sample collection", - "parameters": [], + "parameters": [ + { + "@id": "#parameter/sample_collection_isolation_source", + "parameterName": { + "annotationValue": "isolation_source", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_method", + "parameterName": { + "annotationValue": "collection method", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_preservation", + "parameterName": { + "annotationValue": "sample preservation", + "termAccession": "", + "termSource": "" + } + } + ], "protocolType": { "comments": [], "annotationValue": "sample collection", From 7821de47244df032864d03dce3c9eb60a30a0bb1 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 16:59:49 +0200 Subject: [PATCH 28/41] add multi sample test --- MARS | 2 +- tests/test_isa_json.py | 95 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/MARS b/MARS index a65f10a..002d734 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit a65f10a2f9b5ebccf0ca8dbaaf463ae9a2e10666 +Subproject commit 002d734ae0f9017f94cb7f41499b621781de843c diff --git a/tests/test_isa_json.py b/tests/test_isa_json.py index 3ef020f..e1d6d38 100644 --- a/tests/test_isa_json.py +++ b/tests/test_isa_json.py @@ -248,6 +248,101 @@ def test_update_study_materials_with_accession_categories(): ) +def test_update_study_materials_multiple_samples_from_one_source(): + json_path = "MARS/test-data/biosamples-input-isa-multi.json" + with open(json_path) as json_file: + json_data = json.load(json_file) + + study = json_data["investigation"]["studies"][0] + root_sample = next( + sample for sample in study["materials"]["samples"] if sample["name"] == "root 1" + ) + root_sample["derivesFrom"][0]["@id"] = "#source/330" + root_process = next( + process + for process in study["processSequence"] + if process["@id"] == "#process/sample_collection/431" + ) + root_process["inputs"][0]["@id"] = "#source/330" + + validated_isa_json = IsaJson.model_validate(json_data) + repo_response = RepositoryResponse.model_validate( + { + "targetRepository": "biosamples", + "errors": [], + "info": [], + "accessions": [ + { + "value": "SAMEA_SOURCE_1", + "path": [ + {"key": "investigation"}, + { + "key": "studies", + "where": { + "key": "title", + "value": study["title"], + }, + }, + {"key": "materials"}, + { + "key": "sources", + "where": {"key": "name", "value": "plant 1"}, + }, + ], + }, + { + "value": "SAMEA_LEAF_1", + "path": [ + {"key": "investigation"}, + { + "key": "studies", + "where": { + "key": "title", + "value": study["title"], + }, + }, + {"key": "materials"}, + { + "key": "samples", + "where": {"key": "name", "value": "leaf 1"}, + }, + ], + }, + { + "value": "SAMEA_ROOT_1", + "path": [ + {"key": "investigation"}, + { + "key": "studies", + "where": { + "key": "title", + "value": study["title"], + }, + }, + {"key": "materials"}, + { + "key": "samples", + "where": {"key": "name", "value": "root 1"}, + }, + ], + }, + ], + } + ) + + updated_isa_json = update_isa_json(validated_isa_json, repo_response) + updated_study = updated_isa_json.investigation.studies[0] + updated_samples = updated_study.materials.samples + + assert updated_study.materials.sources[0].characteristics[-1].value.annotationValue == ( + "SAMEA_SOURCE_1" + ) + assert updated_samples[0].derivesFrom[0].id == "#source/330" + assert updated_samples[0].characteristics[-1].value.annotationValue == "SAMEA_LEAF_1" + assert updated_samples[1].derivesFrom[0].id == "#source/330" + assert updated_samples[1].characteristics[-1].value.annotationValue == "SAMEA_ROOT_1" + + def test_update_study_only_with_ena_study_accession_comment(): json_path = "tests/fixtures/isa_jsons/1_after_biosamples.json" isa_json = load_isa_json(json_path, False) From 5059a3d0e62d5239eb65a7d3750924b9af9e437a Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 10 Jul 2026 17:23:07 +0200 Subject: [PATCH 29/41] test refactor --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 002d734..3efcae1 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 002d734ae0f9017f94cb7f41499b621781de843c +Subproject commit 3efcae119a806a3fc5901afdf6dd1af78fb2c205 From 6251721b09c63204f31ef88aa22c96a359f66621 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Mon, 13 Jul 2026 09:56:20 +0200 Subject: [PATCH 30/41] tets out refactored code --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 3efcae1..aaa5a24 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 3efcae119a806a3fc5901afdf6dd1af78fb2c205 +Subproject commit aaa5a24552485c772b15a0998b145c68d4a5b82b From ebef0ec297b6798b8c10d6db9c6f9cc7cc5d8649 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Mon, 13 Jul 2026 14:03:44 +0200 Subject: [PATCH 31/41] update tests --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index aaa5a24..7ae0327 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit aaa5a24552485c772b15a0998b145c68d4a5b82b +Subproject commit 7ae0327c3c3a329db7cc30848778cb55a9b3491d From 256642d4f427bb6a352c668fbfc98594be6f3174 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Mon, 13 Jul 2026 14:39:09 +0200 Subject: [PATCH 32/41] rerun with updated test data --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 7ae0327..5732bc7 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 7ae0327c3c3a329db7cc30848778cb55a9b3491d +Subproject commit 5732bc746578519adc3f8baea76eae7923e38bc1 From 9fa2286c38a1bf0da317e53616fc421631c4441f Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 17 Jul 2026 08:30:35 +0200 Subject: [PATCH 33/41] submit to both holofood and multi --- .../multi-omics-submission-test.yaml | 34 ++++++++++---- MARS | 2 +- scripts/isa_generator.py | 11 +++-- scripts/prepare_poc_submission.py | 46 ++++++++++++------- 4 files changed, 63 insertions(+), 30 deletions(-) diff --git a/.github/workflows/multi-omics-submission-test.yaml b/.github/workflows/multi-omics-submission-test.yaml index 7a962d4..87566c9 100644 --- a/.github/workflows/multi-omics-submission-test.yaml +++ b/.github/workflows/multi-omics-submission-test.yaml @@ -15,9 +15,10 @@ jobs: # Where mars-cli will put .mars/ MARS_SETTINGS_DIR: ${{ github.workspace }}/mars_settings - # Paths for MARS repo and ISA template + # Paths for MARS repo and ISA templates REPOSITORY_SERVICES_PATH: ${{ github.workspace }}/MARS/repository-services ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa-multi.json + HOLOFOOD_ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/HoloFoodinISA_v1.0.json # Credentials from GitHub secrets WEBIN_USERNAME: ${{ secrets.WEBIN_USERNAME }} @@ -73,30 +74,43 @@ jobs: echo "Services did not become healthy in time ❌" exit 1 - - name: Prepare PoC - settings, ISA JSON, credentials, dataFiles + - name: Prepare PoC - settings, ISA JSONs, credentials, dataFiles run: | python scripts/prepare_poc_submission.py - - name: Run mars-cli submit + - name: Submit multi-omics ISA JSON run: | set -euo pipefail - # Build list of --data-files arguments from generated .fastq.gz files DATA_ARGS=() - for f in poc_work/data/*.fastq.gz; do - echo "Using data file: ./$f" - DATA_ARGS+=( --data-files "./$f" ) + for data_file in poc_work/multi-omics/data/*.fastq.gz; do + DATA_ARGS+=( --data-files "$data_file" ) done - echo "Running mars-cli with data args: $DATA_ARGS" + mars-cli --development submit \ + --submit-to-metabolights False \ + --file-transfer ftp \ + --output "output_multi_omics_${{ github.run_id }}_${{ github.run_attempt }}" \ + "${DATA_ARGS[@]}" \ + --credentials-file ./poc_work/credentials.json \ + poc_work/multi-omics/isa.json + + - name: Submit HoloFood ISA JSON + run: | + set -euo pipefail + + DATA_ARGS=() + for data_file in poc_work/holofood/data/*.fastq.gz; do + DATA_ARGS+=( --data-files "$data_file" ) + done mars-cli --development submit \ --submit-to-metabolights False \ --file-transfer ftp \ - --output "output_${{ github.run_id }}_${{ github.run_attempt }}.json" \ + --output "output_holofood_${{ github.run_id }}_${{ github.run_attempt }}" \ "${DATA_ARGS[@]}" \ --credentials-file ./poc_work/credentials.json \ - ./poc_work/isa.json + poc_work/holofood/isa.json - name: Collect repository-service logs if: always() diff --git a/MARS b/MARS index 5732bc7..03bea89 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 5732bc746578519adc3f8baea76eae7923e38bc1 +Subproject commit 03bea89d8f8a5fb241be8079a0f61970c5e3154f diff --git a/scripts/isa_generator.py b/scripts/isa_generator.py index cef1a89..e9849b1 100644 --- a/scripts/isa_generator.py +++ b/scripts/isa_generator.py @@ -125,11 +125,16 @@ def _update_datafiles_with_generated_files( if not isinstance(original_name, str) or not original_name: continue - if original_name.endswith(".fastq.gz"): - base = original_name[:-len(".fastq.gz")] + # ISA data file names can include a relative directory (HoloFood uses + # FILES/RAW_FILES/...). Upload mapping is filename-based, so generated + # PoC files and their ISA names must use only the basename. + original_basename = original_name.replace("\\", "/").rsplit("/", 1)[-1] + + if original_basename.endswith(".fastq.gz"): + base = original_basename[: -len(".fastq.gz")] new_name = f"{base}_{suffix}.fastq.gz" else: - new_name = f"{original_name}_{suffix}.fastq.gz" + new_name = f"{original_basename}_{suffix}.fastq.gz" file_path = data_dir / new_name _write_dummy_fastq_gz(file_path) diff --git a/scripts/prepare_poc_submission.py b/scripts/prepare_poc_submission.py index 23b70f5..c9dde8c 100644 --- a/scripts/prepare_poc_submission.py +++ b/scripts/prepare_poc_submission.py @@ -1,21 +1,23 @@ #!/usr/bin/env python """ -Prepare a PoC environment for MARS-CLI (no pytest, no run script): +Prepare the PoC environments for MARS-CLI (no pytest, no run script): - Ensure settings.ini exists in $MARS_SETTINGS_DIR/.mars, or ~/.mars -- Generate an ISA-JSON from a template, where: +- Generate an ISA-JSON from each configured template, where: * dataFiles entries in the first assay are updated to point to UNIQUE .fastq.gz files * 'file name', 'file type', 'file checksum', 'checksum_method' comments are updated accordingly (MD5 over the .fastq.gz) - Create poc_work/credentials.json from environment variables -The GitHub Action (or you, locally) will then call mars_cli.py directly using: +Set ``HOLOFOOD_ISA_TEMPLATE_PATH`` to prepare the additional HoloFood +submission alongside ``ISA_TEMPLATE_PATH``. - ISA JSON: poc_work/isa.json +The GitHub Action (or you, locally) will then call mars-cli using: + + ISA JSON: poc_work//isa.json Credentials: poc_work/credentials.json - Data files: whatever generate_isa_json_with_data() returned - (typically poc_work/data/*.fastq.gz) + Data files: poc_work//data/*.fastq.gz """ @@ -120,22 +122,34 @@ def main() -> None: work_dir = PROJECT_ROOT / "poc_work" work_dir.mkdir(parents=True, exist_ok=True) - isa_template = resolve_isa_template() - - isa_path, data_files = generate_isa_json_with_data( - work_dir=work_dir, - template_path=isa_template, - ) + isa_templates = {"multi-omics": resolve_isa_template()} + holofood_template = os.environ.get("HOLOFOOD_ISA_TEMPLATE_PATH") + if holofood_template: + holofood_path = Path(holofood_template) + if not holofood_path.exists(): + raise FileNotFoundError(f"HoloFood ISA template not found at {holofood_path}") + isa_templates["holofood"] = holofood_path + + prepared_submissions = [] + for submission_name, isa_template in isa_templates.items(): + submission_dir = work_dir / submission_name + isa_path, data_files = generate_isa_json_with_data( + work_dir=submission_dir, + template_path=isa_template, + ) + prepared_submissions.append((isa_template, isa_path, data_files)) cred_path = write_credentials_json(work_dir) print(f"[MARS-POC] settings.ini: {settings_path}") print(f"[MARS-POC] work dir: {work_dir}") - print(f"[MARS-POC] ISA JSON file: {isa_path}") print(f"[MARS-POC] credentials: {cred_path}") - print(f"[MARS-POC] data files:") - for df in data_files: - print(f" - {df}") + for isa_template, isa_path, data_files in prepared_submissions: + print(f"[MARS-POC] template: {isa_template}") + print(f"[MARS-POC] ISA JSON file: {isa_path}") + print("[MARS-POC] data files:") + for data_file in data_files: + print(f" - {data_file}") if __name__ == "__main__": From f865bda75095cc1d4a57d3209d898f827f519134 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 17 Jul 2026 08:39:32 +0200 Subject: [PATCH 34/41] handle more than onse submission --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 03bea89..7ac047f 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 03bea89d8f8a5fb241be8079a0f61970c5e3154f +Subproject commit 7ac047fea51cc0f61c73f472a718b27675b1f1c2 From a7f02c4153737af52d092a4341dead6a01d19bf2 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 17 Jul 2026 08:52:42 +0200 Subject: [PATCH 35/41] platform value fix --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 7ac047f..632d837 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 7ac047fea51cc0f61c73f472a718b27675b1f1c2 +Subproject commit 632d837b292ac3e8855078d04259c1fe367cfc23 From 56036fcb6e0077b1950c9c70c8a843a07c4ef878 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 17 Jul 2026 09:01:22 +0200 Subject: [PATCH 36/41] holo food attrib fix --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index 632d837..3c91121 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 632d837b292ac3e8855078d04259c1fe367cfc23 +Subproject commit 3c911216277df2abd400af04afbc4eb21fdb9f74 From 9117201045d9418300f46315d10b3bd06ca53ed9 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 17 Jul 2026 11:57:02 +0200 Subject: [PATCH 37/41] betetr handling of files --- .../multi-omics-submission-test.yaml | 4 +- MARS | 2 +- scripts/isa_generator.py | 64 +++++++++++-------- scripts/prepare_poc_submission.py | 4 +- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/.github/workflows/multi-omics-submission-test.yaml b/.github/workflows/multi-omics-submission-test.yaml index 87566c9..3f3f4c8 100644 --- a/.github/workflows/multi-omics-submission-test.yaml +++ b/.github/workflows/multi-omics-submission-test.yaml @@ -83,7 +83,7 @@ jobs: set -euo pipefail DATA_ARGS=() - for data_file in poc_work/multi-omics/data/*.fastq.gz; do + for data_file in poc_work/multi-omics/data/*; do DATA_ARGS+=( --data-files "$data_file" ) done @@ -100,7 +100,7 @@ jobs: set -euo pipefail DATA_ARGS=() - for data_file in poc_work/holofood/data/*.fastq.gz; do + for data_file in poc_work/holofood/data/*; do DATA_ARGS+=( --data-files "$data_file" ) done diff --git a/MARS b/MARS index 3c91121..ecfd4ce 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 3c911216277df2abd400af04afbc4eb21fdb9f74 +Subproject commit ecfd4ce5821bd65e07a3a2888346069d4e2593b5 diff --git a/scripts/isa_generator.py b/scripts/isa_generator.py index e9849b1..77f64dc 100644 --- a/scripts/isa_generator.py +++ b/scripts/isa_generator.py @@ -14,19 +14,29 @@ def _timestamp_suffix() -> str: return datetime.now(UTC).strftime("%Y%m%d%H%M%S") + "_" + uuid.uuid4().hex[:6] -def _write_dummy_fastq_gz(path: Path) -> None: - """ - Write a tiny dummy FASTQ dataset into a .fastq.gz file. - Content doesn't matter, as long as it's valid-ish FASTQ text. - """ +def _write_dummy_data_file(path: Path) -> None: + """Write a tiny dummy file, using gzip for compressed FASTQ extensions.""" path.parent.mkdir(parents=True, exist_ok=True) - with gzip.open(path, "wt") as fh: - fh.write( - "@read1\n" - "ACGTACGTACGTACGT\n" - "+\n" - "FFFFFFFFFFFFFFFF\n" - ) + content = "@read1\nACGTACGTACGTACGT\n+\nFFFFFFFFFFFFFFFF\n" + if path.name.lower().endswith((".fastq.gz", ".fq.gz")): + with gzip.open(path, "wt") as fh: + fh.write(content) + else: + path.write_text(content) + + +def _add_suffix_before_extension(file_name: str, suffix: str) -> str: + """Insert a unique suffix while preserving the original file extension.""" + lower_name = file_name.lower() + for compound_extension in (".fastq.gz", ".fq.gz"): + if lower_name.endswith(compound_extension): + extension = file_name[-len(compound_extension) :] + return f"{file_name[:-len(compound_extension)]}_{suffix}{extension}" + + extension = Path(file_name).suffix + if extension: + return f"{file_name[:-len(extension)]}_{suffix}{extension}" + return f"{file_name}_{suffix}" def _md5_of_file(path: Path) -> str: @@ -87,13 +97,14 @@ def _update_datafiles_with_generated_files( n_files: int | None, ) -> List[Path]: """ - Update assay dataFiles entries with newly generated .fastq.gz files. + Update assay dataFiles entries with newly generated files. Behaviour per touched data file: - - Generate a unique .fastq.gz file based on the existing 'name': - e.g. ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_.fastq.gz - (if name doesn't end with .fastq.gz, just append _.fastq.gz) + - Generate a unique file based on the existing 'name' while preserving + its extension, for example: + ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_.fastq.gz + reads.R1.fq.gz -> reads.R1_.fq.gz - Write a dummy FASTQ into that file and compute its MD5. @@ -101,8 +112,8 @@ def _update_datafiles_with_generated_files( * "name" = new file name * in "comments": - "file name" -> new file name - - "file type" -> "fastq" - - "file checksum" -> MD5 of the .fastq.gz + - "file type" -> the preserved extension + - "file checksum" -> MD5 of the generated file - "checksum_method" -> "MD5" (existing "accession", "submission date", etc. are kept as-is) """ @@ -130,14 +141,10 @@ def _update_datafiles_with_generated_files( # PoC files and their ISA names must use only the basename. original_basename = original_name.replace("\\", "/").rsplit("/", 1)[-1] - if original_basename.endswith(".fastq.gz"): - base = original_basename[: -len(".fastq.gz")] - new_name = f"{base}_{suffix}.fastq.gz" - else: - new_name = f"{original_basename}_{suffix}.fastq.gz" + new_name = _add_suffix_before_extension(original_basename, suffix) file_path = data_dir / new_name - _write_dummy_fastq_gz(file_path) + _write_dummy_data_file(file_path) md5 = _md5_of_file(file_path) df_json["name"] = new_name @@ -148,7 +155,12 @@ def _update_datafiles_with_generated_files( df_json["comments"] = comments _ensure_comment(comments, "file name", new_name) - _ensure_comment(comments, "file type", "fastq") + file_type = ( + "fastq" + if new_name.lower().endswith((".fastq.gz", ".fq.gz")) + else Path(new_name).suffix.lstrip(".") or "data" + ) + _ensure_comment(comments, "file type", file_type) _ensure_comment(comments, "file checksum", md5) _ensure_comment(comments, "checksum_method", "MD5") @@ -169,7 +181,7 @@ def generate_isa_json_with_data( 1. Load ISA-JSON template from template_path. 2. Find all investigation.studies[*].assays[*].dataFiles. 3. For each data file (or the first n_files when limited), generate UNIQUE - .fastq.gz files and update: + files with preserved extensions and update: - dataFiles[i]["name"] - dataFiles[i]["comments"] entries for file name, type, checksum, method. 4. Write the resulting ISA-JSON to work_dir / 'isa.json'. diff --git a/scripts/prepare_poc_submission.py b/scripts/prepare_poc_submission.py index c9dde8c..df188ae 100644 --- a/scripts/prepare_poc_submission.py +++ b/scripts/prepare_poc_submission.py @@ -4,8 +4,8 @@ - Ensure settings.ini exists in $MARS_SETTINGS_DIR/.mars, or ~/.mars - Generate an ISA-JSON from each configured template, where: - * dataFiles entries in the first assay are updated to point to - UNIQUE .fastq.gz files + * dataFiles entries are updated to point to UNIQUE generated files while + preserving their original extensions * 'file name', 'file type', 'file checksum', 'checksum_method' comments are updated accordingly (MD5 over the .fastq.gz) - Create poc_work/credentials.json from environment variables From d6bcdcb2874209f38e4d9213913af3cc7fbdb45e Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Fri, 17 Jul 2026 12:16:49 +0200 Subject: [PATCH 38/41] handle multiple submissions --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index ecfd4ce..a33cda6 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit ecfd4ce5821bd65e07a3a2888346069d4e2593b5 +Subproject commit a33cda6001b1b68c8946fdc8cc91c34cece9f0d7 From 47dc173f85c12202318c04596a443dd31e38ccc0 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Wed, 22 Jul 2026 11:54:28 +0200 Subject: [PATCH 39/41] Update MARS study-level submodule --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index a33cda6..f5b79ea 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit a33cda6001b1b68c8946fdc8cc91c34cece9f0d7 +Subproject commit f5b79ea834589d1d7dd542a61373aa7df59853cd From c53ad4b1b851335595ef1dccb403eeb11896b92d Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Thu, 23 Jul 2026 14:53:29 +0200 Subject: [PATCH 40/41] pin to v0.0.2 --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index f5b79ea..bc78849 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit f5b79ea834589d1d7dd542a61373aa7df59853cd +Subproject commit bc78849d9565ac2a07c66a716cc8a4b6f2f59222 From b0b169acd532c8384996e80fe0b550d6f9b12e97 Mon Sep 17 00:00:00 2001 From: Bert Droesbeke Date: Thu, 23 Jul 2026 15:18:50 +0200 Subject: [PATCH 41/41] reset receipt --- MARS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MARS b/MARS index bc78849..f92a39e 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit bc78849d9565ac2a07c66a716cc8a4b6f2f59222 +Subproject commit f92a39e38e32d0334f12af1aae158bfa97cce950