diff --git a/.github/workflows/multi-omics-submission-test.yaml b/.github/workflows/multi-omics-submission-test.yaml index 886686d..3f3f4c8 100644 --- a/.github/workflows/multi-omics-submission-test.yaml +++ b/.github/workflows/multi-omics-submission-test.yaml @@ -15,9 +15,10 @@ jobs: # Where mars-cli will put .mars/ MARS_SETTINGS_DIR: ${{ github.workspace }}/mars_settings - # Paths for MARS repo and ISA template + # Paths for MARS repo and ISA templates REPOSITORY_SERVICES_PATH: ${{ github.workspace }}/MARS/repository-services - ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa.json + ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/biosamples-input-isa-multi.json + HOLOFOOD_ISA_TEMPLATE_PATH: ${{ github.workspace }}/MARS/test-data/HoloFoodinISA_v1.0.json # Credentials from GitHub secrets WEBIN_USERNAME: ${{ secrets.WEBIN_USERNAME }} @@ -73,30 +74,43 @@ jobs: echo "Services did not become healthy in time ❌" exit 1 - - name: Prepare PoC - settings, ISA JSON, credentials, dataFiles + - name: Prepare PoC - settings, ISA JSONs, credentials, dataFiles run: | python scripts/prepare_poc_submission.py - - name: Run mars-cli submit + - name: Submit multi-omics ISA JSON run: | set -euo pipefail - # Build list of --data-files arguments from generated .fastq.gz files DATA_ARGS=() - for f in poc_work/data/*.fastq.gz; do - echo "Using data file: ./$f" - DATA_ARGS+=( --data-files "./$f" ) + for data_file in poc_work/multi-omics/data/*; do + DATA_ARGS+=( --data-files "$data_file" ) done - echo "Running mars-cli with data args: $DATA_ARGS" + mars-cli --development submit \ + --submit-to-metabolights False \ + --file-transfer ftp \ + --output "output_multi_omics_${{ github.run_id }}_${{ github.run_attempt }}" \ + "${DATA_ARGS[@]}" \ + --credentials-file ./poc_work/credentials.json \ + poc_work/multi-omics/isa.json + + - name: Submit HoloFood ISA JSON + run: | + set -euo pipefail + + DATA_ARGS=() + for data_file in poc_work/holofood/data/*; do + DATA_ARGS+=( --data-files "$data_file" ) + done mars-cli --development submit \ --submit-to-metabolights False \ --file-transfer ftp \ - --output "output_${{ github.run_id }}_${{ github.run_attempt }}.json" \ + --output "output_holofood_${{ github.run_id }}_${{ github.run_attempt }}" \ "${DATA_ARGS[@]}" \ --credentials-file ./poc_work/credentials.json \ - ./poc_work/isa.json + poc_work/holofood/isa.json - name: Collect repository-service logs if: always() diff --git a/MARS b/MARS index 8d9763e..f92a39e 160000 --- a/MARS +++ b/MARS @@ -1 +1 @@ -Subproject commit 8d9763e3501c3243ea5772b1d070e264494200c1 +Subproject commit f92a39e38e32d0334f12af1aae158bfa97cce950 diff --git a/README.md b/README.md index 9378a67..cfffc94 100644 --- a/README.md +++ b/README.md @@ -404,5 +404,3 @@ python mars_cli.py --credential-service-name metabolights --username-credential [To set up and run the MARS tool locally using Docker, follow these steps](../repository-services/README.md) - - diff --git a/mars_lib/isa_json.py b/mars_lib/isa_json.py index 793376b..53695f5 100644 --- a/mars_lib/isa_json.py +++ b/mars_lib/isa_json.py @@ -484,29 +484,26 @@ def update_isa_json(isa_json: IsaJson, repo_response: RepositoryResponse) -> Isa add_accession_to_data_file_node(updated_node, accession.value) else: - # Add study accession to study comments updated_study = apply_filter(study_filter, investigation.studies) - - study_accession_comment: Comment = Comment( + accession_comment = Comment( name=f"{target_repository}_{target_level}_accession", value=accession.value, ) - updated_study.comments.append(study_accession_comment) - - # Add study accession to assay comments - updated_assay = next( - filter( - lambda assay: is_assay_for_target_repo(assay, target_repository), - updated_study.assays, - ), - None, - ) - if updated_assay: - assay_accession_comment: Comment = Comment( - name=f"{target_repository}_{target_level}_accession", - value=accession.value, + + if target_level == "study": + updated_study.comments.append(accession_comment) + else: + updated_assay = next( + filter( + lambda assay: is_assay_for_target_repo( + assay, target_repository + ), + updated_study.assays, + ), + None, ) - updated_assay.comments.append(assay_accession_comment) + if updated_assay: + updated_assay.comments.append(accession_comment) isa_json.investigation = investigation return isa_json @@ -525,10 +522,13 @@ def map_data_files_to_repositories( for assay in assays: target_repo_comment: Comment = detect_target_repo_comment(assay.comments) # This is an effect of everything being optional in the Comment model. - # Should we decide to make the value mandatory, this guard clause would not be necessary anymore. + # Should we decide to make the value mandatory, this guard clause + # would not be necessary anymore. if target_repo_comment.value is None: raise ValueError( - f"At least one assay in the ISA-JSON has no '{TARGET_REPO_KEY}' comment. Mapping not possible. Make sure all assays in the ISA-JSON have this comment!" + f"At least one assay in the ISA-JSON has no " + f"'{TARGET_REPO_KEY}' comment. Mapping not possible. " + f"Make sure all assays in the ISA-JSON have this comment!" ) assay_data_files = [df.name for df in assay.dataFiles] @@ -555,7 +555,11 @@ def map_data_files_to_repositories( [ print_and_log( - msg=f"File '{rf['short_name']}' could not be mapped to any data file in the ISA-JSON. For this reason, it will be skipped during submission!", + msg=( + f"File '{rf['short_name']}' could not be mapped to any data " + f"file in the ISA-JSON. For this reason, it will be skipped " + f"during submission!" + ), level="warning", ) for rf in remaining_files diff --git a/scripts/isa_generator.py b/scripts/isa_generator.py index 555a79b..77f64dc 100644 --- a/scripts/isa_generator.py +++ b/scripts/isa_generator.py @@ -14,19 +14,29 @@ def _timestamp_suffix() -> str: return datetime.now(UTC).strftime("%Y%m%d%H%M%S") + "_" + uuid.uuid4().hex[:6] -def _write_dummy_fastq_gz(path: Path) -> None: - """ - Write a tiny dummy FASTQ dataset into a .fastq.gz file. - Content doesn't matter, as long as it's valid-ish FASTQ text. - """ +def _write_dummy_data_file(path: Path) -> None: + """Write a tiny dummy file, using gzip for compressed FASTQ extensions.""" path.parent.mkdir(parents=True, exist_ok=True) - with gzip.open(path, "wt") as fh: - fh.write( - "@read1\n" - "ACGTACGTACGTACGT\n" - "+\n" - "FFFFFFFFFFFFFFFF\n" - ) + content = "@read1\nACGTACGTACGTACGT\n+\nFFFFFFFFFFFFFFFF\n" + if path.name.lower().endswith((".fastq.gz", ".fq.gz")): + with gzip.open(path, "wt") as fh: + fh.write(content) + else: + path.write_text(content) + + +def _add_suffix_before_extension(file_name: str, suffix: str) -> str: + """Insert a unique suffix while preserving the original file extension.""" + lower_name = file_name.lower() + for compound_extension in (".fastq.gz", ".fq.gz"): + if lower_name.endswith(compound_extension): + extension = file_name[-len(compound_extension) :] + return f"{file_name[:-len(compound_extension)]}_{suffix}{extension}" + + extension = Path(file_name).suffix + if extension: + return f"{file_name[:-len(extension)]}_{suffix}{extension}" + return f"{file_name}_{suffix}" def _md5_of_file(path: Path) -> str: @@ -40,34 +50,33 @@ def _md5_of_file(path: Path) -> str: return h.hexdigest() -def _get_first_assay(isa_obj: dict[str, Any]) -> dict[str, Any] | None: +def _get_all_assays(isa_obj: dict[str, Any]) -> List[dict[str, Any]]: """ - Navigate to investigation.studies[0].assays[0] (if present). + Return all assays found under investigation.studies[*].assays[*]. """ inv = isa_obj.get("investigation") if inv is None: inv = isa_obj if not isinstance(inv, dict): - return None + return [] studies = inv.get("studies") or [] - if not isinstance(studies, list) or not studies: - return None + if not isinstance(studies, list): + return [] - first_study = studies[0] - if not isinstance(first_study, dict): - return None + assays: List[dict[str, Any]] = [] + for study in studies: + if not isinstance(study, dict): + continue - assays = first_study.get("assays") or [] - if not isinstance(assays, list) or not assays: - return None + study_assays = study.get("assays") or [] + if not isinstance(study_assays, list): + continue - first_assay = assays[0] - if not isinstance(first_assay, dict): - return None + assays.extend(assay for assay in study_assays if isinstance(assay, dict)) - return first_assay + return assays def _ensure_comment(comments: List[dict[str, Any]], name: str, value: str) -> None: @@ -83,74 +92,80 @@ def _ensure_comment(comments: List[dict[str, Any]], name: str, value: str) -> No def _update_datafiles_with_generated_files( - assay: dict[str, Any], + assays: List[dict[str, Any]], data_dir: Path, - n_files: int, + n_files: int | None, ) -> List[Path]: """ - For the first assay, update its dataFiles entries with newly generated .fastq.gz files. + Update assay dataFiles entries with newly generated files. - Behaviour per dataFiles[i] (for i < n_files): + Behaviour per touched data file: - - Generate a unique .fastq.gz file based on the existing 'name': - e.g. ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_.fastq.gz - (if name doesn't end with .fastq.gz, just append _.fastq.gz) + - Generate a unique file based on the existing 'name' while preserving + its extension, for example: + ENA_TEST2.R2.fastq.gz -> ENA_TEST2.R2_.fastq.gz + reads.R1.fq.gz -> reads.R1_.fq.gz - Write a dummy FASTQ into that file and compute its MD5. - - Update the dataFiles[i] object: + - Update the dataFiles entry: * "name" = new file name * in "comments": - "file name" -> new file name - - "file type" -> "fastq" - - "file checksum" -> MD5 of the .fastq.gz + - "file type" -> the preserved extension + - "file checksum" -> MD5 of the generated file - "checksum_method" -> "MD5" (existing "accession", "submission date", etc. are kept as-is) """ - data_files_json = assay.get("dataFiles") or [] - if not isinstance(data_files_json, list): - return [] - generated_paths: List[Path] = [] suffix = _timestamp_suffix() + updated_count = 0 - # We only touch up to n_files entries, and only those that look like objects - for i, df_json in enumerate(data_files_json): - if i >= n_files: - break - if not isinstance(df_json, dict): + for assay in assays: + data_files_json = assay.get("dataFiles") or [] + if not isinstance(data_files_json, list): continue - original_name = df_json.get("name") - if not isinstance(original_name, str) or not original_name: - continue + for df_json in data_files_json: + if n_files is not None and updated_count >= n_files: + return generated_paths + if not isinstance(df_json, dict): + continue + + original_name = df_json.get("name") + if not isinstance(original_name, str) or not original_name: + continue + + # ISA data file names can include a relative directory (HoloFood uses + # FILES/RAW_FILES/...). Upload mapping is filename-based, so generated + # PoC files and their ISA names must use only the basename. + original_basename = original_name.replace("\\", "/").rsplit("/", 1)[-1] - # Build unique .fastq.gz name - if original_name.endswith(".fastq.gz"): - base = original_name[:-len(".fastq.gz")] - new_name = f"{base}_{suffix}.fastq.gz" - else: - new_name = f"{original_name}_{suffix}.fastq.gz" + new_name = _add_suffix_before_extension(original_basename, suffix) - file_path = data_dir / new_name - _write_dummy_fastq_gz(file_path) - md5 = _md5_of_file(file_path) + file_path = data_dir / new_name + _write_dummy_data_file(file_path) + md5 = _md5_of_file(file_path) - # Update the JSON entry - df_json["name"] = new_name + df_json["name"] = new_name - comments = df_json.get("comments") - if not isinstance(comments, list): - comments = [] - df_json["comments"] = comments + comments = df_json.get("comments") + if not isinstance(comments, list): + comments = [] + df_json["comments"] = comments - _ensure_comment(comments, "file name", new_name) - _ensure_comment(comments, "file type", "fastq") - _ensure_comment(comments, "file checksum", md5) - _ensure_comment(comments, "checksum_method", "MD5") - # DO NOT touch 'accession' or 'submission date' if present + _ensure_comment(comments, "file name", new_name) + file_type = ( + "fastq" + if new_name.lower().endswith((".fastq.gz", ".fq.gz")) + else Path(new_name).suffix.lstrip(".") or "data" + ) + _ensure_comment(comments, "file type", file_type) + _ensure_comment(comments, "file checksum", md5) + _ensure_comment(comments, "checksum_method", "MD5") - generated_paths.append(file_path) + generated_paths.append(file_path) + updated_count += 1 return generated_paths @@ -158,15 +173,15 @@ def _update_datafiles_with_generated_files( def generate_isa_json_with_data( work_dir: Path, template_path: Path, - n_files: int = 2, + n_files: int | None = None, ) -> Tuple[Path, List[Path]]: """ PoC behaviour: 1. Load ISA-JSON template from template_path. - 2. Find investigation.studies[0].assays[0].dataFiles. - 3. For up to n_files entries in dataFiles, generate UNIQUE .fastq.gz files - and update: + 2. Find all investigation.studies[*].assays[*].dataFiles. + 3. For each data file (or the first n_files when limited), generate UNIQUE + files with preserved extensions and update: - dataFiles[i]["name"] - dataFiles[i]["comments"] entries for file name, type, checksum, method. 4. Write the resulting ISA-JSON to work_dir / 'isa.json'. @@ -177,12 +192,12 @@ def generate_isa_json_with_data( isa_obj = json.loads(template_path.read_text()) - assay = _get_first_assay(isa_obj) + assays = _get_all_assays(isa_obj) generated_paths: List[Path] = [] - if assay is not None: + if assays: data_dir = work_dir / "data" generated_paths = _update_datafiles_with_generated_files( - assay=assay, + assays=assays, data_dir=data_dir, n_files=n_files, ) diff --git a/scripts/prepare_poc_submission.py b/scripts/prepare_poc_submission.py index c47456b..df188ae 100644 --- a/scripts/prepare_poc_submission.py +++ b/scripts/prepare_poc_submission.py @@ -1,21 +1,23 @@ #!/usr/bin/env python """ -Prepare a PoC environment for MARS-CLI (no pytest, no run script): +Prepare the PoC environments for MARS-CLI (no pytest, no run script): - Ensure settings.ini exists in $MARS_SETTINGS_DIR/.mars, or ~/.mars -- Generate an ISA-JSON from a template, where: - * dataFiles entries in the first assay are updated to point to - UNIQUE .fastq.gz files +- Generate an ISA-JSON from each configured template, where: + * dataFiles entries are updated to point to UNIQUE generated files while + preserving their original extensions * 'file name', 'file type', 'file checksum', 'checksum_method' comments are updated accordingly (MD5 over the .fastq.gz) - Create poc_work/credentials.json from environment variables -The GitHub Action (or you, locally) will then call mars_cli.py directly using: +Set ``HOLOFOOD_ISA_TEMPLATE_PATH`` to prepare the additional HoloFood +submission alongside ``ISA_TEMPLATE_PATH``. - ISA JSON: poc_work/isa.json +The GitHub Action (or you, locally) will then call mars-cli using: + + ISA JSON: poc_work//isa.json Credentials: poc_work/credentials.json - Data files: whatever generate_isa_json_with_data() returned - (typically poc_work/data/*.fastq.gz) + Data files: poc_work//data/*.fastq.gz """ @@ -120,23 +122,34 @@ def main() -> None: work_dir = PROJECT_ROOT / "poc_work" work_dir.mkdir(parents=True, exist_ok=True) - isa_template = resolve_isa_template() - - isa_path, data_files = generate_isa_json_with_data( - work_dir=work_dir, - template_path=isa_template, - n_files=2, - ) + isa_templates = {"multi-omics": resolve_isa_template()} + holofood_template = os.environ.get("HOLOFOOD_ISA_TEMPLATE_PATH") + if holofood_template: + holofood_path = Path(holofood_template) + if not holofood_path.exists(): + raise FileNotFoundError(f"HoloFood ISA template not found at {holofood_path}") + isa_templates["holofood"] = holofood_path + + prepared_submissions = [] + for submission_name, isa_template in isa_templates.items(): + submission_dir = work_dir / submission_name + isa_path, data_files = generate_isa_json_with_data( + work_dir=submission_dir, + template_path=isa_template, + ) + prepared_submissions.append((isa_template, isa_path, data_files)) cred_path = write_credentials_json(work_dir) print(f"[MARS-POC] settings.ini: {settings_path}") print(f"[MARS-POC] work dir: {work_dir}") - print(f"[MARS-POC] ISA JSON file: {isa_path}") print(f"[MARS-POC] credentials: {cred_path}") - print(f"[MARS-POC] data files:") - for df in data_files: - print(f" - {df}") + for isa_template, isa_path, data_files in prepared_submissions: + print(f"[MARS-POC] template: {isa_template}") + print(f"[MARS-POC] ISA JSON file: {isa_path}") + print("[MARS-POC] data files:") + for data_file in data_files: + print(f" - {data_file}") if __name__ == "__main__": diff --git a/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json b/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json index 0cf74fc..5e553a5 100644 --- a/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json +++ b/tests/fixtures/isa_jsons/0_Initial_ISA_JSON_in_model.json @@ -82,33 +82,6 @@ "termAccession": "", "termSource": "" } - }, - { - "@id": "#characteristic_category/submission_date_358", - "characteristicType": { - "comments": [], - "annotationValue": "submission date", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/status_359", - "characteristicType": { - "comments": [], - "annotationValue": "status", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/accession_360", - "characteristicType": { - "comments": [], - "annotationValue": "accession", - "termAccession": "", - "termSource": "" - } } ], "dataFiles": [ @@ -158,7 +131,18 @@ { "comments": [], "@id": "#other_material/332", - "characteristics": [], + "characteristics": [ + { + "category": { + "@id": "#characteristic_category/Title_350" + }, + "value": { + "annotationValue": "Arabidopsis leaf amplicon sequencing experiment", + "termAccession": "", + "termSource": "" + } + } + ], "name": "extract 1", "type": "library name", "derivesFrom": [ @@ -173,56 +157,7 @@ { "comments": [], "@id": "#other_material/333", - "characteristics": [ - { - "comments": [], - "category": { - "@id": "#characteristic_category/Title_350" - }, - "value": { - "comments": [], - "annotationValue": "library 1", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/submission_date_358" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/status_359" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/accession_360" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - } - ], + "characteristics": [], "name": "library 1", "type": "library name", "derivesFrom": [ @@ -524,15 +459,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/cell_type_321", - "characteristicType": { - "comments": [], - "annotationValue": "cell_type", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/dev_stage_322", "characteristicType": { @@ -604,6 +530,38 @@ "termAccession": "", "termSource": "" } + }, + { + "@id": "#characteristic_category/genotype_330", + "characteristicType": { + "annotationValue": "genotype", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/growth_condition_331", + "characteristicType": { + "annotationValue": "growth condition", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/organism_part_332", + "characteristicType": { + "annotationValue": "organism part", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/sample_description_333", + "characteristicType": { + "annotationValue": "sample description", + "termAccession": "", + "termSource": "" + } } ], "description": "Nucleic acid sequencing and metabolomics and proteomics of Arabidopsis thaliana in specific experimental conditions to test a specific hypothesis.\r\n", @@ -664,18 +622,6 @@ "termSource": "" } }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/cell_type_321" - }, - "value": { - "comments": [], - "annotationValue": "na", - "termAccession": "", - "termSource": "" - } - }, { "comments": [], "category": { @@ -683,7 +629,7 @@ }, "value": { "comments": [], - "annotationValue": "budding", + "annotationValue": "vegetative rosette stage", "termAccession": "", "termSource": "" } @@ -695,7 +641,7 @@ }, "value": { "comments": [], - "annotationValue": "01/01/2022", + "annotationValue": "2022-01-01", "termAccession": "", "termSource": "" } @@ -707,7 +653,7 @@ }, "value": { "comments": [], - "annotationValue": "seed", + "annotationValue": "whole plant", "termAccession": "", "termSource": "" } @@ -760,6 +706,26 @@ "termSource": "" } }, + { + "category": { + "@id": "#characteristic_category/genotype_330" + }, + "value": { + "annotationValue": "Col-0", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/growth_condition_331" + }, + "value": { + "annotationValue": "16 h light / 8 h dark growth chamber", + "termAccession": "", + "termSource": "" + } + }, { "comments": [], "category": { @@ -787,7 +753,28 @@ "comments": [], "@id": "#sample/331", "name": "leaf 1", - "characteristics": [], + "characteristics": [ + { + "category": { + "@id": "#characteristic_category/organism_part_332" + }, + "value": { + "annotationValue": "leaf", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/sample_description_333" + }, + "value": { + "annotationValue": "young rosette leaf collected for DNA extraction", + "termAccession": "", + "termSource": "" + } + } + ], "factorValues": [], "derivesFrom": [ { @@ -861,7 +848,38 @@ "derivesFrom": [] } ], - "parameterValues": [], + "parameterValues": [ + { + "category": { + "@id": "#parameter/sample_collection_isolation_source" + }, + "value": { + "annotationValue": "leaf tissue", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_method" + }, + "value": { + "annotationValue": "sterile scalpel excision", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_preservation" + }, + "value": { + "annotationValue": "flash frozen in liquid nitrogen", + "termAccession": "", + "termSource": "" + } + } + ], "performer": "", "previousProcess": { "comments": [], @@ -888,7 +906,32 @@ ], "description": "", "name": "sample collection", - "parameters": [], + "parameters": [ + { + "@id": "#parameter/sample_collection_isolation_source", + "parameterName": { + "annotationValue": "isolation_source", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_method", + "parameterName": { + "annotationValue": "collection method", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_preservation", + "parameterName": { + "annotationValue": "sample preservation", + "termAccession": "", + "termSource": "" + } + } + ], "protocolType": { "comments": [], "annotationValue": "sample collection", @@ -1062,4 +1105,4 @@ "submissionDate": "", "title": "Bob's investigation" } -} \ No newline at end of file +} diff --git a/tests/fixtures/isa_jsons/1_after_biosamples.json b/tests/fixtures/isa_jsons/1_after_biosamples.json index e6d713c..fb25781 100644 --- a/tests/fixtures/isa_jsons/1_after_biosamples.json +++ b/tests/fixtures/isa_jsons/1_after_biosamples.json @@ -82,33 +82,6 @@ "termAccession": "", "termSource": "" } - }, - { - "@id": "#characteristic_category/submission_date_358", - "characteristicType": { - "comments": [], - "annotationValue": "submission date", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/status_359", - "characteristicType": { - "comments": [], - "annotationValue": "status", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/accession_360", - "characteristicType": { - "comments": [], - "annotationValue": "accession", - "termAccession": "", - "termSource": "" - } } ], "dataFiles": [ @@ -158,7 +131,18 @@ { "comments": [], "@id": "#other_material/332", - "characteristics": [], + "characteristics": [ + { + "category": { + "@id": "#characteristic_category/Title_350" + }, + "value": { + "annotationValue": "Arabidopsis leaf amplicon sequencing experiment", + "termAccession": "", + "termSource": "" + } + } + ], "name": "extract 1", "type": "library name", "derivesFrom": [ @@ -173,56 +157,7 @@ { "comments": [], "@id": "#other_material/333", - "characteristics": [ - { - "comments": [], - "category": { - "@id": "#characteristic_category/Title_350" - }, - "value": { - "comments": [], - "annotationValue": "library 1", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/submission_date_358" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/status_359" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/accession_360" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - } - ], + "characteristics": [], "name": "library 1", "type": "library name", "derivesFrom": [ @@ -524,15 +459,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/cell_type_321", - "characteristicType": { - "comments": [], - "annotationValue": "cell_type", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/dev_stage_322", "characteristicType": { @@ -604,6 +530,38 @@ "termAccession": "", "termSource": "" } + }, + { + "@id": "#characteristic_category/genotype_330", + "characteristicType": { + "annotationValue": "genotype", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/growth_condition_331", + "characteristicType": { + "annotationValue": "growth condition", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/organism_part_332", + "characteristicType": { + "annotationValue": "organism part", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/sample_description_333", + "characteristicType": { + "annotationValue": "sample description", + "termAccession": "", + "termSource": "" + } } ], "description": "Nucleic acid sequencing and metabolomics and proteomics of Arabidopsis thaliana in specific experimental conditions to test a specific hypothesis.\r\n", @@ -664,18 +622,6 @@ "termSource": "" } }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/cell_type_321" - }, - "value": { - "comments": [], - "annotationValue": "na", - "termAccession": "", - "termSource": "" - } - }, { "comments": [], "category": { @@ -683,7 +629,7 @@ }, "value": { "comments": [], - "annotationValue": "budding", + "annotationValue": "vegetative rosette stage", "termAccession": "", "termSource": "" } @@ -695,7 +641,7 @@ }, "value": { "comments": [], - "annotationValue": "01/01/2022", + "annotationValue": "2022-01-01", "termAccession": "", "termSource": "" } @@ -707,7 +653,7 @@ }, "value": { "comments": [], - "annotationValue": "seed", + "annotationValue": "whole plant", "termAccession": "", "termSource": "" } @@ -760,6 +706,26 @@ "termSource": "" } }, + { + "category": { + "@id": "#characteristic_category/genotype_330" + }, + "value": { + "annotationValue": "Col-0", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/growth_condition_331" + }, + "value": { + "annotationValue": "16 h light / 8 h dark growth chamber", + "termAccession": "", + "termSource": "" + } + }, { "comments": [], "category": { @@ -804,6 +770,26 @@ "@id": "#ontology_annotation/accession_#sample/331", "annotationValue": "SAMEA131504584" } + }, + { + "category": { + "@id": "#characteristic_category/organism_part_332" + }, + "value": { + "annotationValue": "leaf", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/sample_description_333" + }, + "value": { + "annotationValue": "young rosette leaf collected for DNA extraction", + "termAccession": "", + "termSource": "" + } } ], "factorValues": [], @@ -879,7 +865,38 @@ "derivesFrom": [] } ], - "parameterValues": [], + "parameterValues": [ + { + "category": { + "@id": "#parameter/sample_collection_isolation_source" + }, + "value": { + "annotationValue": "leaf tissue", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_method" + }, + "value": { + "annotationValue": "sterile scalpel excision", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_preservation" + }, + "value": { + "annotationValue": "flash frozen in liquid nitrogen", + "termAccession": "", + "termSource": "" + } + } + ], "performer": "", "previousProcess": { "comments": [], @@ -906,7 +923,32 @@ ], "description": "", "name": "sample collection", - "parameters": [], + "parameters": [ + { + "@id": "#parameter/sample_collection_isolation_source", + "parameterName": { + "annotationValue": "isolation_source", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_method", + "parameterName": { + "annotationValue": "collection method", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_preservation", + "parameterName": { + "annotationValue": "sample preservation", + "termAccession": "", + "termSource": "" + } + } + ], "protocolType": { "comments": [], "annotationValue": "sample collection", @@ -1080,4 +1122,4 @@ "submissionDate": "", "title": "Bob's investigation" } -} \ No newline at end of file +} diff --git a/tests/fixtures/isa_jsons/2_after_ena.json b/tests/fixtures/isa_jsons/2_after_ena.json index db856ae..77669d1 100644 --- a/tests/fixtures/isa_jsons/2_after_ena.json +++ b/tests/fixtures/isa_jsons/2_after_ena.json @@ -87,24 +87,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/submission_date_358", - "characteristicType": { - "comments": [], - "annotationValue": "submission date", - "termAccession": "", - "termSource": "" - } - }, - { - "@id": "#characteristic_category/status_359", - "characteristicType": { - "comments": [], - "annotationValue": "status", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/accession_360", "characteristicType": { @@ -163,6 +145,18 @@ "comments": [], "@id": "#other_material/332", "characteristics": [ + { + "category": { + "@id": "#characteristic_category/Title_350" + }, + "value": { + "annotationValue": "Arabidopsis leaf amplicon sequencing experiment", + "termAccession": "", + "termSource": "", + "comments": [] + }, + "comments": [] + }, { "comments": [], "@id": "#material_attribute_value/accession_360", @@ -197,55 +191,6 @@ "comments": [], "@id": "#other_material/333", "characteristics": [ - { - "comments": [], - "category": { - "@id": "#characteristic_category/Title_350" - }, - "value": { - "comments": [], - "annotationValue": "library 1", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/submission_date_358" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/status_359" - }, - "value": { - "comments": [], - "annotationValue": "", - "termAccession": "", - "termSource": "" - } - }, - { - "comments": [], - "@id": "#material_attribute_value/accession_360", - "category": { - "@id": "#characteristic_category/accession_360", - "characteristicType": { - "comments": [], - "annotationValue": "accession", - "termAccession": "", - "termSource": "" - } - } - }, { "comments": [], "category": { @@ -559,15 +504,6 @@ "termSource": "" } }, - { - "@id": "#characteristic_category/cell_type_321", - "characteristicType": { - "comments": [], - "annotationValue": "cell_type", - "termAccession": "", - "termSource": "" - } - }, { "@id": "#characteristic_category/dev_stage_322", "characteristicType": { @@ -639,6 +575,38 @@ "termAccession": "", "termSource": "" } + }, + { + "@id": "#characteristic_category/genotype_330", + "characteristicType": { + "annotationValue": "genotype", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/growth_condition_331", + "characteristicType": { + "annotationValue": "growth condition", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/organism_part_332", + "characteristicType": { + "annotationValue": "organism part", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#characteristic_category/sample_description_333", + "characteristicType": { + "annotationValue": "sample description", + "termAccession": "", + "termSource": "" + } } ], "description": "Nucleic acid sequencing and metabolomics and proteomics of Arabidopsis thaliana in specific experimental conditions to test a specific hypothesis.\r\n", @@ -699,18 +667,6 @@ "termSource": "" } }, - { - "comments": [], - "category": { - "@id": "#characteristic_category/cell_type_321" - }, - "value": { - "comments": [], - "annotationValue": "na", - "termAccession": "", - "termSource": "" - } - }, { "comments": [], "category": { @@ -718,7 +674,7 @@ }, "value": { "comments": [], - "annotationValue": "budding", + "annotationValue": "vegetative rosette stage", "termAccession": "", "termSource": "" } @@ -730,7 +686,7 @@ }, "value": { "comments": [], - "annotationValue": "01/01/2022", + "annotationValue": "2022-01-01", "termAccession": "", "termSource": "" } @@ -742,7 +698,7 @@ }, "value": { "comments": [], - "annotationValue": "seed", + "annotationValue": "whole plant", "termAccession": "", "termSource": "" } @@ -795,6 +751,26 @@ "termSource": "" } }, + { + "category": { + "@id": "#characteristic_category/genotype_330" + }, + "value": { + "annotationValue": "Col-0", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/growth_condition_331" + }, + "value": { + "annotationValue": "16 h light / 8 h dark growth chamber", + "termAccession": "", + "termSource": "" + } + }, { "comments": [], "category": { @@ -839,6 +815,26 @@ "@id": "#ontology_annotation/accession_#sample/331", "annotationValue": "SAMEA131504584" } + }, + { + "category": { + "@id": "#characteristic_category/organism_part_332" + }, + "value": { + "annotationValue": "leaf", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#characteristic_category/sample_description_333" + }, + "value": { + "annotationValue": "young rosette leaf collected for DNA extraction", + "termAccession": "", + "termSource": "" + } } ], "factorValues": [], @@ -914,7 +910,38 @@ "derivesFrom": [] } ], - "parameterValues": [], + "parameterValues": [ + { + "category": { + "@id": "#parameter/sample_collection_isolation_source" + }, + "value": { + "annotationValue": "leaf tissue", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_method" + }, + "value": { + "annotationValue": "sterile scalpel excision", + "termAccession": "", + "termSource": "" + } + }, + { + "category": { + "@id": "#parameter/sample_collection_preservation" + }, + "value": { + "annotationValue": "flash frozen in liquid nitrogen", + "termAccession": "", + "termSource": "" + } + } + ], "performer": "", "previousProcess": { "comments": [], @@ -941,7 +968,32 @@ ], "description": "", "name": "sample collection", - "parameters": [], + "parameters": [ + { + "@id": "#parameter/sample_collection_isolation_source", + "parameterName": { + "annotationValue": "isolation_source", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_method", + "parameterName": { + "annotationValue": "collection method", + "termAccession": "", + "termSource": "" + } + }, + { + "@id": "#parameter/sample_collection_preservation", + "parameterName": { + "annotationValue": "sample preservation", + "termAccession": "", + "termSource": "" + } + } + ], "protocolType": { "comments": [], "annotationValue": "sample collection", @@ -1115,4 +1167,4 @@ "submissionDate": "", "title": "Bob's investigation" } -} \ No newline at end of file +} diff --git a/tests/fixtures/mars_receipts/biosamples_success_response.json b/tests/fixtures/mars_receipts/biosamples_success_response.json index 1fb4458..d7ef41a 100644 --- a/tests/fixtures/mars_receipts/biosamples_success_response.json +++ b/tests/fixtures/mars_receipts/biosamples_success_response.json @@ -13,7 +13,7 @@ "key": "studies", "where": { "key": "title", - "value": "Arabidopsis thaliana" + "value": "Integrated multi-omics profiling of Arabidopsis thaliana under controlled experimental conditions" } }, { @@ -38,7 +38,7 @@ "key": "studies", "where": { "key": "title", - "value": "Arabidopsis thaliana" + "value": "Integrated multi-omics profiling of Arabidopsis thaliana under controlled experimental conditions" } }, { diff --git a/tests/test_isa_json.py b/tests/test_isa_json.py index b358c63..e1d6d38 100644 --- a/tests/test_isa_json.py +++ b/tests/test_isa_json.py @@ -35,7 +35,10 @@ def test_load_isa_json(): # Should test the validation process of the ISA JSON file where root has 'investigation' as key. valid_isa_json02 = load_isa_json("MARS/test-data/biosamples-input-isa.json", False) assert len(valid_isa_json02.investigation.studies) == 1 - assert valid_isa_json02.investigation.studies[0].title == "Arabidopsis thaliana" + assert ( + valid_isa_json02.investigation.studies[0].title + == "Integrated multi-omics profiling of Arabidopsis thaliana under controlled experimental conditions" + ) with pytest.raises(ValidationError): load_isa_json("./tests/fixtures/invalid_investigation.json", True) @@ -245,7 +248,102 @@ def test_update_study_materials_with_accession_categories(): ) -def test_update_study_and_assay_with_ena_study_accession_comment(): +def test_update_study_materials_multiple_samples_from_one_source(): + json_path = "MARS/test-data/biosamples-input-isa-multi.json" + with open(json_path) as json_file: + json_data = json.load(json_file) + + study = json_data["investigation"]["studies"][0] + root_sample = next( + sample for sample in study["materials"]["samples"] if sample["name"] == "root 1" + ) + root_sample["derivesFrom"][0]["@id"] = "#source/330" + root_process = next( + process + for process in study["processSequence"] + if process["@id"] == "#process/sample_collection/431" + ) + root_process["inputs"][0]["@id"] = "#source/330" + + validated_isa_json = IsaJson.model_validate(json_data) + repo_response = RepositoryResponse.model_validate( + { + "targetRepository": "biosamples", + "errors": [], + "info": [], + "accessions": [ + { + "value": "SAMEA_SOURCE_1", + "path": [ + {"key": "investigation"}, + { + "key": "studies", + "where": { + "key": "title", + "value": study["title"], + }, + }, + {"key": "materials"}, + { + "key": "sources", + "where": {"key": "name", "value": "plant 1"}, + }, + ], + }, + { + "value": "SAMEA_LEAF_1", + "path": [ + {"key": "investigation"}, + { + "key": "studies", + "where": { + "key": "title", + "value": study["title"], + }, + }, + {"key": "materials"}, + { + "key": "samples", + "where": {"key": "name", "value": "leaf 1"}, + }, + ], + }, + { + "value": "SAMEA_ROOT_1", + "path": [ + {"key": "investigation"}, + { + "key": "studies", + "where": { + "key": "title", + "value": study["title"], + }, + }, + {"key": "materials"}, + { + "key": "samples", + "where": {"key": "name", "value": "root 1"}, + }, + ], + }, + ], + } + ) + + updated_isa_json = update_isa_json(validated_isa_json, repo_response) + updated_study = updated_isa_json.investigation.studies[0] + updated_samples = updated_study.materials.samples + + assert updated_study.materials.sources[0].characteristics[-1].value.annotationValue == ( + "SAMEA_SOURCE_1" + ) + assert updated_samples[0].derivesFrom[0].id == "#source/330" + assert updated_samples[0].characteristics[-1].value.annotationValue == "SAMEA_LEAF_1" + assert updated_samples[1].derivesFrom[0].id == "#source/330" + assert updated_samples[1].characteristics[-1].value.annotationValue == "SAMEA_ROOT_1" + + +def test_update_study_only_with_ena_study_accession_comment(): json_path = "tests/fixtures/isa_jsons/1_after_biosamples.json" isa_json = load_isa_json(json_path, False) response_file_path = "tests/fixtures/mars_receipts/ena_success_response.json" @@ -267,10 +365,10 @@ def test_update_study_and_assay_with_ena_study_accession_comment(): None, ) assay_comments = ena_assay.comments - accession_comment = filter( - lambda x: x.name == "ena_study_accession", assay_comments - ) - assert next(accession_comment).value == ena_study_accession_number + accession_comments = [ + comment for comment in assay_comments if comment.name == "ena_study_accession" + ] + assert accession_comments == [] def test_update_datafile_comment_with_accession_comment_present():