From 65f67cc2c0227393e99e929ff99f35da024d1bef Mon Sep 17 00:00:00 2001 From: Marc Balestreri Date: Sat, 11 Jul 2026 14:45:47 -0700 Subject: [PATCH 1/6] feat: add GeoSeriesMatrix reader for tag-based GEO parsing --- biolearn/data_library.py | 73 +++++++++++++++++++++++++ biolearn/test/test_geo_series_matrix.py | 53 ++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 biolearn/test/test_geo_series_matrix.py diff --git a/biolearn/data_library.py b/biolearn/data_library.py index b0c62f9..fd1d045 100644 --- a/biolearn/data_library.py +++ b/biolearn/data_library.py @@ -53,6 +53,79 @@ def extract_informal_age(char): } +def _open_series_text(path): + with open(path, "rb") as probe: + is_gzip = probe.read(2) == b"\x1f\x8b" + if is_gzip: + return gzip.open(path, "rt", encoding="utf-8", errors="replace") + return open(path, "rt", encoding="utf-8", errors="replace") + + +class GeoSeriesMatrix: + """Reads a GEO series-matrix header and resolves rows by tag and + characteristic key rather than by absolute line number, which shifts + when GEO re-versions a series.""" + + def __init__(self, file_path): + local_path = cached_download(file_path) + self._tags = {} + self._tag_line = {} + self._characteristics = [] + self._line_values = {} + self.matrix_start = None + with _open_series_text(local_path) as handle: + for lineno, raw in enumerate(handle, start=1): + line = raw.rstrip("\n") + if line.startswith("!series_matrix_table_begin"): + self.matrix_start = lineno + 1 + break + if not line.startswith("!Sample_"): + continue + parts = line.split("\t") + tag = parts[0] + values = [p.strip().strip('"') for p in parts[1:]] + self._line_values[lineno] = values + if tag == "!Sample_characteristics_ch1": + self._characteristics.append( + (self._value_key(values), values) + ) + else: + self._tags.setdefault(tag, values) + self._tag_line.setdefault(tag, lineno) + + @staticmethod + def _value_key(values): + for value in values: + if value and ":" in value: + return value.split(":", 1)[0].strip().lower() + return None + + @property + def id_row(self): + return self._tag_line.get("!Sample_geo_accession") + + def sample_ids(self): + return self._tags.get("!Sample_geo_accession") + + def tag_values(self, tag): + return self._tags.get(tag) + + def characteristic_values(self, key): + key = key.strip().lower() + for candidate_key, values in self._characteristics: + if candidate_key == key: + return values + return None + + def line_values(self, lineno): + return self._line_values.get(lineno) + + def id_offset(self, configured_id_row): + if configured_id_row is None or self.id_row is None: + return 0 + return self.id_row - configured_id_row + + def build_column_mapping(matrix_file_path, from_key_line, to_key_line): # Use the key line for the mapping mapping_df = pd.read_table( diff --git a/biolearn/test/test_geo_series_matrix.py b/biolearn/test/test_geo_series_matrix.py new file mode 100644 index 0000000..00d0a2b --- /dev/null +++ b/biolearn/test/test_geo_series_matrix.py @@ -0,0 +1,53 @@ +from biolearn.data_library import GeoSeriesMatrix +from biolearn.util import get_test_data_file + + +def _series(): + return GeoSeriesMatrix(get_test_data_file("geo_dnam_test_file")) + + +def test_sample_ids_from_geo_accession(): + series = _series() + ids = series.sample_ids() + assert ids == [ + "GSM1009660", + "GSM1009661", + "GSM1009662", + "GSM1009663", + "GSM1009664", + ] + + +def test_id_row_is_geo_accession_line(): + assert _series().id_row == 33 + + +def test_matrix_start_is_after_table_begin(): + # table begin marker is on line 73, header row follows on 74 + assert _series().matrix_start == 74 + + +def test_tag_values_returns_sample_tag(): + titles = _series().tag_values("!Sample_title") + assert len(titles) == 5 + assert _series().tag_values("!Sample_not_present") is None + + +def test_characteristic_by_key_is_case_insensitive(): + series = _series() + gender = series.characteristic_values("Gender") + assert len(gender) == 5 + assert all(v.lower().startswith("gender:") for v in gender) + assert series.characteristic_values("no-such-key") is None + + +def test_line_values_is_one_based(): + # line 33 is the geo accession row + assert _series().line_values(33)[0] == "GSM1009660" + + +def test_id_offset(): + series = _series() + assert series.id_offset(33) == 0 + assert series.id_offset(31) == 2 + assert series.id_offset(None) == 0 From 8e17c69fb6926291fa4c7dd2c5ede98556d8ffc8 Mon Sep 17 00:00:00 2001 From: Marc Balestreri Date: Sat, 11 Jul 2026 14:48:04 -0700 Subject: [PATCH 2/6] feat: resolve GEO column mapping by tag with loud failure --- biolearn/data_library.py | 28 ++++++++++---------- biolearn/test/test_geo_series_matrix.py | 34 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/biolearn/data_library.py b/biolearn/data_library.py index fd1d045..4af25b3 100644 --- a/biolearn/data_library.py +++ b/biolearn/data_library.py @@ -126,20 +126,20 @@ def id_offset(self, configured_id_row): return self.id_row - configured_id_row -def build_column_mapping(matrix_file_path, from_key_line, to_key_line): - # Use the key line for the mapping - mapping_df = pd.read_table( - matrix_file_path, - index_col=0, - skiprows=lambda x: x != from_key_line - 1 and x != to_key_line - 1, - ) - column_mapping = mapping_df.to_dict("records")[0] - - # Reverse the mapping if needed as key is based on first line loaded - reverse_mapping = to_key_line < from_key_line - if reverse_mapping: - column_mapping = {v: k for k, v in column_mapping.items()} - return column_mapping +def build_column_mapping(series, key_tag=None, key_line=None, offset=0): + if key_tag is not None: + keys = series.tag_values(key_tag) + elif key_line is not None: + keys = series.line_values(key_line + offset) + else: + raise ValueError("build_column_mapping needs key_tag or key_line") + ids = series.sample_ids() + if keys is None or ids is None: + raise ValueError( + "Series matrix is missing the columns needed to map samples; " + "the GEO header may have changed" + ) + return dict(zip(keys, ids)) def map_and_prune_columns(data, column_mapping): diff --git a/biolearn/test/test_geo_series_matrix.py b/biolearn/test/test_geo_series_matrix.py index 00d0a2b..4995a4a 100644 --- a/biolearn/test/test_geo_series_matrix.py +++ b/biolearn/test/test_geo_series_matrix.py @@ -51,3 +51,37 @@ def test_id_offset(): assert series.id_offset(33) == 0 assert series.id_offset(31) == 2 assert series.id_offset(None) == 0 + + +import pytest +import pandas as pd +from biolearn.data_library import build_column_mapping, map_and_prune_columns + + +def test_build_column_mapping_by_tag(): + series = _series() + mapping = build_column_mapping(series, key_tag="!Sample_title") + assert len(mapping) == 5 + assert all(str(v).startswith("GSM") for v in mapping.values()) + + +def test_build_column_mapping_by_offset_corrected_line(): + series = _series() + # !Sample_title is line 32; feed a stale line 30 with a +2 offset + mapping = build_column_mapping(series, key_line=30, offset=2) + assert len(mapping) == 5 + assert all(str(v).startswith("GSM") for v in mapping.values()) + + +def test_build_column_mapping_missing_source_raises(): + with pytest.raises(ValueError): + build_column_mapping(_series(), key_tag="!Sample_not_present") + + +def test_map_and_prune_keeps_only_mapped_columns(): + data = pd.DataFrame( + {"sentrixA": [1, 2], "sentrixB": [3, 4], "junk": [5, 6]} + ) + mapping = {"sentrixA": "GSM1", "sentrixB": "GSM2"} + pruned = map_and_prune_columns(data, mapping) + assert list(pruned.columns) == ["GSM1", "GSM2"] From 3aba82843e65877c522b00386890dd505fec2276 Mon Sep 17 00:00:00 2001 From: Marc Balestreri Date: Sat, 11 Jul 2026 14:57:24 -0700 Subject: [PATCH 3/6] feat: resolve GEO metadata by key/tag with offset fallback --- biolearn/data_library.py | 61 +++++++++++++++++++------ biolearn/test/test_geo_series_matrix.py | 46 +++++++++++++++++++ 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/biolearn/data_library.py b/biolearn/data_library.py index 4af25b3..b2219b3 100644 --- a/biolearn/data_library.py +++ b/biolearn/data_library.py @@ -150,23 +150,54 @@ def map_and_prune_columns(data, column_mapping): return data -def load_geo_metadata(metadata_file, filekey, id_row): - load_list = [(key, filekey[key]["row"] - 1) for key in filekey.keys()] - load_list.sort(key=lambda x: x[1]) - load_rows = [x[1] for x in load_list] - column_names = [x[0] for x in load_list] - metadata = pd.read_table( - metadata_file, - index_col=0, - skiprows=lambda x: x != id_row - 1 and x not in load_rows, +def _resolve_field_values(series, spec, offset): + if spec.get("key") is not None: + values = series.characteristic_values(spec["key"]) + source = "characteristic '%s'" % spec["key"] + elif spec.get("tag") is not None: + values = series.tag_values(spec["tag"]) + source = "tag '%s'" % spec["tag"] + elif spec.get("row") is not None: + values = series.line_values(spec["row"] + offset) + source = "row %s" % (spec["row"] + offset) + else: + raise ValueError("Metadata field must specify key, tag, or row") + if values is None: + raise ValueError( + "Series matrix has no %s; the GEO header may have changed" % source + ) + return values + + +def _validate_metadata(metadata, filekey, strict): + if not strict or len(metadata) < 2: + return + for field, spec in filekey.items(): + if spec["parse"] not in ("sex", "numeric"): + continue + if metadata[field].notna().sum() == 0: + raise ValueError( + "Metadata field '%s' resolved to no usable values; the GEO " + "header likely changed" % field + ) + + +def load_geo_metadata(series, filekey, id_row): + offset = series.id_offset(id_row) + uses_semantic = any( + spec.get("key") is not None or spec.get("tag") is not None + for spec in filekey.values() ) - metadata.index = column_names - metadata = metadata.transpose() + columns = {} + for field, spec in filekey.items(): + values = _resolve_field_values(series, spec, offset) + parser = parsers[spec["parse"]] + columns[field] = [parser(value) for value in values] + metadata = pd.DataFrame(columns, index=series.sample_ids()) metadata.index.name = "id" - for col in metadata.columns: - parser_name = filekey[col]["parse"] - parser = parsers[parser_name] - metadata[col] = metadata[col].apply(parser) + _validate_metadata( + metadata, filekey, strict=(offset != 0 or uses_semantic) + ) return metadata diff --git a/biolearn/test/test_geo_series_matrix.py b/biolearn/test/test_geo_series_matrix.py index 4995a4a..95d4131 100644 --- a/biolearn/test/test_geo_series_matrix.py +++ b/biolearn/test/test_geo_series_matrix.py @@ -85,3 +85,49 @@ def test_map_and_prune_keeps_only_mapped_columns(): mapping = {"sentrixA": "GSM1", "sentrixB": "GSM2"} pruned = map_and_prune_columns(data, mapping) assert list(pruned.columns) == ["GSM1", "GSM2"] + + +from biolearn.data_library import load_geo_metadata + + +def test_metadata_by_key_ignores_line_numbers(): + series = _series() + filekey = {"age": {"key": "age", "parse": "numeric"}} + meta = load_geo_metadata(series, filekey, id_row=33) + assert list(meta.index) == series.sample_ids() + assert meta["age"].notna().all() + + +def test_metadata_by_tag(): + series = _series() + filekey = {"title": {"tag": "!Sample_title", "parse": "string"}} + meta = load_geo_metadata(series, filekey, id_row=33) + assert meta["title"].notna().all() + + +def test_metadata_offset_corrects_stale_rows(): + series = _series() + # age characteristic is on line 47. Pretend the config was written when + # the header sat two lines higher: id_row 31 (real is 33) and age row 45. + filekey = {"age": {"row": 45, "parse": "numeric"}} + meta = load_geo_metadata(series, filekey, id_row=31) + assert meta["age"].notna().all() + + +def test_metadata_all_unparseable_raises_when_corrected(): + series = _series() + # With a +2 offset applied, a bad row that lands on a digit-free + # characteristic must fail loudly rather than return a NaN column. + # row 40 + offset 2 = line 42, "sample type: whole blood". + filekey = {"age": {"row": 40, "parse": "numeric"}} + with pytest.raises(ValueError): + load_geo_metadata(series, filekey, id_row=31) + + +def test_legacy_offset_zero_does_not_validate(): + series = _series() + # Offset 0, legacy row that happens to be non-numeric stays permissive + # so the 40 unchanged datasets keep their exact behavior. + filekey = {"plate": {"row": 43, "parse": "string"}} + meta = load_geo_metadata(series, filekey, id_row=33) + assert "plate" in meta.columns From fee4c503d79d47c325fc8e4f7ede12dfba8afd80 Mon Sep 17 00:00:00 2001 From: Marc Balestreri Date: Sat, 11 Jul 2026 15:20:37 -0700 Subject: [PATCH 4/6] feat: route GeoMatrixParser through GeoSeriesMatrix reader --- biolearn/data_library.py | 57 ++++++++++--------------- biolearn/test/test_data_library.py | 11 ++--- biolearn/test/test_geo_series_matrix.py | 9 ++++ 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/biolearn/data_library.py b/biolearn/data_library.py index b2219b3..255f6fc 100644 --- a/biolearn/data_library.py +++ b/biolearn/data_library.py @@ -1009,8 +1009,6 @@ class GeoMatrixParser: seperators = {"space": " ", "comma": ",", "tab": "\t"} def __init__(self, data): - if data.get("id-row") is None: - raise ValueError("Parser not valid: missing id-row") self.id_row = data.get("id-row") self.metadata = data.get("metadata") self.matrix_start = data.get("matrix-start") @@ -1019,14 +1017,17 @@ def __init__(self, data): data.get("matrix-file-seperator") ) self.matrix_file_key_line = data.get("matrix-file-key-line") + self.matrix_file_key_tag = data.get("matrix-file-key-tag") self.matrix_file_format = data.get("matrix-file-format") self.data_type = data.get("data-type") def parse(self, file_path): - metadata = load_geo_metadata(file_path, self.metadata, self.id_row) + series = GeoSeriesMatrix(file_path) + metadata = load_geo_metadata(series, self.metadata, self.id_row) if self.matrix_start: + matrix_start = series.matrix_start or self.matrix_start matrix_data = pd.read_table( - file_path, index_col=0, skiprows=self.matrix_start - 1 + file_path, index_col=0, skiprows=matrix_start - 1 ) matrix_data = matrix_data.drop( ["!series_matrix_table_end"], axis=0 @@ -1050,13 +1051,13 @@ def parse(self, file_path): # NaN values in pval_df will cause corresponding values in methylation_df to be NaN matrix_data = reading_df + pval_df.values matrix_data = self._remap_and_prune_columns( - matrix_data, file_path + matrix_data, series ) elif self.matrix_file_format == "standard": matrix_data = df matrix_data = self._remap_and_prune_columns( - matrix_data, file_path + matrix_data, series ) else: @@ -1069,37 +1070,25 @@ def parse(self, file_path): else: return GeoData(metadata, dnam=matrix_data) - def _remap_and_prune_columns(self, data, matrix_file_path): - if self.matrix_file_key_line is None: - # No key line for mapping so assume the ordering is sufficient - header_row = pd.read_table( - matrix_file_path, - index_col=0, - header=None, - skiprows=lambda x: x != self.id_row - 1, - nrows=1, + def _remap_and_prune_columns(self, data, series): + offset = series.id_offset(self.id_row) + if self.matrix_file_key_tag is not None: + column_mapping = build_column_mapping( + series, key_tag=self.matrix_file_key_tag + ) + elif self.matrix_file_key_line is not None: + column_mapping = build_column_mapping( + series, key_line=self.matrix_file_key_line, offset=offset ) - column_mapping = dict(zip(data.columns, header_row.iloc[0])) else: - # Use the key line for the mapping - mapping_df = pd.read_table( - matrix_file_path, - index_col=0, - skiprows=lambda x: x != self.id_row - 1 - and x != self.matrix_file_key_line - 1, + column_mapping = dict(zip(data.columns, series.sample_ids())) + pruned = map_and_prune_columns(data, column_mapping) + if pruned.shape[1] == 0: + raise ValueError( + "No sample columns matched the series matrix; the GEO header " + "may have changed" ) - column_mapping = mapping_df.to_dict("records")[0] - - # Reverse the mapping if needed as key is based on first line loaded - reverse_mapping = self.id_row < self.matrix_file_key_line - if reverse_mapping: - column_mapping = {v: k for k, v in column_mapping.items()} - - data = data.rename(columns=column_mapping) - data = data[ - [col for col in data.columns if col in column_mapping.values()] - ] - return data + return pruned def _metadata_load_list(self): load_list = [ diff --git a/biolearn/test/test_data_library.py b/biolearn/test/test_data_library.py index 871631d..c4a8fb1 100644 --- a/biolearn/test/test_data_library.py +++ b/biolearn/test/test_data_library.py @@ -142,7 +142,10 @@ def test_missing_path_gives_error(): assert str(e.value) == "'path' key is missing in item" -def test_parser_missing_id_row_gives_error(): +def test_parser_id_row_is_optional(): + # id-row is optional now. The parser auto-detects the sample row from the + # series matrix, so an absent id-row leaves the attribute as None rather + # than raising. file_contents = """ --- items: @@ -150,7 +153,6 @@ def test_parser_missing_id_row_gives_error(): path: https://ftp.ncbi.nlm.nih.gov/geo/series/GSE40nnn/GSE40279/matrix/GSE40279_series_matrix.txt.gz parser: type: geo-matrix - misspelled-id-row: 12 metadata: age: row: 44 @@ -158,9 +160,8 @@ def test_parser_missing_id_row_gives_error(): matrix-start: 72 """ test_file = StringIO(file_contents) - with pytest.raises(ValueError) as e: - parse_library_file(test_file) - assert str(e.value) == "Parser not valid: missing id-row" + sources = parse_library_file(test_file) + assert sources[0].parser.id_row is None def test_missing_parser_gives_error(): diff --git a/biolearn/test/test_geo_series_matrix.py b/biolearn/test/test_geo_series_matrix.py index 95d4131..2708609 100644 --- a/biolearn/test/test_geo_series_matrix.py +++ b/biolearn/test/test_geo_series_matrix.py @@ -131,3 +131,12 @@ def test_legacy_offset_zero_does_not_validate(): filekey = {"plate": {"row": 43, "parse": "string"}} meta = load_geo_metadata(series, filekey, id_row=33) assert "plate" in meta.columns + + +from biolearn.data_library import GeoMatrixParser + + +def test_geo_matrix_parser_id_row_optional(): + # id-row is now optional; auto-detected from the series matrix + parser = GeoMatrixParser({"type": "geo-matrix", "matrix-start": 74}) + assert parser.id_row is None From c8fb74be2b048d77f951ba27c0c1e9229f652b7a Mon Sep 17 00:00:00 2001 From: Marc Balestreri Date: Sat, 11 Jul 2026 15:21:53 -0700 Subject: [PATCH 5/6] feat: route ChallengeDataParser through GeoSeriesMatrix reader --- biolearn/data_library.py | 25 ++++++++++++++++++------- biolearn/test/test_geo_series_matrix.py | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/biolearn/data_library.py b/biolearn/data_library.py index 255f6fc..bdf9ee6 100644 --- a/biolearn/data_library.py +++ b/biolearn/data_library.py @@ -762,12 +762,11 @@ def parse(self, _): class ChallengeDataParser: def __init__(self, data): - if data.get("id-row") is None: - raise ValueError("Parser not valid: missing id-row") self.id_row = data.get("id-row") self.metadata = data.get("metadata") self.matrix_file = data.get("matrix-file") self.matrix_file_key_line = data.get("matrix-file-key-line") + self.matrix_file_key_tag = data.get("matrix-file-key-tag") self.data_type = data.get("data-type") self.protein_matrix_url = "https://storage.googleapis.com/boa-challenge-2024/challenge_alamar_data.csv" self.metadata_url = "https://storage.googleapis.com/boa-challenge-2024/challenge_proteomic_metadata.csv" @@ -775,13 +774,25 @@ def __init__(self, data): def parse(self, file_path): print("Note: This dataset will take a few minutes to load") - # Load methylation data and metadata from GEO - metadata = load_geo_metadata(file_path, self.metadata, self.id_row) + series = GeoSeriesMatrix(file_path) + metadata = load_geo_metadata(series, self.metadata, self.id_row) dnam_data = pd.read_csv(self.matrix_file, index_col=0) - column_mapping = build_column_mapping( - file_path, self.matrix_file_key_line, self.id_row - ) + if self.matrix_file_key_tag is not None: + column_mapping = build_column_mapping( + series, key_tag=self.matrix_file_key_tag + ) + else: + column_mapping = build_column_mapping( + series, + key_line=self.matrix_file_key_line, + offset=series.id_offset(self.id_row), + ) fixed_dnam = map_and_prune_columns(dnam_data, column_mapping) + if fixed_dnam.shape[1] == 0: + raise ValueError( + "No sample columns matched the series matrix; the GEO header " + "may have changed" + ) geodata = GeoData.from_methylation_matrix(fixed_dnam) geodata.metadata = metadata diff --git a/biolearn/test/test_geo_series_matrix.py b/biolearn/test/test_geo_series_matrix.py index 2708609..ae67741 100644 --- a/biolearn/test/test_geo_series_matrix.py +++ b/biolearn/test/test_geo_series_matrix.py @@ -140,3 +140,18 @@ def test_geo_matrix_parser_id_row_optional(): # id-row is now optional; auto-detected from the series matrix parser = GeoMatrixParser({"type": "geo-matrix", "matrix-start": 74}) assert parser.id_row is None + + +from biolearn.data_library import ChallengeDataParser + + +def test_challenge_parser_accepts_key_tag_and_optional_id_row(): + parser = ChallengeDataParser( + { + "matrix-file": "ftp://example/betas.csv.gz", + "matrix-file-key-tag": "!Sample_description", + "metadata": {}, + } + ) + assert parser.matrix_file_key_tag == "!Sample_description" + assert parser.id_row is None From d4313120e54eaa754ea3da1ad51416034b52903f Mon Sep 17 00:00:00 2001 From: Marc Balestreri Date: Sat, 11 Jul 2026 15:25:38 -0700 Subject: [PATCH 6/6] fix: resolve BoAChallengeData by tag and characteristic key (#208) --- biolearn/data/library.yaml | 17 ++++--- biolearn/test/test_geo_series_matrix.py | 63 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/biolearn/data/library.yaml b/biolearn/data/library.yaml index 387549f..feebcc6 100644 --- a/biolearn/data/library.yaml +++ b/biolearn/data/library.yaml @@ -903,30 +903,29 @@ items: path: https://ftp.ncbi.nlm.nih.gov/geo/series/GSE246nnn/GSE246337/matrix/GSE246337_series_matrix.txt.gz parser: type: biomarkers-challenge-2024 - id-row: 31 matrix-file: ftp://ftp.ncbi.nlm.nih.gov/geo/series/GSE246nnn/GSE246337/suppl/GSE246337%5Fbetas.csv.gz - matrix-file-key-line: 53 + matrix-file-key-tag: "!Sample_description" metadata: subject_id: - row: 39 + key: "subject id" parse: string sex: - row: 41 + key: "Sex" parse: sex tissue: - row: 40 + key: "tissue" parse: string race1: - row: 42 + key: "race1" parse: string race2: - row: 43 + key: "race2" parse: string ethnicity: - row: 44 + key: "ethnicity" parse: string age: - row: 45 + key: "age" parse: numeric datalinks: - https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE246337 diff --git a/biolearn/test/test_geo_series_matrix.py b/biolearn/test/test_geo_series_matrix.py index ae67741..6701bf9 100644 --- a/biolearn/test/test_geo_series_matrix.py +++ b/biolearn/test/test_geo_series_matrix.py @@ -155,3 +155,66 @@ def test_challenge_parser_accepts_key_tag_and_optional_id_row(): ) assert parser.matrix_file_key_tag == "!Sample_description" assert parser.id_row is None + + +import os + + +def _write_boa_like_header(tmp_path): + # Old BoA config assumed id-row 31 and key line 53. Emulate the current + # file where everything sits one line lower, with three samples. + lines = [f"!Series_line_{n}" for n in range(1, 31)] + lines.append('!Sample_title\t"t1"\t"t2"\t"t3"') # line 31 + lines.append('!Sample_geo_accession\t"GSM1"\t"GSM2"\t"GSM3"') # line 32 + while len(lines) < 39: + lines.append(f'!Sample_pad_{len(lines)}\t"x"\t"x"\t"x"') + lines.append( + '!Sample_characteristics_ch1\t"subject id: A"\t"subject id: B"\t"subject id: C"' + ) # 40 + lines.append( + '!Sample_characteristics_ch1\t"tissue: blood"\t"tissue: blood"\t"tissue: blood"' + ) # 41 + lines.append( + '!Sample_characteristics_ch1\t"Sex: M"\t"Sex: F"\t"Sex: M"' + ) # 42 + while len(lines) < 53: + lines.append(f'!Sample_pad_{len(lines)}\t"x"\t"x"\t"x"') + lines.append( + '!Sample_description\t"SENTRIX_1"\t"SENTRIX_2"\t"SENTRIX_3"' + ) # 54 + lines.append("!series_matrix_table_begin") + path = os.path.join(tmp_path, "boa_like_series_matrix.txt") + with open(path, "w") as handle: + handle.write("\n".join(lines) + "\n") + return path + + +def test_boa_like_offset_and_key_resolution(tmp_path): + path = _write_boa_like_header(str(tmp_path)) + series = GeoSeriesMatrix(path) + + # Sample ids come from geo accession regardless of the stale id-row + assert series.sample_ids() == ["GSM1", "GSM2", "GSM3"] + + # Column mapping by the description tag yields sentrix -> GSM for all 3 + mapping = build_column_mapping(series, key_tag="!Sample_description") + assert mapping == { + "SENTRIX_1": "GSM1", + "SENTRIX_2": "GSM2", + "SENTRIX_3": "GSM3", + } + + # Metadata by key lands on the right characteristics + filekey = { + "subject_id": {"key": "subject id", "parse": "string"}, + "sex": {"key": "Sex", "parse": "sex"}, + } + meta = load_geo_metadata(series, filekey, id_row=None) + assert list(meta.index) == ["GSM1", "GSM2", "GSM3"] + assert list(meta["subject_id"]) == ["A", "B", "C"] + assert meta["sex"].notna().all() + + # The stale legacy lines still work via the id-row offset (31 -> 32 = +1) + legacy = {"subject_id": {"row": 39, "parse": "string"}} + meta_legacy = load_geo_metadata(series, legacy, id_row=31) + assert list(meta_legacy["subject_id"]) == ["A", "B", "C"]