diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e5034d28..8d5a89a5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,11 +19,13 @@ authors: - `seqspec check` now emits warning diagnostics for overlapping read geometry, with guidance to use `seqspec index --no-overlap` when needed. - `seqspec check --skip external` runs schema and structural checks without accessing onlist or read resources. - A generated examples site under `docs/examples/site`, with rendered assay reports, read templates, region templates, and a searchable assay catalog. +- Optional `sequence_column_index` and `skip_rows` onlist fields for extracting sequences from whitespace-delimited tabular source files. ### Changed - `seqspec upgrade` now upgrades `0.3.0` specs to `0.4.0` in both implementations. - Python and Rust now share the same core command surface for `auth`, `check`, `find`, `file`, `format`, `index`, `info`, `init`, `insert`, `methods`, `modify`, `onlist`, `print`, `split`, `upgrade`, and `version`. +- Python and Rust apply the same onlist row-skipping and column-selection behavior to local, remote, and gzipped sources. - `seqspec build` is deprecated in both CLIs and remains as a compatibility stub. - Older specs are loaded more permissively before upgrade, which makes `0.2.x` and `0.3.x` specs easier to normalize. - `seqspec onlist -s region-type` now errors when matches span multiple reads in a modality. Use `-s read` or `-s region` to disambiguate. diff --git a/docs/SEQSPEC_TOOL.md b/docs/SEQSPEC_TOOL.md index 8f48184c..6069f74d 100644 --- a/docs/SEQSPEC_TOOL.md +++ b/docs/SEQSPEC_TOOL.md @@ -635,7 +635,7 @@ spec = load_spec("spec.yaml") get_onlists(spec, modality="rna", selector="region-type", id="barcode") ``` -- optionally, `-o OUT` when set with `-f`, writes the joined onlist to this file; when set without `-f`, downloads remote onlists locally and prints paths. +- optionally, `-o OUT` when set with `-f`, writes the joined onlist to this file; when set without `-f`, downloads remote onlists locally and prints paths. Local or remote onlists with `sequence_column_index` or `skip_rows` are written as normalized sequence-only files. - `-m MODALITY` is the modality in which you are searching for the region. - `-i ID` is the `id` of the object to search for the onlist. - `-s SELECTOR` is the type of the `id` of the object (default: read). Can be one of: @@ -648,6 +648,11 @@ get_onlists(spec, modality="rna", selector="region-type", id="barcode") - optionally, `--auth-profile PROFILE` uses a named auth profile for protected remote onlists. - `yaml` corresponds to the `seqspec` file and may be plain YAML or `.yaml.gz`. +Onlist rows are split on arbitrary whitespace. The optional onlist fields +`sequence_column_index` (zero-based, default `0`) and `skip_rows` (default `0`) +select sequences from tabular sources while retaining the source URL and checksum +in the specification. + _Note_: `-s region-type` is only valid when the matching regions come from one read geometry. If the same `region_type` appears across multiple reads in the modality, `seqspec onlist` errors and asks you to use `-s read` or `-s region` instead. ### Examples diff --git a/docs/SPECIFICATION.md b/docs/SPECIFICATION.md index d09684ea..da1d4e8e 100644 --- a/docs/SPECIFICATION.md +++ b/docs/SPECIFICATION.md @@ -179,6 +179,8 @@ Each `Region` has the following properties which are useful to annotate the elem - `url`: a freeform string that specifies either the url location of the file, or the local path of the file (relative to this seqspec file) - `urltype`: can be one of ["local", "ftp", "http", "https"] specifies the type of the `url` - `md5`: the md5sum of the uncompressed file in `filename`, must match the pattern `^[a-f0-9]{32}$` + - `sequence_column_index`: optional zero-based index of the whitespace-delimited field containing each sequence; defaults to `0` + - `skip_rows`: optional number of physical rows to skip before reading sequences; defaults to `0` - `regions` can either be `null` or contain a list of `regions` as specified above. Example: @@ -202,6 +204,27 @@ onlist: !Onlist regions: null ``` +Tabular source files can be used without creating a sequence-only derivative. For +example, an onlist whose first row is a header and whose second field contains the +sequence uses: + +```yaml +onlist: + file_id: vendor-plate.tsv + filename: vendor-plate.tsv + filetype: tsv + filesize: 120 + url: https://example.org/vendor-plate.tsv + urltype: https + md5: 5b62453df2771f5aa856f78797f16591 + sequence_column_index: 1 + skip_rows: 1 +``` + +Rows are split on arbitrary whitespace after the requested physical rows are +skipped. The `md5` continues to describe the referenced source file, before this +projection is applied. + For more information about the various fields, please see the JSON schema specification (`seqspec/schema/seqspec.schema.json`). For consistency across assays I suggest following a standard naming conventions for common regions. I've made a collection of "named" regions available; please see `docs/examples/regions` for a list of example regions. ## `Read` Object diff --git a/seqspec/Region.py b/seqspec/Region.py index a153b458..8ff1cfdd 100644 --- a/seqspec/Region.py +++ b/seqspec/Region.py @@ -30,6 +30,8 @@ class Onlist(BaseModel): url: str urltype: str md5: str + sequence_column_index: int = Field(default=0, ge=0) + skip_rows: int = Field(default=0, ge=0) # add a update_spec attribute that computes the md5 for the object @@ -82,6 +84,21 @@ class OnlistInput(BaseModel): default=None, description=("MD5 checksum of the on-list file if available; omit if unknown."), ) + sequence_column_index: Optional[int] = Field( + default=None, + ge=0, + description=( + "Zero-based whitespace-delimited field index containing each sequence. " + "Defaults to 0." + ), + ) + skip_rows: Optional[int] = Field( + default=None, + ge=0, + description=( + "Number of physical rows to skip before reading sequences. Defaults to 0." + ), + ) def to_onlist(self) -> Onlist: return Onlist( @@ -92,6 +109,8 @@ def to_onlist(self) -> Onlist: url=self.url or "", urltype=self.urltype or "local", md5=self.md5 or "", + sequence_column_index=self.sequence_column_index or 0, + skip_rows=self.skip_rows or 0, ) diff --git a/seqspec/_core.pyi b/seqspec/_core.pyi index ce900a69..6a99d85a 100644 --- a/seqspec/_core.pyi +++ b/seqspec/_core.pyi @@ -33,6 +33,8 @@ class Onlist: url: str urltype: str md5: str + sequence_column_index: int + skip_rows: int def __init__( self, @@ -43,6 +45,8 @@ class Onlist: url: str, urltype: str, md5: str, + sequence_column_index: int, + skip_rows: int, ) -> None: ... @staticmethod def from_json(json_str: str) -> "Onlist": ... diff --git a/seqspec/schema/seqspec.schema.json b/seqspec/schema/seqspec.schema.json index 2cb90a51..5d7d6dca 100644 --- a/seqspec/schema/seqspec.schema.json +++ b/seqspec/schema/seqspec.schema.json @@ -409,6 +409,18 @@ "description": "md5sum for the file pointed to by filename", "type": "string", "pattern": "^[a-f0-9]{32}$" + }, + "sequence_column_index": { + "description": "Zero-based whitespace-delimited field index containing each sequence. Defaults to 0.", + "type": "integer", + "minimum": 0, + "default": 0 + }, + "skip_rows": { + "description": "Number of physical rows to skip before reading sequences. Defaults to 0.", + "type": "integer", + "minimum": 0, + "default": 0 } }, "required": [ diff --git a/seqspec/seqspec_onlist.py b/seqspec/seqspec_onlist.py index 4bb9df24..e99282c8 100644 --- a/seqspec/seqspec_onlist.py +++ b/seqspec/seqspec_onlist.py @@ -242,19 +242,20 @@ def download_onlists_to_path( for onlist in onlists: if onlist.urltype == "local": - # Local file - just return the path local_path = base_path / Path(local_onlist_locator(onlist)) - downloaded_paths.append({"file_id": onlist.file_id, "url": str(local_path)}) + if onlist.sequence_column_index == 0 and onlist.skip_rows == 0: + downloaded_paths.append( + {"file_id": onlist.file_id, "url": str(local_path)} + ) + continue + onlist_elements = read_local_list(onlist, str(base_path)) else: - # Remote file - download it onlist_elements = read_remote_list(onlist, auth_profile=auth_profile) - # Create unique filename for this onlist - filename = f"{onlist.file_id}_{output_path.name}" - download_path = output_path.parent / filename - write_onlist(onlist_elements, download_path) - downloaded_paths.append( - {"file_id": onlist.file_id, "url": str(download_path)} - ) + + filename = f"{onlist.file_id}_{output_path.name}" + download_path = output_path.parent / filename + write_onlist(onlist_elements, download_path) + downloaded_paths.append({"file_id": onlist.file_id, "url": str(download_path)}) return downloaded_paths diff --git a/seqspec/utils.py b/seqspec/utils.py index 423d5add..1c6177b3 100644 --- a/seqspec/utils.py +++ b/seqspec/utils.py @@ -525,9 +525,20 @@ def write_pydantic_to_file_or_stdout( print(yaml.dump(dump, sort_keys=False)) -def yield_onlist_contents(stream): - for line in stream: - yield line.strip().split()[0] +def yield_onlist_contents(stream, sequence_column_index: int, skip_rows: int): + for row_number, line in enumerate(stream, start=1): + if row_number <= skip_rows: + continue + + fields = line.strip().split() + if not fields: + continue + if sequence_column_index >= len(fields): + raise ValueError( + f"onlist row {row_number} has {len(fields)} field(s); " + f"cannot select zero-based column index {sequence_column_index}" + ) + yield fields[sequence_column_index] def local_resource_url(url: str, filename: str, resource: str) -> str: @@ -551,7 +562,9 @@ def read_local_list(onlist: Onlist, base_path: str = "") -> List[str]: stream = io.TextIOWrapper(stream) results = [] - for i in yield_onlist_contents(stream): + for i in yield_onlist_contents( + stream, onlist.sequence_column_index, onlist.skip_rows + ): results.append(i) stream.close() return results @@ -587,7 +600,9 @@ def read_remote_list( stream = io.TextIOWrapper(binary_stream) results = [] - for i in yield_onlist_contents(stream): + for i in yield_onlist_contents( + stream, onlist.sequence_column_index, onlist.skip_rows + ): # add the new line when writing to file results.append(i) finally: diff --git a/src/compat.rs b/src/compat.rs index 2876f7c2..917c892c 100644 --- a/src/compat.rs +++ b/src/compat.rs @@ -268,6 +268,8 @@ pub struct CompatOnlist { pub url: Option, pub urltype: Option, pub md5: Option, + pub sequence_column_index: Option, + pub skip_rows: Option, } impl CompatOnlist { @@ -280,6 +282,8 @@ impl CompatOnlist { self.url.unwrap_or_default(), self.urltype.unwrap_or_else(|| "local".to_string()), self.md5.unwrap_or_default(), + self.sequence_column_index.unwrap_or_default(), + self.skip_rows.unwrap_or_default(), ) } } @@ -362,3 +366,29 @@ impl CompatRegion { ) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compat_onlist_preserves_projection_fields() { + let yaml = r#" +file_id: plate +filename: plate.tsv +filetype: tsv +filesize: 42 +url: plate.tsv +urltype: local +md5: abc123 +sequence_column_index: 1 +skip_rows: 1 +"#; + let compat: CompatOnlist = serde_yaml::from_str(yaml).unwrap(); + + let onlist = compat.into_onlist(); + + assert_eq!(onlist.sequence_column_index, 1); + assert_eq!(onlist.skip_rows, 1); + } +} diff --git a/src/models/onlist.rs b/src/models/onlist.rs index fbf64525..8b1a0389 100644 --- a/src/models/onlist.rs +++ b/src/models/onlist.rs @@ -9,6 +9,10 @@ pub struct Onlist { pub url: String, pub urltype: String, pub md5: String, + #[serde(default)] + pub sequence_column_index: usize, + #[serde(default)] + pub skip_rows: usize, } impl Onlist { @@ -20,6 +24,8 @@ impl Onlist { url: String, urltype: String, md5: String, + sequence_column_index: usize, + skip_rows: usize, ) -> Self { Self { file_id, @@ -29,6 +35,8 @@ impl Onlist { url, urltype, md5, + sequence_column_index, + skip_rows, } } @@ -53,6 +61,8 @@ mod tests { "barcodes.txt".into(), "local".into(), "abc123".into(), + 0, + 0, ) } @@ -66,16 +76,38 @@ mod tests { assert_eq!(ol.url, "barcodes.txt"); assert_eq!(ol.urltype, "local"); assert_eq!(ol.md5, "abc123"); + assert_eq!(ol.sequence_column_index, 0); + assert_eq!(ol.skip_rows, 0); } #[test] fn test_onlist_json_roundtrip() { - let ol = sample_onlist(); + let mut ol = sample_onlist(); + ol.sequence_column_index = 1; + ol.skip_rows = 1; let json = ol.to_json().unwrap(); let ol2 = Onlist::from_json(&json).unwrap(); assert_eq!(ol, ol2); } + #[test] + fn test_onlist_json_defaults_projection_for_legacy_data() { + let json = r#"{ + "file_id":"ol1", + "filename":"barcodes.txt", + "filetype":"txt", + "filesize":1024, + "url":"barcodes.txt", + "urltype":"local", + "md5":"abc123" + }"#; + + let onlist = Onlist::from_json(json).unwrap(); + + assert_eq!(onlist.sequence_column_index, 0); + assert_eq!(onlist.skip_rows, 0); + } + #[test] fn test_onlist_clone() { let ol = sample_onlist(); diff --git a/src/models/region.rs b/src/models/region.rs index ea5e2591..9009c034 100644 --- a/src/models/region.rs +++ b/src/models/region.rs @@ -562,6 +562,8 @@ mod tests { "list.txt".into(), "local".into(), "".into(), + 0, + 0, ); let mut r = Region::new( "r".into(), @@ -635,6 +637,8 @@ mod tests { "".into(), "local".into(), "".into(), + 0, + 0, ); let parent = joined( "parent", @@ -671,6 +675,8 @@ mod tests { "".into(), "local".into(), "".into(), + 0, + 0, ); let r2 = Region::new( "bc".into(), diff --git a/src/schema/seqspec.schema.json b/src/schema/seqspec.schema.json index 2cb90a51..5d7d6dca 100644 --- a/src/schema/seqspec.schema.json +++ b/src/schema/seqspec.schema.json @@ -409,6 +409,18 @@ "description": "md5sum for the file pointed to by filename", "type": "string", "pattern": "^[a-f0-9]{32}$" + }, + "sequence_column_index": { + "description": "Zero-based whitespace-delimited field index containing each sequence. Defaults to 0.", + "type": "integer", + "minimum": 0, + "default": 0 + }, + "skip_rows": { + "description": "Number of physical rows to skip before reading sequences. Defaults to 0.", + "type": "integer", + "minimum": 0, + "default": 0 } }, "required": [ diff --git a/src/seqspec_methods.rs b/src/seqspec_methods.rs index 8b765f47..a087629b 100644 --- a/src/seqspec_methods.rs +++ b/src/seqspec_methods.rs @@ -218,6 +218,8 @@ mod tests { "".into(), "local".into(), "".into(), + 0, + 0, ); let region = Region::new( "bc".into(), diff --git a/src/seqspec_onlist.rs b/src/seqspec_onlist.rs index 58125530..ed67636c 100644 --- a/src/seqspec_onlist.rs +++ b/src/seqspec_onlist.rs @@ -212,11 +212,44 @@ fn download_onlists_to_path( eprintln!("{}", err); std::process::exit(1); })); + if ol.sequence_column_index == 0 && ol.skip_rows == 0 { + out.push(PathInfo { + url: local.to_string_lossy().to_string(), + }); + continue; + } + let content = utils::read_local_list(&local, ol.sequence_column_index, ol.skip_rows) + .unwrap_or_else(|err| { + eprintln!("{}", err); + std::process::exit(1); + }); + let filename = format!( + "{}_{}", + ol.file_id, + output_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + ); + let download_path = output_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(filename); + write_onlist(&content, &download_path); out.push(PathInfo { - url: local.to_string_lossy().to_string(), + url: download_path.to_string_lossy().to_string(), }); } else { - let content = utils::read_remote_list(&ol.url, remote_access).unwrap_or_default(); + let content = utils::read_remote_list( + &ol.url, + remote_access, + ol.sequence_column_index, + ol.skip_rows, + ) + .unwrap_or_else(|err| { + eprintln!("{}", err); + std::process::exit(1); + }); let filename = format!( "{}_{}", ol.file_id, @@ -252,9 +285,26 @@ fn join_onlists_and_save( eprintln!("{}", err); std::process::exit(1); }); - utils::read_local_list(&base_path.join(locator)).unwrap_or_default() + utils::read_local_list( + &base_path.join(locator), + ol.sequence_column_index, + ol.skip_rows, + ) + .unwrap_or_else(|err| { + eprintln!("{}", err); + std::process::exit(1); + }) } else { - utils::read_remote_list(&ol.url, remote_access).unwrap_or_default() + utils::read_remote_list( + &ol.url, + remote_access, + ol.sequence_column_index, + ol.skip_rows, + ) + .unwrap_or_else(|err| { + eprintln!("{}", err); + std::process::exit(1); + }) }; contents.push(content); } @@ -475,6 +525,8 @@ mod tests { "nested/whitelist.txt".into(), "local".into(), String::new(), + 0, + 0, )]; let urls = get_onlist_urls(&onlists, &base); @@ -496,6 +548,8 @@ mod tests { "nested/whitelist.txt".into(), "local".into(), String::new(), + 0, + 0, )]; let output = root.join("joined.txt"); let remote_access = RemoteAccess::anonymous(); @@ -508,4 +562,31 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + + #[test] + fn test_download_projected_local_onlist_writes_normalized_copy() { + let root = unique_test_dir("seqspec-onlist-projection"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("plate.tsv"), "Name Barcode\nA01 AAAA\nA02 CCCC\n").unwrap(); + let onlist = Onlist::new( + "plate".into(), + "plate.tsv".into(), + "tsv".into(), + 0, + "plate.tsv".into(), + "local".into(), + String::new(), + 1, + 1, + ); + let output = root.join("normalized.txt"); + let remote_access = RemoteAccess::anonymous(); + + let downloaded = download_onlists_to_path(&vec![onlist], &output, &root, &remote_access); + + let normalized = PathBuf::from(&downloaded[0].url); + assert_ne!(normalized, root.join("plate.tsv")); + assert_eq!(std::fs::read_to_string(normalized).unwrap(), "AAAA\nCCCC\n"); + std::fs::remove_dir_all(root).unwrap(); + } } diff --git a/src/seqspec_upgrade.rs b/src/seqspec_upgrade.rs index c5706c1b..368e2a72 100644 --- a/src/seqspec_upgrade.rs +++ b/src/seqspec_upgrade.rs @@ -140,6 +140,8 @@ fn upgrade_0_2_0_to_0_4_0(spec: Assay) -> Assay { url: "".to_string(), urltype: "".to_string(), md5, + sequence_column_index: 0, + skip_rows: 0, }; // update the region by id with new onlist fields top.update_region_by_id( diff --git a/src/utils.rs b/src/utils.rs index c5a48d22..46cc6e1c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -171,8 +171,42 @@ pub fn local_onlist_locator(onlist: &Onlist) -> Result<&str, String> { local_resource_url(&onlist.url, &onlist.filename, "onlist") } -/// Read a local text file into Vec, handling optional .gz -pub fn read_local_list(path: &std::path::Path) -> Result, String> { +fn project_onlist_text( + text: &str, + sequence_column_index: usize, + skip_rows: usize, +) -> Result, String> { + text.lines() + .enumerate() + .skip(skip_rows) + .filter_map(|(row_index, line)| { + let fields: Vec<&str> = line.split_whitespace().collect(); + if fields.is_empty() { + return None; + } + Some( + fields + .get(sequence_column_index) + .map(|field| (*field).to_string()) + .ok_or_else(|| { + format!( + "onlist row {} has {} field(s); cannot select zero-based column index {}", + row_index + 1, + fields.len(), + sequence_column_index + ) + }), + ) + }) + .collect() +} + +/// Read and project a local onlist text file, handling optional .gz. +pub fn read_local_list( + path: &std::path::Path, + sequence_column_index: usize, + skip_rows: usize, +) -> Result, String> { let p = if path.exists() { path.to_path_buf() } else { @@ -188,21 +222,20 @@ pub fn read_local_list(path: &std::path::Path) -> Result, String> { let mut s = String::new(); use std::io::Read; dec.read_to_string(&mut s).map_err(|e| e.to_string())?; - Ok(s.lines() - .map(|l| l.trim().to_string()) - .filter(|l| !l.is_empty()) - .collect()) + project_onlist_text(&s, sequence_column_index, skip_rows) } else { let s = std::fs::read_to_string(&p).map_err(|e| e.to_string())?; - Ok(s.lines() - .map(|l| l.trim().to_string()) - .filter(|l| !l.is_empty()) - .collect()) + project_onlist_text(&s, sequence_column_index, skip_rows) } } -/// Fetch a remote text file (http/https/ftp) and return lines -pub fn read_remote_list(url: &str, remote_access: &RemoteAccess) -> Result, String> { +/// Fetch and project a remote onlist (http/https/ftp). +pub fn read_remote_list( + url: &str, + remote_access: &RemoteAccess, + sequence_column_index: usize, + skip_rows: usize, +) -> Result, String> { let text = remote_access .with_reader(url, |mut reader| { let mut data = Vec::new(); @@ -218,11 +251,7 @@ pub fn read_remote_list(url: &str, remote_access: &RemoteAccess) -> Result Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(text.as_bytes()).unwrap(); + encoder.finish().unwrap() + } + + fn serve_once(body: Vec) -> (String, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 4096]; + stream.read(&mut request).unwrap(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .unwrap(); + stream.write_all(&body).unwrap(); + }); + (format!("http://{}/plate.tsv.gz", addr), server) + } + fn dogma_spec() -> Assay { load_spec(&PathBuf::from("tests/fixtures/spec.yaml")) } @@ -654,7 +707,7 @@ mod tests { #[test] fn test_read_local_list_plain() { let path = PathBuf::from("tests/fixtures/onlist_joined.txt"); - let result = read_local_list(&path).unwrap(); + let result = read_local_list(&path, 0, 0).unwrap(); assert_eq!(result.len(), 736320); assert_eq!(result[0], "AAACAGCCAAACAACA"); } @@ -662,7 +715,7 @@ mod tests { #[test] fn test_read_local_list_gz() { let path = PathBuf::from("tests/fixtures/RNA-737K-arc-v1.txt.gz"); - let result = read_local_list(&path).unwrap(); + let result = read_local_list(&path, 0, 0).unwrap(); assert_eq!(result.len(), 736320); } @@ -670,17 +723,66 @@ mod tests { fn test_read_local_list_gz_fallback() { // Try path without .gz extension — read_local_list should find the .gz variant let path = PathBuf::from("tests/fixtures/RNA-737K-arc-v1.txt"); - let result = read_local_list(&path).unwrap(); + let result = read_local_list(&path, 0, 0).unwrap(); assert_eq!(result.len(), 736320); } #[test] fn test_read_local_list_not_found() { let path = PathBuf::from("tests/fixtures/nonexistent.txt"); - let result = read_local_list(&path); + let result = read_local_list(&path, 0, 0); assert!(result.is_err()); } + #[test] + fn test_read_local_list_projects_column_after_skipping_header() { + let path = PathBuf::from("tests/fixtures/tabular_onlist.txt"); + let result = read_local_list(&path, 1, 1).unwrap(); + + assert_eq!( + result, + vec!["TCAGTTGTCGAAGG".to_string(), "CTGGACCTAATACC".to_string()] + ); + } + + #[test] + fn test_read_local_list_projects_gzipped_column_after_skipping_header() { + let path = std::env::temp_dir().join(format!( + "seqspec-tabular-onlist-{}-{}.tsv.gz", + std::process::id(), + std::thread::current().name().unwrap_or("test") + )); + std::fs::write(&path, gzip_bytes("Name Barcode\nA01 AAAA\nA02 CCCC\n")).unwrap(); + + let result = read_local_list(&path, 1, 1).unwrap(); + + assert_eq!(result, vec!["AAAA".to_string(), "CCCC".to_string()]); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn test_read_remote_list_projects_gzipped_column_after_skipping_header() { + let (url, server) = serve_once(gzip_bytes("Name Barcode\nA01 AAAA\nA02 CCCC\n")); + + let result = read_remote_list(&url, &RemoteAccess::anonymous(), 1, 1).unwrap(); + server.join().unwrap(); + + assert_eq!(result, vec!["AAAA".to_string(), "CCCC".to_string()]); + } + + #[test] + fn test_read_local_list_errors_when_projection_column_is_missing() { + let root = + std::env::temp_dir().join(format!("seqspec-malformed-onlist-{}", std::process::id())); + std::fs::write(&root, "Name Barcode\nA01\n").unwrap(); + + let err = read_local_list(&root, 1, 1).unwrap_err(); + + assert!(err.contains("row 2 has 1 field")); + assert!(err.contains("column index 1")); + std::fs::remove_file(root).unwrap(); + } + #[test] fn test_local_onlist_locator_prefers_url_when_present() { let onlist = Onlist::new( @@ -691,6 +793,8 @@ mod tests { "nested/whitelist.txt".into(), "local".into(), String::new(), + 0, + 0, ); assert_eq!( @@ -709,6 +813,8 @@ mod tests { String::new(), "local".into(), String::new(), + 0, + 0, ); assert_eq!( diff --git a/tests/fixtures/tabular_onlist.txt b/tests/fixtures/tabular_onlist.txt new file mode 100644 index 00000000..d402df9d --- /dev/null +++ b/tests/fixtures/tabular_onlist.txt @@ -0,0 +1,4 @@ +Name B1 +A01 TCAGTTGTCGAAGG + +A02 CTGGACCTAATACC diff --git a/tests/test_region.py b/tests/test_region.py index e4ee8055..1c53f1f1 100644 --- a/tests/test_region.py +++ b/tests/test_region.py @@ -540,6 +540,8 @@ def test_onlist_creation(): assert onlist.url == "file://barcodes.txt" assert onlist.urltype == "local" assert onlist.md5 == "d41d8cd98f00b204e9800998ecf8427e" + assert onlist.sequence_column_index == 0 + assert onlist.skip_rows == 0 def test_onlist_input(): """Test OnlistInput class""" @@ -550,7 +552,9 @@ def test_onlist_input(): filesize=1000, url="file://barcodes.txt", urltype="local", - md5="d41d8cd98f00b204e9800998ecf8427e" + md5="d41d8cd98f00b204e9800998ecf8427e", + sequence_column_index=1, + skip_rows=1, ) onlist = onlist_input.to_onlist() @@ -561,6 +565,22 @@ def test_onlist_input(): assert onlist.url == "file://barcodes.txt" assert onlist.urltype == "local" assert onlist.md5 == "d41d8cd98f00b204e9800998ecf8427e" + assert onlist.sequence_column_index == 1 + assert onlist.skip_rows == 1 + + +def test_onlist_projection_fields_must_be_nonnegative(): + with pytest.raises(ValueError): + Onlist( + file_id="test_file", + filename="barcodes.txt", + filetype="txt", + filesize=1000, + url="barcodes.txt", + urltype="local", + md5="", + sequence_column_index=-1, + ) def test_region_coordinate_creation(): """Test RegionCoordinate creation""" diff --git a/tests/test_seqspec_onlist.py b/tests/test_seqspec_onlist.py index b9086035..af9c5b30 100644 --- a/tests/test_seqspec_onlist.py +++ b/tests/test_seqspec_onlist.py @@ -101,6 +101,30 @@ def fake_read_remote_list(onlist, base_path="", auth_profile=None): assert Path(downloaded[0]["url"]).read_text().splitlines() == ["AAA", "CCC"] +def test_download_projected_local_onlist_writes_normalized_copy(tmp_path): + source = tmp_path / "plate.tsv" + source.write_text("Name Barcode\nA01 AAAA\nA02 CCCC\n") + onlist = Onlist( + file_id="plate", + filename="plate.tsv", + filetype="tsv", + filesize=source.stat().st_size, + url="plate.tsv", + urltype="local", + md5="", + sequence_column_index=1, + skip_rows=1, + ) + + downloaded = download_onlists_to_path( + [onlist], tmp_path / "normalized.txt", tmp_path + ) + + normalized = Path(downloaded[0]["url"]) + assert normalized != source + assert normalized.read_text().splitlines() == ["AAAA", "CCCC"] + + def test_join_onlists_and_save_threads_auth_profile(tmp_path): calls = [] diff --git a/tests/test_utils.py b/tests/test_utils.py index 5647032b..1addc527 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -362,6 +362,105 @@ def test_read_local_list_prefers_url_when_present(tmp_path): assert read_local_list(onlist, str(tmp_path)) == ["AAAA", "CCCC"] +def test_read_local_list_projects_column_after_skipping_header(): + onlist = Onlist( + file_id="tabular_onlist", + filename="tabular_onlist.txt", + filetype="txt", + filesize=0, + url="tests/fixtures/tabular_onlist.txt", + urltype="local", + md5="", + sequence_column_index=1, + skip_rows=1, + ) + + assert read_local_list(onlist) == [ + "TCAGTTGTCGAAGG", + "CTGGACCTAATACC", + ] + + +def test_read_local_list_projects_gzipped_column_after_skipping_header(tmp_path): + path = tmp_path / "plate.tsv.gz" + path.write_bytes(gzip.compress(b"Name Barcode\nA01 AAAA\nA02 CCCC\n")) + onlist = Onlist( + file_id="local_plate", + filename="plate.tsv.gz", + filetype="tsv", + filesize=0, + url="plate.tsv.gz", + urltype="local", + md5="", + sequence_column_index=1, + skip_rows=1, + ) + + assert read_local_list(onlist, str(tmp_path)) == ["AAAA", "CCCC"] + + +def test_read_local_list_errors_when_projection_column_is_missing(tmp_path): + (tmp_path / "malformed.txt").write_text("Name Barcode\nA01\n") + onlist = Onlist( + file_id="malformed", + filename="malformed.txt", + filetype="txt", + filesize=0, + url="malformed.txt", + urltype="local", + md5="", + sequence_column_index=1, + skip_rows=1, + ) + + with pytest.raises(ValueError, match="row 2 has 1 field.*column index 1"): + read_local_list(onlist, str(tmp_path)) + + +def test_read_remote_list_projects_column_after_skipping_header(): + onlist = Onlist( + file_id="remote_plate", + filename="plate.tsv", + filetype="tsv", + filesize=0, + url="https://example.org/plate.tsv", + urltype="https", + md5="", + sequence_column_index=1, + skip_rows=1, + ) + response = MagicMock() + response.content = b"Name Barcode\nA01 AAAA\nA02 CCCC\n" + + with ( + patch("seqspec.utils.get_remote_auth_token", return_value=None), + patch("seqspec.utils.requests.get", return_value=response), + ): + assert read_remote_list(onlist) == ["AAAA", "CCCC"] + + +def test_read_remote_list_projects_gzipped_column_after_skipping_header(): + onlist = Onlist( + file_id="remote_plate", + filename="plate.tsv.gz", + filetype="tsv", + filesize=0, + url="https://example.org/plate.tsv.gz", + urltype="https", + md5="", + sequence_column_index=1, + skip_rows=1, + ) + response = MagicMock() + response.content = gzip.compress(b"Name Barcode\nA01 AAAA\nA02 CCCC\n") + + with ( + patch("seqspec.utils.get_remote_auth_token", return_value=None), + patch("seqspec.utils.requests.get", return_value=response), + ): + assert read_remote_list(onlist) == ["AAAA", "CCCC"] + + def test_local_resource_url_errors_when_url_is_empty(): with pytest.raises(ValueError, match="local file 'display.fastq.gz' has empty url"): local_resource_url("", "display.fastq.gz", "file")