Skip to content

Adapting to data model changes #383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
manulera merged 6 commits into from
Dec 9, 2025
Merged

Adapting to data model changes #383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a59c1f
update pydna and fix genbank endpoint
manulera Dec 5, 2025
4b50bbd
closes #382
manulera Dec 6, 2025
3c98642
improve test coverage
manulera Dec 9, 2025
6711478
enable setting of max length via env vars
manulera Dec 9, 2025
b961dd3
fix pombe example
manulera Dec 9, 2025
e1de76e
fix annotation
manulera Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/notebooks/external_sequences.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"input=[] assembly_accession='GCF_000146045.2' sequence_accession='NC_001136.10' locus_tag='YDR294C' gene_id=851888 start=1049459 end=1053228 strand=-1\n",
"input=[] repository_id='NC_001136.10' coordinates=SimpleLocation(ExactPosition(1049458), ExactPosition(1053228), strand=-1) assembly_accession='GCF_000146045.2' locus_tag='YDR294C' gene_id=851888\n",
"None\n",
"╙── YDR294C (Dseqrecord(-3770))\n",
" └─╼ GenomeCoordinatesSource\n"
Expand Down Expand Up @@ -264,7 +264,7 @@
{
"data": {
"text/plain": [
"RepositoryIdSource(input=[], repository_id='NC_001136.10', repository_name='genbank')"
"NCBISequenceSource(input=[], repository_id='NC_001136.10', coordinates=SimpleLocation(ExactPosition(1049458), ExactPosition(1053228), strand=-1))"
]
},
"execution_count": null,
Expand Down
513 changes: 251 additions & 262 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@ pydantic = "^2.7.1"
pandas = "^2.2.3"
openpyxl = "^3.1.5"
pyyaml = "^6.0.2"
opencloning-linkml = "0.4.8"
primer3-py = "2.2.0"
biopython = "^1.85"
packaging = "^25.0"
pairwise-alignments-to-msa = "^0.1.1"
pydna = {git = "https://github.com/pydna-group/pydna", rev = "e95e8603b082e50b7873eab0ae1f8abf7b4d25bc"}
pydna = {git = "https://github.com/pydna-group/pydna", rev = "6f25090ec653e38219021aa46fa4440f888e958c"}

Copilot AI Dec 6, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The opencloning-linkml dependency is missing from the [tool.poetry.dependencies] section but is present in poetry.lock at version 0.4.9. This dependency should be explicitly declared in pyproject.toml (e.g., opencloning-linkml = "0.4.9") to ensure reproducible builds and clear dependency management.

Suggested change
pydna = {git = "https://github.com/pydna-group/pydna", rev = "6f25090ec653e38219021aa46fa4440f888e958c"}
pydna = {git = "https://github.com/pydna-group/pydna", rev = "6f25090ec653e38219021aa46fa4440f888e958c"}
opencloning-linkml = "0.4.9"

Copilot uses AI. Check for mistakes.

[tool.poetry.group.dev.dependencies]
autopep8 = "^2.0.4"
Expand Down
7 changes: 7 additions & 0 deletions src/opencloning/app_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ def parse_bool(value: str) -> bool:

# External services settings =================================
NCBI_API_KEY = os.environ.get('NCBI_API_KEY')
NCBI_MAX_SEQUENCE_LENGTH = (
int(os.environ.get('NCBI_MAX_SEQUENCE_LENGTH'))
if os.environ.get('NCBI_MAX_SEQUENCE_LENGTH') is not None
else 500000
)
PLANNOTATE_URL = os.environ['PLANNOTATE_URL'] if 'PLANNOTATE_URL' in os.environ else None
PLANNOTATE_TIMEOUT = int(os.environ['PLANNOTATE_TIMEOUT']) if 'PLANNOTATE_TIMEOUT' in os.environ else 20
# Handle trailing slash:
Expand Down Expand Up @@ -58,6 +63,7 @@ class Settings(BaseModel):
BATCH_CLONING: bool
RECORD_STUBS: bool
NCBI_API_KEY: str | None
NCBI_MAX_SEQUENCE_LENGTH: int
ALLOWED_ORIGINS: list[str]
PLANNOTATE_URL: str | None
PLANNOTATE_TIMEOUT: int
Expand All @@ -73,6 +79,7 @@ class Settings(BaseModel):
BATCH_CLONING=BATCH_CLONING,
RECORD_STUBS=RECORD_STUBS,
NCBI_API_KEY=NCBI_API_KEY,
NCBI_MAX_SEQUENCE_LENGTH=NCBI_MAX_SEQUENCE_LENGTH,
ALLOWED_ORIGINS=ALLOWED_ORIGINS,
PLANNOTATE_URL=PLANNOTATE_URL,
PLANNOTATE_TIMEOUT=PLANNOTATE_TIMEOUT,
Expand Down
144 changes: 31 additions & 113 deletions src/opencloning/batch_cloning/pombe/pombe_clone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,34 @@
import os
from opencloning.endpoints.external_import import genome_coordinates, get_from_repository_id_addgene, read_from_file
from opencloning.endpoints.assembly import pcr, homologous_recombination
from opencloning.pydantic_models import BaseCloningStrategy

from opencloning_linkml.datamodel import (
GenomeCoordinatesSource,
AddgeneIdSource,
PCRSource,
Primer as PrimerModel,
HomologousRecombinationSource,
UploadedFileSource,
TextFileSequence,
)

from opencloning.ncbi_requests import get_annotations_from_query
from pydna.assembly2 import homologous_recombination_integration, pcr_assembly
from opencloning.dna_functions import request_from_addgene
from opencloning.ncbi_requests import get_annotations_from_query, get_genome_region_from_annotation
import asyncio
import json
from Bio import SeqIO
from pydna.primer import Primer
from pydna.opencloning_models import CloningStrategy
from fastapi.datastructures import UploadFile
from pydna.parsers import parse as pydna_parse


async def main(
gene: str, assembly_accession: str, output_dir: str, plasmid: str | dict = '19343', padding: int = 1000
gene: str,
assembly_accession: str,
output_dir: str,
plasmid_input: UploadFile | str = '19343',
padding: int = 1000,
):
print(f"\033[92mCloning {gene}\033[0m")
# Parse primers =================================================================================
primer_records = list(SeqIO.parse(os.path.join(output_dir, gene, 'primers.fa'), 'fasta'))
checking_primers = list(SeqIO.parse(os.path.join(output_dir, 'checking_primers.fa'), 'fasta'))
primer_records = primer_records[:3] + checking_primers[1:] + primer_records[3:] + checking_primers[:1]
primers = []
for primer in primer_records:
primers.append(PrimerModel(sequence=str(primer.seq), id=0, name=primer.id))
primers = [Primer(p) for p in SeqIO.parse(os.path.join(output_dir, gene, 'primers.fa'), 'fasta')]
common_primers = [Primer(p) for p in SeqIO.parse(os.path.join(output_dir, 'checking_primers.fa'), 'fasta')]

# Get plasmid sequence =================================================================================
if isinstance(plasmid_input, UploadFile):
file_content = (await plasmid_input.read()).decode()

plasmid = pydna_parse(file_content)[0]
else:
plasmid = await request_from_addgene(plasmid_input)

# Get genome region =====================================================================
annotations = await get_annotations_from_query(gene, assembly_accession)
Expand All @@ -41,104 +39,24 @@ async def main(
if len(annotations) != 1:
raise ValueError(f'No right annotation found for {gene}')

annotation = annotations[0]

gene_range = annotation['genomic_regions'][0]['gene_range']['range'][0]
sequence_accession = annotation['genomic_regions'][0]['gene_range']['accession_version']
locus_tag = annotation.get('locus_tag', None)
gene_id = annotation.get('gene_id', None)
start = int(gene_range['begin'])
end = int(gene_range['end'])
orientation = 1 if gene_range['orientation'] == 'plus' else -1

source = GenomeCoordinatesSource(
id=0,
start=start - padding,
end=end + padding,
strand=orientation,
assembly_accession=assembly_accession,
sequence_accession=sequence_accession,
locus_tag=locus_tag,
gene_id=gene_id,
output_name=gene,
)
locus = await genome_coordinates(source)

cloning_strategy = BaseCloningStrategy(
sequences=[],
sources=[],
primers=[],
description=f'Cloning strategy for deleting the gene {gene} using PCR and homologous recombination',
)
for primer in primers:
cloning_strategy.add_primer(primer)
locus_seq: TextFileSequence = TextFileSequence.model_validate(locus['sequences'][0])
locus_source: GenomeCoordinatesSource = GenomeCoordinatesSource.model_validate(locus['sources'][0])
cloning_strategy.add_source_and_sequence(locus_source, locus_seq)

# Get plasmid sequence =================s================================================================
if not isinstance(plasmid, str):
if plasmid.filename.endswith('.fa') or plasmid.filename.endswith('.fasta'):
resp = await read_from_file(plasmid, None, None, True, None, None, None)
else:
resp = await read_from_file(plasmid, None, None, None, None, None, None)
# Verify that plasmid is circular
if not pydna_parse(resp['sequences'][0].file_content)[0].circular:
raise ValueError('Plasmid is not circular')
plasmid_source: UploadedFileSource = UploadedFileSource.model_validate(resp['sources'][0])
else:
addgene_source = AddgeneIdSource(
id=0,
repository_id=plasmid,
repository_name='addgene',
)
resp = await get_from_repository_id_addgene(addgene_source)
plasmid_source: AddgeneIdSource = AddgeneIdSource.model_validate(resp['sources'][0])

plasmid_seq: TextFileSequence = TextFileSequence.model_validate(resp['sequences'][0])
cloning_strategy.add_source_and_sequence(plasmid_source, plasmid_seq)
locus = await get_genome_region_from_annotation(annotations[0], 1000, 1000)

# PCR ================================================================================================
pcr_source = PCRSource(id=0, output_name='amplified_marker')
resp = await pcr(pcr_source, [plasmid_seq], [primers[0], primers[1]], 15, 0)
pcr_products = pcr_assembly(plasmid, primers[0], primers[1], limit=14, mismatches=0)
pcr_products[0].name = 'amplified_marker'
alleles = homologous_recombination_integration(locus, [pcr_products[0]], 40)
pcr_check1 = pcr_assembly(alleles[0], primers[2], common_primers[1], limit=14, mismatches=0)[0]
pcr_check1.name = 'check_pcr_left'
pcr_check2 = pcr_assembly(alleles[0], primers[3], common_primers[0], limit=14, mismatches=0)[0]
pcr_check2.name = 'check_pcr_right'

pcr_product: TextFileSequence = TextFileSequence.model_validate(resp['sequences'][0])
pcr_source: PCRSource = PCRSource.model_validate(resp['sources'][0])
cloning_strategy.add_source_and_sequence(pcr_source, pcr_product)

# Homologous recombination ========================================================================
hrec_source = HomologousRecombinationSource(id=0, output_name='deletion_allele')
resp = await homologous_recombination(hrec_source, [locus_seq, pcr_product], 50)

hrec_product: TextFileSequence = TextFileSequence.model_validate(resp['sequences'][0])
hrec_source: HomologousRecombinationSource = HomologousRecombinationSource.model_validate(resp['sources'][0])
cloning_strategy.add_source_and_sequence(hrec_source, hrec_product)

# Checking pcr 1 ======================================================================================
check_pcr_source_left = PCRSource(id=0, output_name='check_pcr_left')
resp = await pcr(check_pcr_source_left, [hrec_product], [primers[2], primers[3]], 15, 0)

check_pcr_product_left: TextFileSequence = TextFileSequence.model_validate(resp['sequences'][0])
check_pcr_source_left: PCRSource = PCRSource.model_validate(resp['sources'][0])
cloning_strategy.add_source_and_sequence(check_pcr_source_left, check_pcr_product_left)

# Checking pcr 2 ======================================================================================
check_pcr_source_right = PCRSource(id=0, output_name='check_pcr_right')
resp = await pcr(check_pcr_source_right, [hrec_product], [primers[4], primers[5]], 15, 0)

check_pcr_product_right: TextFileSequence = TextFileSequence.model_validate(resp['sequences'][0])
check_pcr_source_right: PCRSource = PCRSource.model_validate(resp['sources'][0])
cloning_strategy.add_source_and_sequence(check_pcr_source_right, check_pcr_product_right)

cloning_strategy.description = (
f'Cloning strategy for deleting the gene {gene} using PCR and homologous recombination'
)
cs = CloningStrategy.from_dseqrecords([pcr_check1, pcr_check2])

if not os.path.exists(os.path.join(output_dir, gene)):
os.makedirs(os.path.join(output_dir, gene))

with open(os.path.join(output_dir, gene, 'cloning_strategy.json'), 'w') as f:
json.dump(cloning_strategy.model_dump(), f, indent=2)
f.write(cs.model_dump_json(indent=2))


if __name__ == '__main__':
Expand Down
7 changes: 4 additions & 3 deletions src/opencloning/batch_cloning/pombe/pombe_summary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,17 @@ def process_folder(working_dir: str):
# We do this to have action to .end and .start
pcr_sources = [PCRSource.model_validate(s.model_dump()) for s in pcr_sources]
locus_source = next(s for s in strategy.sources if s.type == 'GenomeCoordinatesSource')
locus_location = Location.fromstring(locus_source.coordinates)
hrec_source = next(s for s in strategy.sources if s.type == 'HomologousRecombinationSource')
# We do this to have action to .end and .start
hrec_source: HomologousRecombinationSource = HomologousRecombinationSource.model_validate(hrec_source.model_dump())

chromosome = chromosomes[locus_source.sequence_accession]
chromosome = chromosomes[locus_source.repository_id]
Comment thread
cursor[bot] marked this conversation as resolved.
insertion_start = (
locus_source.start + location_boundaries(Location.fromstring(hrec_source.input[0].right_location))[1]
locus_location.start + location_boundaries(Location.fromstring(hrec_source.input[0].right_location))[1]
)
insertion_end = (
locus_source.start + location_boundaries(Location.fromstring(hrec_source.input[-1].left_location))[0]
locus_location.start + location_boundaries(Location.fromstring(hrec_source.input[-1].left_location))[0]

@cursor cursor Bot Dec 9, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: Off-by-one error in insertion coordinate calculations

The insertion_start and insertion_end calculations use locus_location.start which is 0-indexed (BioPython convention), but the previous implementation used locus_source.start which was 1-indexed (NCBI coordinate convention). Since these coordinates represent genomic positions that will be written to a summary JSON for human consumption, using the 0-indexed locus_location.start will produce coordinates that are off by one from the expected NCBI-style 1-indexed genomic coordinates.

Fix in Cursor Fix in Web

)

# Write out the sequences in genbank format and extract some relevant info
Expand Down
11 changes: 2 additions & 9 deletions src/opencloning/batch_cloning/ziqiang_et_al2024/ziqiang_et_al2024.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@
"database_id": null,
"input": [],
"repository_id": "71287",
"repository_name": "addgene",
"sequence_file_url": "https://media.addgene.org/snapgene-media/v2.0.0/sequences/352460/6c8b0369-e549-4517-95da-8d07146ca49d/addgene-plasmid-71287-sequence-352460.gbk",
"addgene_sequence_type": "addgene-full"
},
Expand All @@ -92,7 +91,6 @@
"database_id": null,
"input": [],
"repository_id": "71287",
"repository_name": "addgene",
"sequence_file_url": "https://media.addgene.org/snapgene-media/v2.0.0/sequences/352460/6c8b0369-e549-4517-95da-8d07146ca49d/addgene-plasmid-71287-sequence-352460.gbk",
"addgene_sequence_type": "addgene-full"
},
Expand All @@ -103,7 +101,6 @@
"database_id": null,
"input": [],
"repository_id": "213912",
"repository_name": "addgene",
"sequence_file_url": "https://media.addgene.org/snapgene-media/v2.0.0/sequences/437264/8bd82b44-a1c3-4f81-8936-ebcc16d171e9/addgene-plasmid-213912-sequence-437264.gbk",
"addgene_sequence_type": "addgene-full"
},
Expand All @@ -114,7 +111,6 @@
"database_id": null,
"input": [],
"repository_id": "213913",
"repository_name": "addgene",
"sequence_file_url": "https://media.addgene.org/snapgene-media/v2.0.0/sequences/437263/af62d64e-1f22-4dce-aafa-7935d1665700/addgene-plasmid-213913-sequence-437263.gbk",
"addgene_sequence_type": "addgene-full"
},
Expand All @@ -125,7 +121,6 @@
"database_id": null,
"input": [],
"repository_id": "133748",
"repository_name": "addgene",
"sequence_file_url": "https://media.addgene.org/snapgene-media/v2.0.0/sequences/271264/c0ae5f43-46e6-4175-a0c1-9a00cbb92034/addgene-plasmid-133748-sequence-271264.gbk",
"addgene_sequence_type": "addgene-full"
},
Expand All @@ -135,8 +130,7 @@
"output_name": "pDONR_P2r-P3",
"database_id": null,
"input": [],
"repository_id": "gateway_cloning_vectors/pDONR_P2r-P3",
"repository_name": "snapgene"
"repository_id": "gateway_cloning_vectors/pDONR_P2r-P3"
},
{
"id": 7,
Expand Down Expand Up @@ -207,7 +201,6 @@
"database_id": null,
"input": [],
"repository_id": "63143",
"repository_name": "addgene",
"sequence_file_url": "https://media.addgene.org/snapgene-media/v2.0.0/sequences/223326/8c81fe9a-fb57-40bc-93d6-e17d83502061/addgene-plasmid-63143-sequence-223326.gbk",
"addgene_sequence_type": "addgene-full"
}
Expand Down Expand Up @@ -258,7 +251,7 @@
],
"description": "",
"files": null,
"schema_version": "0.4.3",
"schema_version": "0.4.9",
"backend_version": null,
"frontend_version": null
}
Loading
Loading