From 0bcfe31a7574625f29639c5323c5cdb192736203 Mon Sep 17 00:00:00 2001 From: David Roe Date: Sun, 19 Jul 2026 12:29:46 -0400 Subject: [PATCH 1/2] Add Szpiro ratio for elliptic curves over number fields (LMFDB#6292) Add a szpiro_ratio column (classical Szpiro ratio log(Norm(D_min))/log(Norm(N)), Hindry p.8) to the ECNF section: curve page display, search box, search-results column and sort order, all guarded by a module-level flag so the site keeps working until the column is added to ec_nfcurves. The ratio is left NULL for the 712 curves with everywhere good reduction, for which it is not defined. scripts/ecnf/generate_szpiro_ratio.py computes the column for all 767518 curves from stored data (conductor_norm, normdisc, non_min_p, local_data) in about 3 minutes, cross-checking the two available formulas against each other, and has a --verify mode that recomputes random rows from scratch with Sage. Verified: full generation run with --verify 25 (all match to 1e-12); base changes of 11.a1/11.a2/37.a1 agree with the classical Q-side ratios including ramified-prime scaling; lmfdb/ecnf/test_ecnf.py passes (13 tests) with the column absent; pyflakes clean. Co-Authored-By: Claude Fable 5 --- lmfdb/ecnf/WebEllipticCurve.py | 8 ++ lmfdb/ecnf/main.py | 18 ++- lmfdb/ecnf/templates/ecnf-curve.html | 9 ++ lmfdb/ecnf/test_ecnf.py | 20 +++ scripts/ecnf/generate_szpiro_ratio.py | 191 ++++++++++++++++++++++++++ 5 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 scripts/ecnf/generate_szpiro_ratio.py diff --git a/lmfdb/ecnf/WebEllipticCurve.py b/lmfdb/ecnf/WebEllipticCurve.py index 27c7c1125c..785c8aa6e0 100644 --- a/lmfdb/ecnf/WebEllipticCurve.py +++ b/lmfdb/ecnf/WebEllipticCurve.py @@ -420,6 +420,14 @@ def make_E(self): self.fact_mindisc = latex_factorization(badprimes, mindisc_ords) self.fact_mindisc_norm = latex_factorization(badnorms, mindisc_ords, sign=signDnorm) + # Szpiro ratio log(Norm(mindisc))/log(Norm(cond)). The + # attribute is set from the database row when the + # szpiro_ratio column exists; it is None for curves with + # everywhere good reduction, for which the ratio is not + # defined. The default here keeps this class working until + # the column has been added to ec_nfcurves. + self.szpiro_ratio = getattr(self, "szpiro_ratio", None) + j = self.field.parse_NFelt(self.jinv) self.j = web_latex(j) self.fact_j = None diff --git a/lmfdb/ecnf/main.py b/lmfdb/ecnf/main.py index aed122ccb2..d8c618c3ee 100644 --- a/lmfdb/ecnf/main.py +++ b/lmfdb/ecnf/main.py @@ -375,6 +375,10 @@ def parse_cm_list(inp, query, qfield): Ra = PolynomialRing(QQ,'a') +# The szpiro_ratio column is computed and uploaded by scripts/ecnf/generate_szpiro_ratio.py. +# All uses are guarded by this flag so that the site keeps working until the column is added. +HAVE_SZPIRO_RATIO = "szpiro_ratio" in db.ec_nfcurves.search_cols + ecnf_columns = SearchColumns([ MultiProcessedCol("label", "ec.curve_label", "Label", ["short_label", "field_label", "conductor_label", "iso_label", "number"], lambda label, field, conductor, iso, number: '%s' % ( @@ -395,6 +399,7 @@ def parse_cm_list(inp, query, qfield): ProcessedCol("conductor_norm", "ec.conductor", "Conductor norm", lambda v: web_latex_factored_integer(ZZ(v)), align="center"), ProcessedCol("normdisc", "ec.discriminant", "Discriminant norm", lambda v: web_latex_factored_integer(ZZ(v)), align="center", default=False), FloatCol("root_analytic_conductor", "lfunction.root_analytic_conductor", "Root analytic conductor", prec=5, default=False), + *([FloatCol("szpiro_ratio", "ec.szpiro_ratio", "Szpiro ratio", prec=5, default=False)] if HAVE_SZPIRO_RATIO else []), ProcessedCol("bad_primes", "ec.bad_reduction", "Bad primes", lambda primes: ", ".join(''.join(str(p.replace('w', 'a')).split('*')) for p in primes) if primes else r"\textsf{none}", default=lambda info: info.get("bad_primes"), mathmode=True, align="center"), @@ -512,6 +517,8 @@ def elliptic_curve_search(info, query): parse_ints(info,query,'class_deg','class_deg') parse_ints(info,query,'sha','analytic order of Ш') parse_floats(info,query,'reg','regulator') + if HAVE_SZPIRO_RATIO: + parse_floats(info,query,'szpiro_ratio','Szpiro ratio') parse_nf_jinv(info,query,'jinv','j-invariant',field_label=query.get('field_label')) if info.get('one') == "yes": @@ -795,7 +802,8 @@ class ECNFSearchArray(SearchArray): ("reg", "regulator", ["reg", 'degree', 'signature', 'abs_disc', 'field_label', 'conductor_norm', 'conductor_label', 'iso_nlabel', 'number']), ("sha", "analytic Ш", ["sha", 'degree', 'signature', 'abs_disc', 'field_label', 'conductor_norm', 'conductor_label', 'iso_nlabel', 'number']), ("class_size", "isogeny class size", ["class_size", 'degree', 'signature', 'abs_disc', 'field_label', 'conductor_norm', 'conductor_label', 'iso_nlabel', 'number']), - ("class_deg", "isogeny class degree", ["class_deg", 'degree', 'signature', 'abs_disc', 'field_label', 'conductor_norm', 'conductor_label', 'iso_nlabel', 'number'])] + ("class_deg", "isogeny class degree", ["class_deg", 'degree', 'signature', 'abs_disc', 'field_label', 'conductor_norm', 'conductor_label', 'iso_nlabel', 'number'])] \ + + ([("szpiro_ratio", "Szpiro ratio", ["szpiro_ratio", 'degree', 'signature', 'abs_disc', 'field_label', 'conductor_norm', 'conductor_label', 'iso_nlabel', 'number'])] if HAVE_SZPIRO_RATIO else []) jump_example = "2.2.5.1-31.1-a1" jump_egspan = "e.g. 2.2.5.1-31.1-a1 or 2.2.5.1-31.1-a" jump_knowl = "ec.search_input" @@ -947,6 +955,12 @@ def __init__(self): label="Base change of", knowl="ec.base_change", example="11a.1") + szpiro_ratio = TextBox( + name="szpiro_ratio", + label="Szpiro ratio", + knowl="ec.szpiro_ratio", + example="4.5", + example_span="4.5 or 4-5") count = CountBox() self.browse_array = [ @@ -960,6 +974,7 @@ def __init__(self): [class_size, class_deg], [galois_image, nonmax_primes], [base_change_label, reduction], + *([[szpiro_ratio]] if HAVE_SZPIRO_RATIO else []), [jinv], [count] ] @@ -969,5 +984,6 @@ def __init__(self): [deg_sig, bad_primes, Qcurves, torsion_structure, include_cm], [sha, isodeg, class_size, reduction, galois_image], [base_change_label, regulator, one, class_deg, nonmax_primes], + *([[szpiro_ratio]] if HAVE_SZPIRO_RATIO else []), [jinv], ] diff --git a/lmfdb/ecnf/templates/ecnf-curve.html b/lmfdb/ecnf/templates/ecnf-curve.html index 03297f5e55..f32ec653dd 100644 --- a/lmfdb/ecnf/templates/ecnf-curve.html +++ b/lmfdb/ecnf/templates/ecnf-curve.html @@ -249,6 +249,15 @@

{{ KNOWL('ec.invariants', title='Invariants')}}

+{% if ec.szpiro_ratio %} + + {{ KNOWL('ec.szpiro_ratio', title='Szpiro ratio') }}: + $\sigma$ + ≈ + ${{ ec.szpiro_ratio }}$ + +{% endif %} + diff --git a/lmfdb/ecnf/test_ecnf.py b/lmfdb/ecnf/test_ecnf.py index 13c9ac60cd..94e3d6667c 100644 --- a/lmfdb/ecnf/test_ecnf.py +++ b/lmfdb/ecnf/test_ecnf.py @@ -117,6 +117,26 @@ def test_isodeg(self): L = self.tc.get('/EllipticCurve/?start=0&torsion=1&isodeg=2') assert 'No matches' in L.get_data(as_text=True) + def test_szpiro_ratio(self): + r""" + Test that Szpiro ratio display and search work whether or not + the szpiro_ratio column has been added to ec_nfcurves + """ + from lmfdb.ecnf.main import HAVE_SZPIRO_RATIO + # 2.2.5.1-31.1-a1 has Szpiro ratio exactly 1 (Norm(D_min) = Norm(N) = 31) + L = self.tc.get('/EllipticCurve/2.2.5.1/31.1/a/1') + t = L.get_data(as_text=True) + assert 'Conductor norm' in t + if HAVE_SZPIRO_RATIO: + assert 'Szpiro ratio' in t + else: + assert 'Szpiro ratio' not in t + # When the column is missing, a szpiro_ratio constraint is + # ignored rather than producing an error; when it is present, + # this is a genuine search that matches 2.2.5.1-31.1-a1. + L = self.tc.get('/EllipticCurve/?field=2.2.5.1&szpiro_ratio=0.5-1.5') + assert '31.1-a1' in L.get_data(as_text=True) + def test_cm_disc_search(self): r""" Test that searching for CM field discriminant works diff --git a/scripts/ecnf/generate_szpiro_ratio.py b/scripts/ecnf/generate_szpiro_ratio.py new file mode 100644 index 0000000000..3151d4f145 --- /dev/null +++ b/scripts/ecnf/generate_szpiro_ratio.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +r"""Generate the szpiro_ratio column for the ec_nfcurves table (issue #6292). + +The Szpiro ratio of an elliptic curve `E` over a number field `K` is + + sigma = log(Norm(D_min)) / log(Norm(N)), + +where `D_min` is the minimal discriminant ideal and `N` the conductor of +`E` (Hindry, "Why is it difficult to compute the Mordell-Weil group?", +top of p.8). Both norms are determined by columns already stored in +ec_nfcurves: + +- ``conductor_norm`` is Norm(N); +- ``normdisc`` is the norm of the discriminant of the *stored model*, + which may be negative (it is the norm of the discriminant as a field + element) and may fail to be minimal at the (at most one) prime listed + in ``non_min_p``; at such a prime the stored valuation of the model + discriminant exceeds that of the minimal discriminant by 12; +- ``local_data[i]['ord_disc']`` is the valuation of the *minimal* + discriminant at the i-th bad (or non-minimal) prime, whose norm is + ``local_data[i]['normp']``. + +So Norm(D_min) = prod(normp^ord_disc) = |normdisc| / prod(normp^12 over +non_min_p). Curves with everywhere good reduction (conductor_norm = 1) +have D_min = (1) as well, so sigma is not defined (0/0) and we store NULL. + +For efficiency the script streams a light projection for curves whose +stored model is globally minimal (the vast majority) and only fetches +``local_data`` for the non-minimal rows; for the latter both formulas +above are computed and checked against each other. A random sample of +minimal rows is cross-checked against local_data as well. + +Run from the top-level lmfdb directory (requires a working config.ini; +read access is enough): + + sage -python scripts/ecnf/generate_szpiro_ratio.py ec_nfcurves_szpiro.txt + +Optional flags: ``--limit N`` (sample run), ``--sample-check N`` (number +of minimal-model rows to cross-check against local_data, default 1000), +``--verify N`` (recompute N random rows from scratch with Sage from the +a-invariants; slow but fully independent). + +The output file is in the format expected by ``update_from_file``. To +upload (needs an account with write access; not possible on devmirror): + + sage -python + >>> from lmfdb import db + >>> db.ec_nfcurves.add_column("szpiro_ratio", "double precision", + ... description="Szpiro ratio log(Norm(D_min))/log(Norm(N)), NULL for curves with everywhere good reduction") + >>> db.ec_nfcurves.update_from_file("ec_nfcurves_szpiro.txt") +""" +import argparse +import os +import sys +from math import log + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) +from lmfdb import db + + +def min_disc_norm(rec): + """Norm of the minimal discriminant ideal, computed from local_data.""" + return prod_pow([(ld["normp"], ld["ord_disc"]) for ld in rec["local_data"]]) + + +def min_disc_norm_from_normdisc(rec): + """Norm of the minimal discriminant ideal, computed from normdisc. + + The stored model is minimal outside the primes in non_min_p, at each + of which the valuation of its discriminant is 12 more than that of + the minimal discriminant. + """ + D = abs(int(rec["normdisc"])) + for p in rec["non_min_p"]: + normp = next(ld["normp"] for ld in rec["local_data"] if ld["p"] == p) + D, r = divmod(D, normp**12) + assert r == 0, "normdisc not divisible by normp^12 for %s" % rec["label"] + return D + + +def szpiro_ratio(Dnorm, Nnorm): + """log(Dnorm)/log(Nnorm), or None if not defined (trivial conductor).""" + if Nnorm == 1: + # everywhere good reduction: D_min = (1) too, sigma undefined + assert Dnorm == 1, "conductor norm 1 but Dnorm = %s" % Dnorm + return None + return log(Dnorm) / log(Nnorm) + + +def prod_pow(pairs): + D = 1 + for p, e in pairs: + D *= p**e + return D + + +def generate(outfile, limit=None, sample_check=1000): + from random import randrange + + total = db.ec_nfcurves.count() + nulls = written = 0 + check_freq = max(1, total // sample_check) if sample_check else 0 + with open(outfile, "w") as F: + F.write("label|szpiro_ratio\ntext|double precision\n\n") + + # Curves whose stored model is globally minimal: here + # Norm(D_min) = |normdisc| and we do not need local_data. + # A random sample is cross-checked against local_data. + checked = 0 + for i, rec in enumerate(db.ec_nfcurves.search( + {"non_min_p": []}, + ["label", "conductor_norm", "normdisc"], + sort=[], limit=limit)): + Dnorm = abs(int(rec["normdisc"])) + sigma = szpiro_ratio(Dnorm, rec["conductor_norm"]) + if check_freq and i % check_freq == randrange(check_freq): + full = db.ec_nfcurves.lookup(rec["label"], ["local_data", "non_min_p"]) + assert min_disc_norm(full) == Dnorm, \ + "normdisc inconsistent with local_data for %s" % rec["label"] + checked += 1 + F.write("%s|%s\n" % (rec["label"], r"\N" if sigma is None else repr(sigma))) + written += 1 + nulls += sigma is None + if written % 100000 == 0: + print("%s/%s done" % (written, total)) + + # Curves stored with a non-minimal model: compute from + # local_data and cross-check against normdisc. + for rec in db.ec_nfcurves.search( + {"non_min_p": {"$ne": []}}, + ["label", "conductor_norm", "normdisc", "non_min_p", "local_data"], + sort=[], limit=limit): + Dnorm = min_disc_norm(rec) + assert Dnorm == min_disc_norm_from_normdisc(rec), \ + "normdisc inconsistent with local_data for %s" % rec["label"] + sigma = szpiro_ratio(Dnorm, rec["conductor_norm"]) + F.write("%s|%s\n" % (rec["label"], r"\N" if sigma is None else repr(sigma))) + written += 1 + nulls += sigma is None + print("Wrote %s rows (%s NULL, i.e. everywhere good reduction) to %s" + % (written, nulls, outfile)) + print("Cross-checked %s minimal-model rows against local_data" % checked) + if limit is None and written != total: + print("WARNING: table has %s rows but %s were written" % (total, written)) + + +def verify(datafile, nchecks=20): + """Recompute szpiro_ratio for random rows of the output file from scratch. + + This is an independent check: the curve is rebuilt in Sage from its + a-invariants and the norms of its conductor and minimal discriminant + ideal are recomputed, without using conductor_norm, normdisc or + local_data. + """ + from random import sample + from sage.all import EllipticCurve + from lmfdb.ecnf.WebEllipticCurve import FIELD, parse_ainvs + + with open(datafile) as F: + lines = F.read().splitlines()[3:] + for line in sample(lines, nchecks): + label, _, sigma = line.partition("|") + rec = db.ec_nfcurves.lookup(label, ["field_label", "ainvs", "base_change"]) + K = FIELD(rec["field_label"]).K() + E = EllipticCurve(parse_ainvs(K, rec["ainvs"])) + Nnorm = E.conductor().norm() + Dnorm = E.minimal_discriminant_ideal().norm() + if Nnorm == 1: + assert sigma == r"\N", "%s: expected NULL, file has %s" % (label, sigma) + print("%s: everywhere good reduction, NULL ok" % label) + else: + recomputed = log(Dnorm) / log(Nnorm) + assert abs(recomputed - float(sigma)) < 1e-12, \ + "%s: file has %s but Sage gives %s" % (label, sigma, recomputed) + print("%s: %s ok (Norm(D_min)=%s, Norm(N)=%s, base change of %s)" + % (label, sigma, Dnorm, Nnorm, rec["base_change"] or "nothing")) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("outfile", help="output file (update_from_file format)") + parser.add_argument("--limit", type=int, default=None, + help="only process this many rows from each query (sample run)") + parser.add_argument("--sample-check", type=int, default=1000, + help="number of minimal-model rows to cross-check against local_data") + parser.add_argument("--verify", type=int, default=0, metavar="N", + help="after generating, recompute N random rows from scratch with Sage") + args = parser.parse_args() + generate(args.outfile, limit=args.limit, sample_check=args.sample_check) + if args.verify: + verify(args.outfile, args.verify) From 1ab4984ed178c554b0a763fca2c7cfdb1eafb5a1 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 5 Aug 2026 02:02:22 -0400 Subject: [PATCH 2/2] Harden the Szpiro ratio deployment path and tests (LMFDB#6292) Review follow-up on the Szpiro ratio for elliptic curves over number fields. The mathematical definition and the frontend are unchanged. - main.py: document HAVE_SZPIRO_RATIO as a startup-time compatibility guard. It, the result columns, the search array and the sort choices are all built at import, so adding the column does not activate the feature in a running worker; the workers have to be restarted. - generate_szpiro_ratio.py: spell out the migration order (generate, knowl, add column, upload, index, restart, smoke test), including the ec_nfcurves_szpiro_ratio_sort btree matching the sort tuple, created after update_from_file so the swap does not build it twice. - generate_szpiro_ratio.py: fail closed. Every data-integrity assert is now an explicit ValueError/RuntimeError naming the curve; the two partition counts must add up to the table count and the rows written must match the expected number; rows stream to a temporary file that replaces the output path only after all cross-checks, the count checks and any --verify have passed, so no partial file is left where it could be uploaded. --sample-check N now makes exactly N checks (positions drawn with random.sample over the rows actually processed, not a Bernoulli frequency from the full table count), the numeric flags must be nonnegative, --verify above the generated row count errors clearly, and --seed makes the sampling reproducible. - test_ecnf.py: the positive search test no longer passes when the constraint is ignored. With the column present the test asserts the displayed value 1.0 on the curve page, that a 0.5-1.5 range contains the full curve url while a disjoint 1.1-1.5 range does not, that sort_order=szpiro_ratio un-hides the column, and that an everywhere-good-reduction curve shows no ratio row; without it, the compatibility branch only claims that the pages still load. New schema-independent tests cover the generator helpers. Verified: pytest -k szpiro (2 passed) and pyflakes/pylint/ruff clean; the curve-page markup and the everywhere-good-reduction omission checked against a simulated column-present row, since devmirror has no szpiro_ratio column yet; a bounded run (--limit 1000 --sample-check 100 --verify 20) wrote exactly 2000 rows and made exactly 100 cross-checks; an injected mismatch exited nonzero leaving the previous output intact and no temporary file behind. Co-Authored-By: Claude Opus 5 --- lmfdb/ecnf/main.py | 6 +- lmfdb/ecnf/test_ecnf.py | 120 +++++++++- scripts/ecnf/generate_szpiro_ratio.py | 326 ++++++++++++++++++++------ 3 files changed, 361 insertions(+), 91 deletions(-) diff --git a/lmfdb/ecnf/main.py b/lmfdb/ecnf/main.py index 5d747141fb..c06e4c4ffe 100644 --- a/lmfdb/ecnf/main.py +++ b/lmfdb/ecnf/main.py @@ -376,7 +376,11 @@ def parse_cm_list(inp, query, qfield): Ra = PolynomialRing(QQ,'a') # The szpiro_ratio column is computed and uploaded by scripts/ecnf/generate_szpiro_ratio.py. -# All uses are guarded by this flag so that the site keeps working until the column is added. +# All uses are guarded by this flag, which is a startup-time compatibility guard: it is +# evaluated once when this module is imported, so old-schema deployments keep working, but +# the web workers must be restarted after the column has been added and populated so that +# ecnf_columns, the search array and the sort choices below (also built at import time) are +# rebuilt. Refreshing the schema of a running worker is not enough. HAVE_SZPIRO_RATIO = "szpiro_ratio" in db.ec_nfcurves.search_cols ecnf_columns = SearchColumns([ diff --git a/lmfdb/ecnf/test_ecnf.py b/lmfdb/ecnf/test_ecnf.py index 554e9c17d3..564fef09c4 100644 --- a/lmfdb/ecnf/test_ecnf.py +++ b/lmfdb/ecnf/test_ecnf.py @@ -1,5 +1,27 @@ +import os +import re + from lmfdb.tests import LmfdbTest + +def szpiro_generator(): + """The scripts/ecnf/generate_szpiro_ratio.py module, loaded by path. + + ``scripts`` is not part of the lmfdb package, so it cannot simply be + imported; ``None`` is returned when it is absent (e.g. when only the + package has been installed). + """ + import importlib.util + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir, + "scripts", "ecnf", "generate_szpiro_ratio.py") + if not os.path.exists(path): + return None + spec = importlib.util.spec_from_file_location("generate_szpiro_ratio", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + class EllCurveTest(LmfdbTest): # All tests should pass @@ -123,23 +145,95 @@ def test_isodeg(self): def test_szpiro_ratio(self): r""" - Test that Szpiro ratio display and search work whether or not - the szpiro_ratio column has been added to ec_nfcurves + Test that the Szpiro ratio is displayed, searchable and sortable + once ec_nfcurves has the szpiro_ratio column, and that the pages + still work (without offering it) while the column is missing """ from lmfdb.ecnf.main import HAVE_SZPIRO_RATIO - # 2.2.5.1-31.1-a1 has Szpiro ratio exactly 1 (Norm(D_min) = Norm(N) = 31) - L = self.tc.get('/EllipticCurve/2.2.5.1/31.1/a/1') - t = L.get_data(as_text=True) + # 2.2.5.1-31.1-a1 has Szpiro ratio exactly 1 (Norm(D_min) = Norm(N) = 31); + # 3.3.1369.1-1.1-a1 has everywhere good reduction, so no ratio at all. + curve_url = '/EllipticCurve/2.2.5.1/31.1/a/1' + egr_url = '/EllipticCurve/3.3.1369.1/1.1/a/1' + t = self.tc.get(curve_url).get_data(as_text=True) assert 'Conductor norm' in t - if HAVE_SZPIRO_RATIO: - assert 'Szpiro ratio' in t - else: + + if not HAVE_SZPIRO_RATIO: + # Compatibility branch: the pages load and offer no ratio anywhere. + # A szpiro_ratio constraint is deliberately ignored rather than + # raising, so the search below only shows that the page still works; + # it says nothing about filtering, which needs the column. assert 'Szpiro ratio' not in t - # When the column is missing, a szpiro_ratio constraint is - # ignored rather than producing an error; when it is present, - # this is a genuine search that matches 2.2.5.1-31.1-a1. - L = self.tc.get('/EllipticCurve/?field=2.2.5.1&szpiro_ratio=0.5-1.5') - assert '31.1-a1' in L.get_data(as_text=True) + assert 'Szpiro ratio' not in self.tc.get('/EllipticCurve/').get_data(as_text=True) + assert self.tc.get('/EllipticCurve/?field=2.2.5.1&szpiro_ratio=0.5-1.5').status_code == 200 + return + + # Displayed on the curve page, with the right value. Match the label + # text rather than the knowl markup around it: KNOWL() renders a plain + # label until ec.szpiro_ratio has been created, and an anchor after. + row = re.search(r'Szpiro ratio.*?', t, re.DOTALL) + assert row is not None, 'no Szpiro ratio row on %s' % curve_url + assert re.search(r'\$\s*1\.0\s*\$', row.group(0)), row.group(0) + # Omitted for a curve with everywhere good reduction, where it is undefined. + assert 'Szpiro ratio' not in self.tc.get(egr_url).get_data(as_text=True) + + # A range containing 1.0 finds the curve and a disjoint range does not: + # the pair is what shows that the constraint reaches the query at all. + t = self.tc.get('/EllipticCurve/?field=2.2.5.1&szpiro_ratio=0.5-1.5').get_data(as_text=True) + assert curve_url in t + t = self.tc.get('/EllipticCurve/?field=2.2.5.1&szpiro_ratio=1.1-1.5').get_data(as_text=True) + assert curve_url not in t + + # Sorting by the ratio works, and shows the column even though it is + # off by default: without the sort its results-table header carries + # display:none (it is always in the html, for the column selector). + L = self.tc.get('/EllipticCurve/?field=2.2.5.1&sort_order=szpiro_ratio') + assert L.status_code == 200 + th = re.search(r'>> from lmfdb import db - >>> db.ec_nfcurves.add_column("szpiro_ratio", "double precision", - ... description="Szpiro ratio log(Norm(D_min))/log(Norm(N)), NULL for curves with everywhere good reduction") - >>> db.ec_nfcurves.update_from_file("ec_nfcurves_szpiro.txt") +All website uses of the column are guarded by + + HAVE_SZPIRO_RATIO = "szpiro_ratio" in db.ec_nfcurves.search_cols + +in lmfdb/ecnf/main.py. That flag, the result columns, the search array +and the sort choices are all built once when the module is imported, so +adding the column does **not** activate the feature in a running web +worker, and neither does ``db.refresh_tables()``: the workers have to be +restarted. Carry out the migration in this order. + +1. Generate and verify the data file (this script):: + + sage -python scripts/ecnf/generate_szpiro_ratio.py ec_nfcurves_szpiro.txt --verify 25 + +2. Create the ``ec.szpiro_ratio`` knowl (web UI), so the label on the + curve page is live as soon as the feature appears. + +3. Add the column (needs an account with write access; not possible on + devmirror):: + + sage -python + >>> from lmfdb import db + >>> db.ec_nfcurves.add_column("szpiro_ratio", "double precision", + ... description="Szpiro ratio log(Norm(D_min))/log(Norm(N)), NULL for curves with everywhere good reduction") + +4. Upload the data and check that every row was set:: + + >>> db.ec_nfcurves.update_from_file("ec_nfcurves_szpiro.txt") + >>> db.ec_nfcurves.count({"szpiro_ratio": {"$exists": True}}) + >>> db.ec_nfcurves.count() - db.ec_nfcurves.count({"conductor_norm": 1}) # same number + >>> db.ec_nfcurves.count({"conductor_norm": 1, "szpiro_ratio": {"$exists": True}}) # 0 + + (``update_from_file`` merges into a new table and swaps it in, so it + can be undone with ``reload_revert``.) + +5. Create the index backing the new range search and sort. Do this + *after* the upload: the swap in step 4 rebuilds every index of the + table from scratch, so an index created first would be built twice:: + + >>> szpiro_sort = ["szpiro_ratio", "degree", "signature", "abs_disc", + ... "field_label", "conductor_norm", "conductor_label", + ... "iso_nlabel", "number"] + >>> if "ec_nfcurves_szpiro_ratio_sort" not in db.ec_nfcurves.list_indexes(): + ... db.ec_nfcurves.create_index(szpiro_sort, name="ec_nfcurves_szpiro_ratio_sort") + + The column list is exactly the sort tuple registered for "Szpiro + ratio" in ``ECNFSearchArray.sorts``. Use ``create_index`` rather + than raw SQL: it records the definition in ``meta_indexes``, which is + what lets later reloads rebuild the index. Then check that it is + there and that the planner uses it:: + + >>> db.ec_nfcurves.list_indexes(verbose=True) + >>> db.ec_nfcurves.analyze({}, projection=["label", "szpiro_ratio"], + ... limit=50, sort=szpiro_sort, explain_only=True) + >>> db.ec_nfcurves.analyze({"szpiro_ratio": {"$gte": 6, "$lte": 7}}, + ... projection=["label", "szpiro_ratio"], + ... limit=50, sort=szpiro_sort, explain_only=True) + + Both plans should use ec_nfcurves_szpiro_ratio_sort instead of a + sequential scan followed by a sort. Do not restart the website until + the index has finished building. + +6. Restart/reload **every** web worker, so that HAVE_SZPIRO_RATIO, the + result columns, the search boxes and the sort choices are rebuilt. + +7. Smoke test display, search and sort: /EllipticCurve/2.2.5.1/31.1/a/1 + shows a Szpiro ratio of 1.0; /EllipticCurve/?field=2.2.5.1&szpiro_ratio=0.5-1.5 + contains it and ...&szpiro_ratio=1.1-1.5 does not; and + /EllipticCurve/?sort_order=szpiro_ratio returns promptly. """ import argparse import os +import random import sys +import tempfile from math import log sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -70,19 +147,46 @@ def min_disc_norm_from_normdisc(rec): of which the valuation of its discriminant is 12 more than that of the minimal discriminant. """ - D = abs(int(rec["normdisc"])) + normdisc = abs(int(rec["normdisc"])) + D = normdisc for p in rec["non_min_p"]: - normp = next(ld["normp"] for ld in rec["local_data"] if ld["p"] == p) + normp = None + for ld in rec["local_data"]: + if ld["p"] == p: + normp = ld["normp"] + break + if normp is None: + raise ValueError( + "%s: no local_data entry for the non-minimal prime %s " + "(local_data covers %s)" + % (rec["label"], p, ", ".join(ld["p"] for ld in rec["local_data"]) or "no primes")) D, r = divmod(D, normp**12) - assert r == 0, "normdisc not divisible by normp^12 for %s" % rec["label"] + if r: + raise ValueError( + "%s: |normdisc| = %s is not divisible by normp^12 = %s^12 at the " + "non-minimal prime %s" % (rec["label"], normdisc, normp, p)) return D -def szpiro_ratio(Dnorm, Nnorm): - """log(Dnorm)/log(Nnorm), or None if not defined (trivial conductor).""" +def szpiro_ratio(Dnorm, Nnorm, label=None): + """log(Dnorm)/log(Nnorm), or None if not defined (trivial conductor). + + ``Dnorm`` is the norm of the minimal discriminant ideal and ``Nnorm`` + the norm of the conductor; both must be positive integers. ``label`` + is only used to make the error messages identify the offending curve. + """ + where = "" if label is None else "%s: " % label + if Dnorm <= 0: + raise ValueError("%sminimal discriminant norm is %s, expected a positive integer" + % (where, Dnorm)) + if Nnorm <= 0: + raise ValueError("%sconductor norm is %s, expected a positive integer" + % (where, Nnorm)) if Nnorm == 1: # everywhere good reduction: D_min = (1) too, sigma undefined - assert Dnorm == 1, "conductor norm 1 but Dnorm = %s" % Dnorm + if Dnorm != 1: + raise ValueError("%sconductor norm is 1 but the minimal discriminant has norm %s" + % (where, Dnorm)) return None return log(Dnorm) / log(Nnorm) @@ -94,57 +198,108 @@ def prod_pow(pairs): return D -def generate(outfile, limit=None, sample_check=1000): - from random import randrange +def generate(outfile, limit=None, sample_check=1000, verify_count=0, seed=None): + """Write the szpiro_ratio data file for ec_nfcurves, or fail without writing it. + + Rows go to a temporary file in the same directory as ``outfile``, + which is replaced only once everything below has succeeded; on any + failure the temporary file is removed and ``outfile`` is untouched. + """ + rand = random.Random(seed) total = db.ec_nfcurves.count() - nulls = written = 0 - check_freq = max(1, total // sample_check) if sample_check else 0 - with open(outfile, "w") as F: - F.write("label|szpiro_ratio\ntext|double precision\n\n") - - # Curves whose stored model is globally minimal: here - # Norm(D_min) = |normdisc| and we do not need local_data. - # A random sample is cross-checked against local_data. - checked = 0 - for i, rec in enumerate(db.ec_nfcurves.search( - {"non_min_p": []}, - ["label", "conductor_norm", "normdisc"], - sort=[], limit=limit)): - Dnorm = abs(int(rec["normdisc"])) - sigma = szpiro_ratio(Dnorm, rec["conductor_norm"]) - if check_freq and i % check_freq == randrange(check_freq): - full = db.ec_nfcurves.lookup(rec["label"], ["local_data", "non_min_p"]) - assert min_disc_norm(full) == Dnorm, \ - "normdisc inconsistent with local_data for %s" % rec["label"] - checked += 1 - F.write("%s|%s\n" % (rec["label"], r"\N" if sigma is None else repr(sigma))) - written += 1 - nulls += sigma is None - if written % 100000 == 0: - print("%s/%s done" % (written, total)) - - # Curves stored with a non-minimal model: compute from - # local_data and cross-check against normdisc. - for rec in db.ec_nfcurves.search( - {"non_min_p": {"$ne": []}}, - ["label", "conductor_norm", "normdisc", "non_min_p", "local_data"], - sort=[], limit=limit): - Dnorm = min_disc_norm(rec) - assert Dnorm == min_disc_norm_from_normdisc(rec), \ - "normdisc inconsistent with local_data for %s" % rec["label"] - sigma = szpiro_ratio(Dnorm, rec["conductor_norm"]) - F.write("%s|%s\n" % (rec["label"], r"\N" if sigma is None else repr(sigma))) - written += 1 - nulls += sigma is None - print("Wrote %s rows (%s NULL, i.e. everywhere good reduction) to %s" - % (written, nulls, outfile)) - print("Cross-checked %s minimal-model rows against local_data" % checked) - if limit is None and written != total: - print("WARNING: table has %s rows but %s were written" % (total, written)) - - -def verify(datafile, nchecks=20): + minimal_count = db.ec_nfcurves.count({"non_min_p": []}) + nonminimal_count = db.ec_nfcurves.count({"non_min_p": {"$ne": []}}) + if minimal_count + nonminimal_count != total: + raise RuntimeError( + "ec_nfcurves has %s rows, but %s have non_min_p = [] and %s have " + "non_min_p != []; the %s remaining rows (most likely NULL non_min_p) " + "would be silently omitted" + % (total, minimal_count, nonminimal_count, + total - minimal_count - nonminimal_count)) + + processed_minimal = minimal_count if limit is None else min(limit, minimal_count) + processed_nonminimal = nonminimal_count if limit is None else min(limit, nonminimal_count) + expected = processed_minimal + processed_nonminimal + + # Cross-check exactly this many minimal rows, at positions drawn up front, + # rather than with a probability derived from the size of the whole table. + target_checks = min(sample_check, processed_minimal) + check_positions = set(rand.sample(range(processed_minimal), target_checks)) + + directory = os.path.dirname(os.path.abspath(outfile)) + fd, tmpfile = tempfile.mkstemp(dir=directory, prefix=os.path.basename(outfile) + ".") + try: + nulls = written = checked = 0 + with os.fdopen(fd, "w") as F: + F.write("label|szpiro_ratio\ntext|double precision\n\n") + + # Curves whose stored model is globally minimal: here + # Norm(D_min) = |normdisc| and we do not need local_data. + # The sampled positions are cross-checked against local_data. + for i, rec in enumerate(db.ec_nfcurves.search( + {"non_min_p": []}, + ["label", "conductor_norm", "normdisc"], + sort=[], limit=limit)): + Dnorm = abs(int(rec["normdisc"])) + sigma = szpiro_ratio(Dnorm, rec["conductor_norm"], rec["label"]) + if i in check_positions: + full = db.ec_nfcurves.lookup(rec["label"], ["label", "local_data", "non_min_p"]) + from_local_data = min_disc_norm(full) + if from_local_data != Dnorm: + raise ValueError( + "%s: |normdisc| = %s but local_data gives Norm(D_min) = %s" + % (rec["label"], Dnorm, from_local_data)) + checked += 1 + F.write("%s|%s\n" % (rec["label"], r"\N" if sigma is None else repr(sigma))) + written += 1 + nulls += sigma is None + if written % 100000 == 0: + print("%s/%s done" % (written, expected)) + + # Curves stored with a non-minimal model: compute from + # local_data and cross-check against normdisc. + for rec in db.ec_nfcurves.search( + {"non_min_p": {"$ne": []}}, + ["label", "conductor_norm", "normdisc", "non_min_p", "local_data"], + sort=[], limit=limit): + Dnorm = min_disc_norm(rec) + from_normdisc = min_disc_norm_from_normdisc(rec) + if Dnorm != from_normdisc: + raise ValueError( + "%s: local_data gives Norm(D_min) = %s but normdisc gives %s" + % (rec["label"], Dnorm, from_normdisc)) + sigma = szpiro_ratio(Dnorm, rec["conductor_norm"], rec["label"]) + F.write("%s|%s\n" % (rec["label"], r"\N" if sigma is None else repr(sigma))) + written += 1 + nulls += sigma is None + + if checked != target_checks: + raise RuntimeError("asked for %s minimal-model cross-checks but made %s" + % (target_checks, checked)) + if written != expected: + raise RuntimeError( + "expected to write %s rows (%s minimal + %s non-minimal) but wrote %s" + % (expected, processed_minimal, processed_nonminimal, written)) + print("Generated %s rows (%s NULL, i.e. everywhere good reduction)" % (written, nulls)) + print("Cross-checked %s minimal-model rows against local_data" % checked) + + # Verify before installing the file, so that a failure leaves nothing behind. + if verify_count: + verify(tmpfile, verify_count, rand=rand) + + # mkstemp creates the file 0600; the data file is meant to be readable + # by whoever uploads it, as it would be if we had just opened outfile. + os.chmod(tmpfile, 0o644) + os.replace(tmpfile, outfile) + except BaseException: + if os.path.exists(tmpfile): + os.unlink(tmpfile) + raise + print("Wrote %s" % outfile) + + +def verify(datafile, nchecks=20, rand=None): """Recompute szpiro_ratio for random rows of the output file from scratch. This is an independent check: the curve is rebuilt in Sage from its @@ -152,13 +307,19 @@ def verify(datafile, nchecks=20): ideal are recomputed, without using conductor_norm, normdisc or local_data. """ - from random import sample from sage.all import EllipticCurve from lmfdb.ecnf.WebEllipticCurve import FIELD, parse_ainvs + if rand is None: + rand = random.Random() with open(datafile) as F: lines = F.read().splitlines()[3:] - for line in sample(lines, nchecks): + if nchecks > len(lines): + raise ValueError( + "asked to verify %s rows from scratch, but the generated file has " + "only %s data rows; use --verify %s or fewer (or raise --limit)" + % (nchecks, len(lines), len(lines))) + for line in rand.sample(lines, nchecks): label, _, sigma = line.partition("|") rec = db.ec_nfcurves.lookup(label, ["field_label", "ainvs", "base_change"]) K = FIELD(rec["field_label"]).K() @@ -166,12 +327,18 @@ def verify(datafile, nchecks=20): Nnorm = E.conductor().norm() Dnorm = E.minimal_discriminant_ideal().norm() if Nnorm == 1: - assert sigma == r"\N", "%s: expected NULL, file has %s" % (label, sigma) + if sigma != r"\N": + raise ValueError("%s: has everywhere good reduction, so the file should " + "give \\N, but it gives %s" % (label, sigma)) print("%s: everywhere good reduction, NULL ok" % label) else: + if sigma == r"\N": + raise ValueError("%s: file gives \\N but the conductor has norm %s" + % (label, Nnorm)) recomputed = log(Dnorm) / log(Nnorm) - assert abs(recomputed - float(sigma)) < 1e-12, \ - "%s: file has %s but Sage gives %s" % (label, sigma, recomputed) + if abs(recomputed - float(sigma)) >= 1e-12: + raise ValueError("%s: file has %s but Sage gives %s" + % (label, sigma, recomputed)) print("%s: %s ok (Norm(D_min)=%s, Norm(N)=%s, base change of %s)" % (label, sigma, Dnorm, Nnorm, rec["base_change"] or "nothing")) @@ -184,8 +351,13 @@ def verify(datafile, nchecks=20): parser.add_argument("--sample-check", type=int, default=1000, help="number of minimal-model rows to cross-check against local_data") parser.add_argument("--verify", type=int, default=0, metavar="N", - help="after generating, recompute N random rows from scratch with Sage") + help="before installing the file, recompute N random rows from scratch with Sage") + parser.add_argument("--seed", type=int, default=None, + help="seed for the rows sampled by --sample-check and --verify") args = parser.parse_args() - generate(args.outfile, limit=args.limit, sample_check=args.sample_check) - if args.verify: - verify(args.outfile, args.verify) + for opt in ("limit", "sample_check", "verify"): + value = getattr(args, opt) + if value is not None and value < 0: + parser.error("--%s must be nonnegative (got %s)" % (opt.replace("_", "-"), value)) + generate(args.outfile, limit=args.limit, sample_check=args.sample_check, + verify_count=args.verify, seed=args.seed)