From 6cb294edb8415fb00cea71eb540a624a4ab11a04 Mon Sep 17 00:00:00 2001 From: David Roe Date: Sun, 19 Jul 2026 14:00:29 -0400 Subject: [PATCH] Redesign the API home page as a navigable table directory (LMFDB#4964) Group the tables on /api into datasets with human-readable names (so hgcwa etc. are explained), show row counts (free: in-memory meta_tables totals) and short table descriptions (one bulk query of the tables. knowls via a new knowldb.get_table_descriptions), add a client-side filter box and a jump-to-dataset strip, collapse the long usage docs into a
block, link the stats and access options pages, and actually hide test tables by default (the show_hidden flag and /api/all were dead code). All existing URLs and machine formats are unchanged. Verified with the flask test client (page loads in ~0.16s warm; all 193 table links resolve; JSON/YAML output unchanged), in a live browser on port 38046 (filter, no-match state and details work with no console errors), sage -python -m pytest lmfdb/api/test_api.py (7 passed) and pyflakes. git merge-tree confirms no conflicts with open PRs #17/#18/#22. Co-Authored-By: Claude Fable 5 --- lmfdb/api/api.py | 89 ++++++++++++++++++++++++- lmfdb/api/templates/api.html | 123 ++++++++++++++++++++++++++++------- lmfdb/api/test_api.py | 34 +++++++++- lmfdb/knowledge/knowl.py | 14 ++++ 4 files changed, 231 insertions(+), 29 deletions(-) diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py index 832f5ba8a2..bb497daa94 100644 --- a/lmfdb/api/api.py +++ b/lmfdb/api/api.py @@ -46,15 +46,81 @@ def hidden_collection(c): # """ # return set([t[0] for t in sum([val['key'] for name, val in collection.index_information().items() if name!='_id_'],[])]) +# Human-readable names for the datasets (the prefix before the first +# underscore in a table name), so that the API home page can explain +# what, e.g., hgcwa stands for. Prefixes not listed here are displayed +# without a description. +dataset_names = { + "artin": "Artin representations", + "av": "Abelian varieties over finite fields", + "belyi": "Belyi maps", + "bmf": "Bianchi modular forms", + "char": "Dirichlet characters", + "cluster": "Cluster pictures", + "data": "Data uploads", + "ec": "Elliptic curves", + "fq": "Function fields", + "g2c": "Genus 2 curves", + "gps": "Groups", + "halfmf": "Half-integral weight modular forms", + "hecke": "Hecke algebras", + "hgcwa": "Higher genus curves with automorphisms", + "hgm": "Hypergeometric motives", + "hmf": "Hilbert modular forms", + "hmsurfaces": "Hilbert modular surfaces", + "inv": "Database inventory", + "lat": "Integral lattices", + "lf": "$p$-adic fields", + "lfunc": "L-functions", + "lmfdb": "LMFDB internals", + "maass": "Maass forms", + "mf": "Classical modular forms", + "modcurve": "Modular curves", + "modlgal": "mod-$\\ell$ Galois representations", + "modlmf": "mod-$\\ell$ modular forms", + "nf": "Number fields", + "noncong": "Noncongruence modular forms", + "pg": "Postgres statistics", + "quaternion": "Quaternion algebras", + "shimcurve": "Shimura curves", + "shimura": "Shimura curves (old)", + "smf": "Siegel modular forms", + "test": "Test tables", + "weil": "Weil polynomials", +} + + def get_database_info(show_hidden=False): + """ + Returns a dictionary describing the tables available through the API, + grouped by dataset (the prefix before the first underscore). + + Each value is a list of tuples ``(tablename, shortname, count, description)``, + sorted by table name. This does not query the database once per table: + the row counts are in-memory totals loaded from meta_tables at startup, + and the descriptions come from a single bulk query of the knowl database. + + INPUT: + + - ``show_hidden`` -- whether to include tables (such as test tables) + that are hidden from the main API page by default + """ + try: + from lmfdb.knowledge.knowl import knowldb + descriptions = knowldb.get_table_descriptions() + except Exception: + # The API index should still work if the knowl database is unavailable + descriptions = {} info = defaultdict(list) for table in db.tablenames: + if hidden_collection(table) and not show_hidden: + continue i = table.find('_') if i == -1: raise RuntimeError database = table[:i] coll = getattr(db, table) - info[database].append((table, table[i+1:], coll.count())) + info[database].append((table, table[i+1:], coll.count(), descriptions.get(table, ""))) return info @api_page.route("/options") @@ -74,8 +140,25 @@ def options(): @api_page.route("/") def index(show_hidden=False): databases = get_database_info(show_hidden) - title = "API" - return render_template("api.html", **locals()) + ntables = sum(len(tables) for tables in databases.values()) + nrows = sum(count for tables in databases.values() for _, _, count, _ in tables) + nhidden = sum(1 for table in db.tablenames if hidden_collection(table)) + dataset_totals = {database: (len(tables), sum(count for _, _, count, _ in tables)) + for database, tables in databases.items()} + return render_template("api.html", + title="API", + databases=databases, + dataset_names=dataset_names, + dataset_totals=dataset_totals, + ntables=ntables, + nrows=nrows, + nhidden=nhidden, + show_hidden=show_hidden, + learnmore=[ + ("Access options", url_for(".options")), + ("Table statistics", url_for(".stats")), + ("Auxiliary datasets", url_for("datasets"))], + bread=[("API", " ")]) @api_page.route("/all") def full_index(): diff --git a/lmfdb/api/templates/api.html b/lmfdb/api/templates/api.html index d604f5391d..0d0730ee0f 100644 --- a/lmfdb/api/templates/api.html +++ b/lmfdb/api/templates/api.html @@ -2,17 +2,38 @@ {% block content %}
-This is a very basic API for accessing the LMFDB Database. -It lists available databases and collections, -links to their respective descriptions, -and has very limited query capabilities. +This page is the entry point to the API of the LMFDB, which provides +direct access to the underlying database. +The tables listed below are grouped into datasets by +the prefix of their name; click on a table name to see its first records, +its schema, and links to the results of your query in machine-readable formats. +See also the table statistics page +for the size of each table on disk, and the +access options page for other ways +of getting at the data.
+Please use this API responsibly! +
+ +
+Query syntax and examples +
Queries are url encoded key=value parameters, where the value has a prefix to specify the type. Keys starting with "_" are meta-parameters further refining query. They can be combined by specifying several ones delimited by & to further drill down to the desired objects. @@ -20,9 +41,6 @@ Each list is limited by a maximum of 100 results and the next entry contains the query to request more objects. The overall limit is at about 10000 results and beyond that it is necessary to further refine the query to the results in question.
-
-Please use this API responsibly! -

Type-prefixes for query values

    @@ -47,6 +65,7 @@

    Meta-parameters

    where the prefix "-" indicates to sort in descending order.
  • Finally, _delim is used to specify the delimiter (default: ",")
    • +

Examples

@@ -86,23 +105,79 @@

Examples

-->
+
-

Available Collections

-
    -{% for db, collections in databases.items()|sort %} -
  • - {{ db }} - -
    - {% for (fullname, shortname, count) in collections %} - {{ shortname }} ({{ count }}) - {% if not loop.last %}·{% endif %} - {% endfor %} -
    • +

    Available tables

    + +
    +The database contains {{ ntables }} tables in {{ databases|length }} datasets, +with a total of {{ "{:,}".format(nrows) }} rows. +{% if show_hidden %} +All tables are shown, including test tables; +hide test tables. +{% elif nhidden %} +{{ nhidden }} test table{% if nhidden != 1 %}s are{% else %} is{% endif %} not shown; +show all tables. +{% endif %} +
    + +
    + + +
    + +
    +Jump to: +{% for database in databases|sort %} +{{ database }}{% if not loop.last %} ·{% endif %} {% endfor %} -
+ + +{% for database, tables in databases.items()|sort %} +
+

{{ database }}{% if database in dataset_names %} — {{ dataset_names[database] }}{% endif %} + ({{ dataset_totals[database][0] }} table{% if dataset_totals[database][0] != 1 %}s{% endif %}, + {{ "{:,}".format(dataset_totals[database][1]) }} row{% if dataset_totals[database][1] != 1 %}s{% endif %})

+ + + {% for (fullname, shortname, count, description) in tables|sort %} + + + + + + {% endfor %} + +
{{ fullname }}{{ "{:,}".format(count) }}{{ description }}
+
+{% endfor %} + + + {% endblock %} diff --git a/lmfdb/api/test_api.py b/lmfdb/api/test_api.py index 44af9d883c..78919eab70 100644 --- a/lmfdb/api/test_api.py +++ b/lmfdb/api/test_api.py @@ -1,12 +1,42 @@ +import re + from lmfdb.tests import LmfdbTest class ApiTest(LmfdbTest): def test_api_home(self): r""" - Check that the top-level api page works + Check that the top-level api page works: tables grouped into datasets + with row counts and descriptions, the collapsed usage docs, the filter + box, and links to the stats and access options pages """ data = self.tc.get("/api", follow_redirects=True).get_data(as_text=True) - assert "API for accessing the LMFDB Database" in data + assert "entry point to the API" in data + assert "Query syntax and examples" in data + assert 'id="api-filter"' in data + # datasets are explained + assert "Higher genus curves with automorphisms" in data + assert 'id="hgcwa"' in data + # links to the stats and access options pages + assert '"/api/stats"' in data + assert '"/api/options"' in data + # test tables are hidden by default, but can be shown + assert "test_table" not in data + assert '"/api/all"' in data + + def test_api_home_links(self): + r""" + Check that the tables listed on /api/all (which includes everything + on /api/) are exactly the search tables, with working links + """ + data = self.tc.get("/api/all", follow_redirects=True).get_data(as_text=True) + assert "test_table" in data + links = re.findall(r'([^<]+)', data) + assert sorted(name for _, name in links) == sorted(self.db.tablenames) + assert all(href == name for href, name in links) + # the anchors in the jump strip match the dataset sections + sections = set(re.findall(r'
', data)) - {"api-tables"} + assert jumps == sections def test_api_databases(self): r""" diff --git a/lmfdb/knowledge/knowl.py b/lmfdb/knowledge/knowl.py index e98dc31fb2..5f061db2b3 100644 --- a/lmfdb/knowledge/knowl.py +++ b/lmfdb/knowledge/knowl.py @@ -384,6 +384,20 @@ def get_table_description(self, table): if L: return Knowl(L[0][0], data=dict(zip(fields, L[0]))) + def get_table_descriptions(self): + """ + The descriptions of all tables (the ``tables.`` knowls), + fetched in a single query. + + OUTPUT: + + A dictionary with table names as keys and description strings as values; + tables with no description knowl are omitted. + """ + selecter = SQL("SELECT id, content FROM (SELECT DISTINCT ON (id) id, content FROM kwl_knowls WHERE id LIKE %s AND type = %s AND status >= %s ORDER BY id, timestamp) knowls ORDER BY id") + L = self._safe_execute(selecter, ["tables.%", 2, 0]) + return {rec[0].split(".", 1)[1]: rec[1] for rec in L} + def set_table_description(self, table, description): uid = db.login() kid = f"tables.{table}"