diff --git a/lmfdb/api/api.py b/lmfdb/api/api.py index 4800ca5f05..6290ae5b28 100644 --- a/lmfdb/api/api.py +++ b/lmfdb/api/api.py @@ -36,8 +36,12 @@ def pretty_document(rec, sep=", ", id=True): def hidden_collection(c): """ hide some collections from the main page (still available via direct requests) + + Test tables follow two naming conventions (``test_something`` and + ``something_test``) and both are hidden, as are the auxiliary tables + whose names carry one of the suffixes below. """ - return c.startswith("test") or c.endswith(".rand") or c.endswith(".stats") or c.endswith(".chunks") or c.endswith(".new") or c.endswith(".old") + return c.startswith("test") or c.endswith("_test") or c.endswith(".rand") or c.endswith(".stats") or c.endswith(".chunks") or c.endswith(".new") or c.endswith(".old") #def collection_indexed_keys(collection): # """ @@ -46,15 +50,83 @@ def hidden_collection(c): # """ # return set([t[0] for t in sum([val['key'] for name, val in collection.index_information().items() if name!='_id_'],[])]) +# Human-readable names for the datasets (the prefix before the first +# underscore in a table name), so that the API home page can explain +# what, e.g., hgcwa stands for. Prefixes not listed here are displayed +# without a description. +dataset_names = { + "artin": "Artin representations", + "av": "Abelian varieties over finite fields", + "belyi": "Belyi maps", + "bmf": "Bianchi modular forms", + "char": "Dirichlet characters", + "cluster": "Cluster pictures", + "data": "Data uploads", + "ec": "Elliptic curves", + "fq": "Finite fields", + "g2c": "Genus 2 curves", + "gps": "Groups", + "halfmf": "Half-integral weight modular forms", + "hecke": "Hecke algebras", + "hgcwa": "Higher genus curves with automorphisms", + "hgm": "Hypergeometric motives", + "hmf": "Hilbert modular forms", + "hmsurfaces": "Hilbert modular surfaces", + "inv": "Database inventory", + "lat": "Integral lattices", + "lf": "$p$-adic fields", + "lfunc": "L-functions", + "lmfdb": "LMFDB internals", + "maass": "Maass forms", + "mf": "Classical modular forms", + "modcurve": "Modular curves", + "modlgal": "mod-$\\ell$ Galois representations", + "modlmf": "mod-$\\ell$ modular forms", + "nf": "Number fields", + "noncong": "Noncongruence modular forms", + "pg": "Postgres statistics", + "quaternion": "Quaternion algebras", + "shimcurve": "Shimura curves", + "shimura": "Shimura curves (old)", + "smf": "Siegel modular forms", + "test": "Test tables", + "weil": "Weil polynomials", +} + + def get_database_info(show_hidden=False): + """ + Returns a dictionary describing the tables available through the API, + grouped by dataset (the prefix before the first underscore). + + Each value is a list of tuples ``(tablename, shortname, count, description)``, + sorted by table name. This does not query the database once per table: + the row counts are in-memory totals loaded from meta_tables at startup, + and the descriptions come from a single bulk query of the knowl database. + + INPUT: + + - ``show_hidden`` -- whether to include the tables (test tables and some + auxiliary ones) that are hidden from the main API page by default + """ + try: + from lmfdb.knowledge.knowl import knowldb + descriptions = knowldb.get_table_descriptions() + except Exception: + # The API index should still work if the knowl database is unavailable, + # but a failure here silently empties a column of the page, so log it + logger.exception("Could not load the table descriptions for the API index") + descriptions = {} info = defaultdict(list) for table in db.tablenames: + if hidden_collection(table) and not show_hidden: + continue i = table.find('_') if i == -1: raise RuntimeError database = table[:i] coll = getattr(db, table) - info[database].append((table, table[i+1:], coll.count())) + info[database].append((table, table[i+1:], coll.count(), descriptions.get(table, ""))) return info @api_page.route("/options") @@ -74,8 +146,25 @@ def options(): @api_page.route("/") def index(show_hidden=False): databases = get_database_info(show_hidden) - title = "API" - return render_template("api.html", **locals()) + ntables = sum(len(tables) for tables in databases.values()) + nrows = sum(count for tables in databases.values() for _, _, count, _ in tables) + nhidden = sum(1 for table in db.tablenames if hidden_collection(table)) + dataset_totals = {database: (len(tables), sum(count for _, _, count, _ in tables)) + for database, tables in databases.items()} + return render_template("api.html", + title="API", + databases=databases, + dataset_names=dataset_names, + dataset_totals=dataset_totals, + ntables=ntables, + nrows=nrows, + nhidden=nhidden, + show_hidden=show_hidden, + learnmore=[ + ("Access options", url_for(".options")), + ("Table statistics", url_for(".stats")), + ("Auxiliary datasets", url_for("datasets"))], + bread=[("API", " ")]) @api_page.route("/all") def full_index(): diff --git a/lmfdb/api/templates/api.html b/lmfdb/api/templates/api.html index d604f5391d..d933bee555 100644 --- a/lmfdb/api/templates/api.html +++ b/lmfdb/api/templates/api.html @@ -2,17 +2,38 @@ {% block content %}
-This is a very basic API for accessing the LMFDB Database. -It lists available databases and collections, -links to their respective descriptions, -and has very limited query capabilities. +This page is the entry point to the API of the LMFDB, which provides +direct access to the underlying database. +The tables listed below are grouped into datasets by +the prefix of their name; click on a table name to see its first records, +its schema, and links to the results of your query in machine-readable formats. +See also the table statistics page +for the size of each table on disk, and the +access options page for other ways +of getting at the data.
+Please use this API responsibly! +
+ +
+Query syntax and examples +
Queries are url encoded key=value parameters, where the value has a prefix to specify the type. Keys starting with "_" are meta-parameters further refining query. They can be combined by specifying several ones delimited by & to further drill down to the desired objects. @@ -20,9 +41,6 @@ Each list is limited by a maximum of 100 results and the next entry contains the query to request more objects. The overall limit is at about 10000 results and beyond that it is necessary to further refine the query to the results in question.
-
-Please use this API responsibly! -

Type-prefixes for query values

Examples

@@ -86,23 +105,79 @@

Examples

-->
+
-

Available Collections

- + + +{% for database, tables in databases.items()|sort %} +
+

{{ database }}{% if database in dataset_names %} — {{ dataset_names[database] }}{% endif %} + ({{ dataset_totals[database][0] }} table{% if dataset_totals[database][0] != 1 %}s{% endif %}, + {{ "{:,}".format(dataset_totals[database][1]) }} row{% if dataset_totals[database][1] != 1 %}s{% endif %})

+ + + {% for (fullname, shortname, count, description) in tables|sort %} + + + + + + {% endfor %} + +
{{ fullname }}{{ "{:,}".format(count) }}{{ description }}
+
+{% endfor %} + + + {% endblock %} diff --git a/lmfdb/api/test_api.py b/lmfdb/api/test_api.py index 44af9d883c..c34263ce8f 100644 --- a/lmfdb/api/test_api.py +++ b/lmfdb/api/test_api.py @@ -1,12 +1,139 @@ +import re +from unittest.mock import patch + from lmfdb.tests import LmfdbTest +from lmfdb.api.api import hidden_collection +from lmfdb.knowledge.knowl import knowldb +from lmfdb.utils.psycopg_compat import SQL + + +def table_links(page): + r""" + The (href, table name) pairs of the table links on an api index page + """ + return re.findall(r'([^<]+)', page) + + +class DescriptionKnowlTest(LmfdbTest): + r""" + The api index displays the ``tables.`` description knowls, so the + lookups behind it have to return the current revision of each knowl. + """ + + def _edited(self, pattern, limit=3): + r""" + The ids of some description knowls matching ``pattern`` whose content + has been edited, i.e. that have more than one visible revision + """ + selecter = SQL("SELECT id FROM kwl_knowls WHERE id LIKE %s AND type = %s AND status >= %s GROUP BY id HAVING COUNT(DISTINCT content) > 1 ORDER BY id LIMIT %s") + return [rec[0] for rec in knowldb._safe_execute(selecter, [pattern, 2, 0, limit])] + + def _current(self, kid): + r""" + The content of the newest revision of a knowl visible on this server + """ + # get_edit_history sorts the revisions by increasing timestamp + return knowldb.get_edit_history(kid)[-1]["content"] + + def test_table_descriptions_are_current(self): + r""" + Check that both the single and the bulk table description lookups + return the newest revision of an edited knowl + """ + edited = self._edited("tables.%") + assert edited, "no edited table description knowl to test against" + bulk = knowldb.get_table_descriptions() + for kid in edited: + table = kid.split(".", 1)[1] + current = self._current(kid) + assert bulk[table] == current, "stale description for %s" % table + assert knowldb.get_table_description(table).content == current + + def test_column_descriptions_are_current(self): + r""" + Check that the column description lookup returns the newest revision + of an edited knowl + """ + edited = self._edited("columns.%", limit=1) + assert edited, "no edited column description knowl to test against" + kid = edited[0] + _, table, col = kid.split(".") + assert knowldb.get_column_descriptions(table)[col].content == self._current(kid) + class ApiTest(LmfdbTest): def test_api_home(self): r""" - Check that the top-level api page works + Check that the top-level api page works: tables grouped into datasets + with row counts and descriptions, the collapsed usage docs, the filter + box, and links to the stats and access options pages + """ + data = self.tc.get("/api", follow_redirects=True).get_data(as_text=True) + assert "entry point to the API" in data + assert "Query syntax and examples" in data + assert 'id="api-filter"' in data + # datasets are explained + assert "Higher genus curves with automorphisms" in data + assert 'id="hgcwa"' in data + # fq_fields holds finite fields, not function fields + assert "fq — Finite fields" in data + # links to the stats and access options pages + assert '"/api/stats"' in data + assert '"/api/options"' in data + # hidden tables are not shown by default, but can be + assert "test_table" not in data + assert '"/api/all"' in data + + def test_api_home_links(self): + r""" + Check that /api/ lists exactly the tables that are not hidden, that + /api/all lists all of them, and that the links work + """ + hidden = {name for name in self.db.tablenames if hidden_collection(name)} + data = self.tc.get("/api", follow_redirects=True).get_data(as_text=True) + links = table_links(data) + assert all(href == name for href, name in links) + assert sorted(name for _, name in links) == sorted(set(self.db.tablenames) - hidden) + + data = self.tc.get("/api/all", follow_redirects=True).get_data(as_text=True) + links = table_links(data) + assert all(href == name for href, name in links) + assert sorted(name for _, name in links) == sorted(self.db.tablenames) + # the anchors in the jump strip match the dataset sections + sections = set(re.findall(r'
', data)) - {"api-tables"} + assert jumps == sections + + def test_api_home_hidden(self): + r""" + Check that tables following either test naming convention (``test_x`` + and ``x_test``) are hidden from /api/ and shown on /api/all + """ + tests = {name for name in self.db.tablenames + if name.startswith("test") or name.endswith("_test")} + assert "test_table" in tests, "no test_-prefixed table in the database" + assert any(name.endswith("_test") for name in tests), "no _test-suffixed table in the database" + assert tests <= {name for name in self.db.tablenames if hidden_collection(name)} + + shown = {name for _, name in table_links(self.tc.get("/api", follow_redirects=True).get_data(as_text=True))} + assert not (tests & shown) + shown = {name for _, name in table_links(self.tc.get("/api/all", follow_redirects=True).get_data(as_text=True))} + assert tests <= shown + + def test_api_home_descriptions(self): + r""" + Check that the description of a table is shown in its row and is + searchable by the filter box """ + description = "The table that the api tests use" + with patch.object(knowldb, "get_table_descriptions", return_value={"test_table": description}): + data = self.tc.get("/api/all", follow_redirects=True).get_data(as_text=True) + assert '%s' % description in data + assert 'data-desc="%s"' % description.lower() in data + # the real descriptions are not all empty either data = self.tc.get("/api", follow_redirects=True).get_data(as_text=True) - assert "API for accessing the LMFDB Database" in data + described = [desc for desc in re.findall(r'([^<]*)', data) if desc.strip()] + assert len(described) > 20, "only %s tables have a description" % len(described) def test_api_databases(self): r""" diff --git a/lmfdb/knowledge/knowl.py b/lmfdb/knowledge/knowl.py index e72983f672..6acb00a1a2 100644 --- a/lmfdb/knowledge/knowl.py +++ b/lmfdb/knowledge/knowl.py @@ -492,9 +492,23 @@ def get_comments(self, ID): selecter = SQL("SELECT id, last_author, timestamp FROM (SELECT DISTINCT ON (id) id, last_author, timestamp FROM kwl_knowls WHERE type = %s AND source = %s AND status >= 0 ORDER BY id, timestamp) knowls ORDER BY timestamp DESC") return self._safe_execute(selecter, [-2, ID]) + def _description_selecter(self, fields, match): + """ + A query for the current revision of the description knowls whose id + satisfies ``match`` (an SQL fragment such as ``id = %s``), which is + followed by the type and the minimum status in the parameter list. + + ``DISTINCT ON (id)`` keeps the first row of each id, so the revisions + have to be sorted newest first for this to be the current one, as in + :meth:`get_knowl`; sorting them the other way returns the revision a + description had when it was first written. + """ + sqlfields = SQL(", ").join(map(Identifier, fields)) + return SQL("SELECT {0} FROM (SELECT DISTINCT ON (id) {0} FROM kwl_knowls WHERE {1} AND type = %s AND status >= %s ORDER BY id, timestamp DESC) knowls ORDER BY id").format(sqlfields, match) + def get_column_descriptions(self, table): fields = ['id'] + self._default_fields - selecter = SQL("SELECT {0} FROM (SELECT DISTINCT ON (id) {0} FROM kwl_knowls WHERE id LIKE %s AND type = %s AND status >= %s ORDER BY id, timestamp) knowls ORDER BY id").format(SQL(", ").join(map(Identifier, fields))) + selecter = self._description_selecter(fields, SQL("id LIKE %s")) L = self._safe_execute(selecter, [f"columns.{table}.%", 2, 0]) return {rec[0].split(".")[-1]: Knowl(rec[0], data=dict(zip(fields, rec))) for rec in L} @@ -518,11 +532,25 @@ def drop_column(self, table, col): def get_table_description(self, table): fields = ['id'] + self._default_fields - selecter = SQL("SELECT {0} FROM (SELECT DISTINCT ON (id) {0} FROM kwl_knowls WHERE id = %s AND type = %s AND status >= %s ORDER BY id, timestamp) knowls ORDER BY id LIMIT 1").format(SQL(", ").join(map(Identifier, fields))) + selecter = self._description_selecter(fields, SQL("id = %s")) L = self._safe_execute(selecter, [f"tables.{table}", 2, 0]) if L: return Knowl(L[0][0], data=dict(zip(fields, L[0]))) + def get_table_descriptions(self): + """ + The descriptions of all tables (the ``tables.`` knowls), + fetched in a single query. + + OUTPUT: + + A dictionary with table names as keys and description strings as values; + tables with no description knowl are omitted. + """ + selecter = self._description_selecter(["id", "content"], SQL("id LIKE %s")) + L = self._safe_execute(selecter, ["tables.%", 2, 0]) + return {rec[0].split(".", 1)[1]: rec[1] for rec in L} + def set_table_description(self, table, description): uid = db.login() kid = f"tables.{table}"