Skip to content

Redesign the API home page as a navigable table directory (LMFDB#4964) #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
roed-math wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Redesign the API home page as a navigable table directory (LMFDB#4964) #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 86 additions & 3 deletions lmfdb/api/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,81 @@ def hidden_collection(c):
# """
# return set([t[0] for t in sum([val['key'] for name, val in collection.index_information().items() if name!='_id_'],[])])

# Human-readable names for the datasets (the prefix before the first
# underscore in a table name), so that the API home page can explain
# what, e.g., hgcwa stands for. Prefixes not listed here are displayed
# without a description.
dataset_names = {
"artin": "Artin representations",
"av": "Abelian varieties over finite fields",
"belyi": "Belyi maps",
"bmf": "Bianchi modular forms",
"char": "Dirichlet characters",
"cluster": "Cluster pictures",
"data": "Data uploads",
"ec": "Elliptic curves",
"fq": "Function fields",
"g2c": "Genus 2 curves",
"gps": "Groups",
"halfmf": "Half-integral weight modular forms",
"hecke": "Hecke algebras",
"hgcwa": "Higher genus curves with automorphisms",
"hgm": "Hypergeometric motives",
"hmf": "Hilbert modular forms",
"hmsurfaces": "Hilbert modular surfaces",
"inv": "Database inventory",
"lat": "Integral lattices",
"lf": "$p$-adic fields",
"lfunc": "L-functions",
"lmfdb": "LMFDB internals",
"maass": "Maass forms",
"mf": "Classical modular forms",
"modcurve": "Modular curves",
"modlgal": "mod-$\\ell$ Galois representations",
"modlmf": "mod-$\\ell$ modular forms",
"nf": "Number fields",
"noncong": "Noncongruence modular forms",
"pg": "Postgres statistics",
"quaternion": "Quaternion algebras",
"shimcurve": "Shimura curves",
"shimura": "Shimura curves (old)",
"smf": "Siegel modular forms",
"test": "Test tables",
"weil": "Weil polynomials",
}


def get_database_info(show_hidden=False):
"""
Returns a dictionary describing the tables available through the API,
grouped by dataset (the prefix before the first underscore).

Each value is a list of tuples ``(tablename, shortname, count, description)``,
sorted by table name. This does not query the database once per table:
the row counts are in-memory totals loaded from meta_tables at startup,
and the descriptions come from a single bulk query of the knowl database.

INPUT:

- ``show_hidden`` -- whether to include tables (such as test tables)
that are hidden from the main API page by default
"""
try:
from lmfdb.knowledge.knowl import knowldb
descriptions = knowldb.get_table_descriptions()
except Exception:
# The API index should still work if the knowl database is unavailable
descriptions = {}
info = defaultdict(list)
for table in db.tablenames:
if hidden_collection(table) and not show_hidden:
continue
i = table.find('_')
if i == -1:
raise RuntimeError
database = table[:i]
coll = getattr(db, table)
info[database].append((table, table[i+1:], coll.count()))
info[database].append((table, table[i+1:], coll.count(), descriptions.get(table, "")))
return info

@api_page.route("/options")
Expand All @@ -74,8 +140,25 @@ def options():
@api_page.route("/")
def index(show_hidden=False):
databases = get_database_info(show_hidden)
title = "API"
return render_template("api.html", **locals())
ntables = sum(len(tables) for tables in databases.values())
nrows = sum(count for tables in databases.values() for _, _, count, _ in tables)
nhidden = sum(1 for table in db.tablenames if hidden_collection(table))
dataset_totals = {database: (len(tables), sum(count for _, _, count, _ in tables))
for database, tables in databases.items()}
return render_template("api.html",
title="API",
databases=databases,
dataset_names=dataset_names,
dataset_totals=dataset_totals,
ntables=ntables,
nrows=nrows,
nhidden=nhidden,
show_hidden=show_hidden,
learnmore=[
("Access options", url_for(".options")),
("Table statistics", url_for(".stats")),
("Auxiliary datasets", url_for("datasets"))],
bread=[("API", " ")])

@api_page.route("/all")
def full_index():
Expand Down
123 changes: 99 additions & 24 deletions lmfdb/api/templates/api.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,45 @@

{% block content %}
<style>
.api-entries > li { margin-bottom: 14px; }
details#api-docs { margin: 12px 0; }
details#api-docs > summary { cursor: pointer; font-weight: bold; }
div#api-filter-box { margin: 12px 0; }
span#api-filter-count { margin-left: 12px; color: #666; }
div#api-jump { margin: 6px 0 14px 0; }
div.api-dataset { margin-bottom: 16px; }
h4.api-dataset-head { margin-bottom: 3px; }
span.api-dataset-total { font-weight: normal; color: #666; }
table.api-table-list td { padding-right: 16px; }
td.api-table-name { white-space: nowrap; }
td.api-table-count { text-align: right; white-space: nowrap; }
</style>


<div>
This is a very basic API for accessing the LMFDB Database.
It lists available databases and collections,
links to their respective descriptions,
and has very limited query capabilities.
This page is the entry point to the API of the LMFDB, which provides
direct access to the underlying database.
The <a href="#api-tables">tables</a> listed below are grouped into datasets by
the prefix of their name; click on a table name to see its first records,
its schema, and links to the results of your query in machine-readable formats.
See also the <a href="{{ url_for('.stats') }}">table statistics</a> page
for the size of each table on disk, and the
<a href="{{ url_for('.options') }}">access options</a> page for other ways
of getting at the data.
</div>
<div>
<b>Please use this API responsibly!</b>
</div>

<details id="api-docs">
<summary>Query syntax and examples</summary>
<div>
Queries are url encoded <b><code>key=value</code></b> parameters, where the value has a prefix to specify the type.
Keys starting with "<code>_</code>" are meta-parameters further refining query.
They can be combined by specifying several ones delimited by <code>&amp;</code> to further drill down to the desired objects.
The result formats are JSON, YAML or HTML (default).
Each list is limited by a maximum of 100 results and the <code>next</code> entry contains the query to request more objects.
The overall limit is at about 10000 results and beyond that it is necessary to further refine the query to the results in question.
</div>
<div>
<b>Please use this API responsibly!</b>
</div>
<h4>Type-prefixes for query values</h4>
<div>
<ul>
Expand All @@ -47,6 +65,7 @@ <h4>Meta-parameters</h4>
where the prefix "-" indicates to sort in descending order.
</li>
<li>Finally, <code>_delim</code> is used to specify the delimiter (default: "<code>,</code>")</li>
</ul>
</div>
<div>
<h4>Examples</h4>
Expand Down Expand Up @@ -86,23 +105,79 @@ <h4>Examples</h4>
-->
</ul>
</div>
</details>

<h3>Available Collections</h3>
<ul class="api-entries">
{% for db, collections in databases.items()|sort %}
<li>
<b>{{ db }}</b>
<!--
(<a href="https://github.com/LMFDB/lmfdb-inventory/blob/master/db-{{ db }}.md"
target="_blank">description</a>)
-->
<br/>
{% for (fullname, shortname, count) in collections %}
<a href="{{ url_for('.api_query', table=fullname) }}">{{ shortname }} ({{ count }})</a>
{% if not loop.last %}&middot;{% endif %}
{% endfor %}
</li>
<h3 id="api-tables">Available tables</h3>

<div>
The database contains {{ ntables }} tables in {{ databases|length }} datasets,
with a total of {{ "{:,}".format(nrows) }} rows.
{% if show_hidden %}
All tables are shown, including test tables;
<a href="{{ url_for('.index') }}">hide test tables</a>.
{% elif nhidden %}
{{ nhidden }} test table{% if nhidden != 1 %}s are{% else %} is{% endif %} not shown;
<a href="{{ url_for('.full_index') }}">show all tables</a>.
{% endif %}
</div>

<div id="api-filter-box">
<input type="text" id="api-filter" size="30" placeholder="Filter tables" autocomplete="off" oninput="filterTables()" />
<span id="api-filter-count"></span>
</div>

<div id="api-jump">
Jump to:
{% for database in databases|sort %}
<a href="#{{ database }}">{{ database }}</a>{% if not loop.last %} &middot;{% endif %}
{% endfor %}
</ul>
</div>

{% for database, tables in databases.items()|sort %}
<div class="api-dataset" id="{{ database }}" data-dataset="{{ database }} {{ dataset_names.get(database, '')|lower }}">
<h4 class="api-dataset-head">{{ database }}{% if database in dataset_names %} &mdash; {{ dataset_names[database] }}{% endif %}
<span class="api-dataset-total">({{ dataset_totals[database][0] }} table{% if dataset_totals[database][0] != 1 %}s{% endif %},
{{ "{:,}".format(dataset_totals[database][1]) }} row{% if dataset_totals[database][1] != 1 %}s{% endif %})</span></h4>
<table class="ntdata api-table-list">
<tbody>
{% for (fullname, shortname, count, description) in tables|sort %}
<tr class="api-table-row" data-name="{{ fullname|lower }}" data-desc="{{ description|lower }}">
<td class="api-table-name"><a href="{{ url_for('.api_query', table=fullname) }}">{{ fullname }}</a></td>
<td class="api-table-count" title="number of rows">{{ "{:,}".format(count) }}</td>
<td class="api-table-desc">{{ description }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
<div id="api-no-match" style="display: none;">No tables match your filter.</div>

<script type="text/javascript">
function filterTables() {
var needle = document.getElementById("api-filter").value.toLowerCase().trim();
var datasets = document.getElementsByClassName("api-dataset");
var shown = 0;
for (var i = 0; i < datasets.length; i++) {
var dsmatch = datasets[i].getAttribute("data-dataset").indexOf(needle) !== -1;
var rows = datasets[i].getElementsByClassName("api-table-row");
var visible = 0;
for (var j = 0; j < rows.length; j++) {
var match = !needle || dsmatch ||
rows[j].getAttribute("data-name").indexOf(needle) !== -1 ||
rows[j].getAttribute("data-desc").indexOf(needle) !== -1;
rows[j].style.display = match ? "" : "none";
if (match) visible++;
}
datasets[i].style.display = visible ? "" : "none";
shown += visible;
}
// anchor links to hidden sections are not useful while filtering
document.getElementById("api-jump").style.display = needle ? "none" : "";
document.getElementById("api-filter-count").textContent =
needle ? "showing " + shown + " of {{ ntables }} tables" : "";
document.getElementById("api-no-match").style.display = shown ? "none" : "";
}
</script>

{% endblock %}
34 changes: 32 additions & 2 deletions lmfdb/api/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,42 @@
import re

from lmfdb.tests import LmfdbTest

class ApiTest(LmfdbTest):
def test_api_home(self):
r"""
Check that the top-level api page works
Check that the top-level api page works: tables grouped into datasets
with row counts and descriptions, the collapsed usage docs, the filter
box, and links to the stats and access options pages
"""
data = self.tc.get("/api", follow_redirects=True).get_data(as_text=True)
assert "API for accessing the LMFDB Database" in data
assert "entry point to the API" in data
assert "Query syntax and examples" in data
assert 'id="api-filter"' in data
# datasets are explained
assert "Higher genus curves with automorphisms" in data
assert 'id="hgcwa"' in data
# links to the stats and access options pages
assert '"/api/stats"' in data
assert '"/api/options"' in data
# test tables are hidden by default, but can be shown
assert "test_table" not in data
assert '"/api/all"' in data

def test_api_home_links(self):
r"""
Check that the tables listed on /api/all (which includes everything
on /api/) are exactly the search tables, with working links
"""
data = self.tc.get("/api/all", follow_redirects=True).get_data(as_text=True)
assert "test_table" in data
links = re.findall(r'<td class="api-table-name"><a href="/api/([^"]+)/">([^<]+)</a>', data)
assert sorted(name for _, name in links) == sorted(self.db.tablenames)
assert all(href == name for href, name in links)
# the anchors in the jump strip match the dataset sections
sections = set(re.findall(r'<div class="api-dataset" id="([^"]+)"', data))
jumps = set(re.findall(r'<a href="#([^"]+)">', data)) - {"api-tables"}
assert jumps == sections

def test_api_databases(self):
r"""
Expand Down
14 changes: 14 additions & 0 deletions lmfdb/knowledge/knowl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,20 @@ def get_table_description(self, table):
if L:
return Knowl(L[0][0], data=dict(zip(fields, L[0])))

def get_table_descriptions(self):
"""
The descriptions of all tables (the ``tables.<name>`` knowls),
fetched in a single query.

OUTPUT:

A dictionary with table names as keys and description strings as values;
tables with no description knowl are omitted.
"""
selecter = SQL("SELECT id, content FROM (SELECT DISTINCT ON (id) id, content FROM kwl_knowls WHERE id LIKE %s AND type = %s AND status >= %s ORDER BY id, timestamp) knowls ORDER BY id")
L = self._safe_execute(selecter, ["tables.%", 2, 0])
return {rec[0].split(".", 1)[1]: rec[1] for rec in L}

def set_table_description(self, table, description):
uid = db.login()
kid = f"tables.{table}"
Expand Down
Loading