From 3571feeb663f3c85b0a8fbf359e095c82bceca46 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 22 Jul 2026 02:12:09 -0400 Subject: [PATCH 1/4] Fix reST and Markdown glitches surfaced by the documentation build - search/lucky INPUT blocks: bullet continuation lines were over- or staircase-indented, which docutils parses as stray block quotes and broken definition lists (three errors per docstring under Sphinx). Whitespace-only reflow; rendered text unchanged. - NotificationListener: ``LISTEN``ed leaves the inline literal unterminated (no separator after the closing backticks); reworded. - QueryLanguage.md and DataManagement.md had no top-level title (they began at ##); added H1s, which the site needs for navigation and which GitHub renders better too. - Searching.md's table of contents pointed at a raw anchor that Markdown processors do not index; point it at the heading's own slug (the anchor itself stays, so old deep links keep working). Co-Authored-By: Claude Fable 5 --- DataManagement.md | 1 + QueryLanguage.md | 1 + Searching.md | 2 +- psycodict/notifications.py | 3 ++- psycodict/searchtable.py | 55 +++++++++++++++++++------------------- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/DataManagement.md b/DataManagement.md index 76c17ff..1999990 100644 --- a/DataManagement.md +++ b/DataManagement.md @@ -1,3 +1,4 @@ +# Data management ## Introduction diff --git a/QueryLanguage.md b/QueryLanguage.md index ea1d9fb..d865999 100644 --- a/QueryLanguage.md +++ b/QueryLanguage.md @@ -1,3 +1,4 @@ +# The query language ## Introduction diff --git a/Searching.md b/Searching.md index 391825c..2ea0fb0 100644 --- a/Searching.md +++ b/Searching.md @@ -30,7 +30,7 @@ database and a table variable such as `nf = db.nf_fields`. - [Stability](#stability) - [`search`](#search) - - [Parameters](#search-parameters) + - [Parameters](#parameters) - [Return type: iterator vs. list](#return-type-iterator-vs-list) - [Projections](#projections) - [Sorting](#sorting) diff --git a/psycodict/notifications.py b/psycodict/notifications.py index d5cff74..6152601 100644 --- a/psycodict/notifications.py +++ b/psycodict/notifications.py @@ -149,7 +149,8 @@ class NotificationListener: - ``config`` -- a :class:`~psycodict.config.Configuration`; the ``postgresql`` options are used to open the dedicated connection - ``channels`` -- a channel name, or an iterable of them (default: - ``("psycodict_schema",)``); each is validated and ``LISTEN``ed on + ``("psycodict_schema",)``); each is validated and subscribed with + ``LISTEN`` - ``**connect_kwargs`` -- extra keyword arguments passed on to ``psycopg.connect`` (e.g. keepalive settings), overriding the configuration where they overlap diff --git a/psycodict/searchtable.py b/psycodict/searchtable.py index a7f71b5..3c9b75a 100644 --- a/psycodict/searchtable.py +++ b/psycodict/searchtable.py @@ -906,27 +906,27 @@ def lucky(self, query={}, projection=2, offset=0, sort=[], raw=None, raw_values= INPUT: - ``query`` -- a mongo-style dictionary specifying the query. - Generally, the keys will correspond to columns, - and values will either be specific numbers (specifying an equality test) - or dictionaries giving more complicated constraints. - The main exception is that "$or" can be a top level key, - specifying a list of constraints of which at least one must be true. + Generally, the keys will correspond to columns, + and values will either be specific numbers (specifying an equality test) + or dictionaries giving more complicated constraints. + The main exception is that "$or" can be a top level key, + specifying a list of constraints of which at least one must be true. - ``projection`` -- which columns are desired. This can be specified either as a list of columns to include; - a dictionary specifying columns to include (using all True values) - or exclude (using all False values); - a string giving a single column (only returns the value, not a dictionary); - or an integer code (0 means only return the label, - 1 means return all search columns, - 2 means all columns (default)). + a dictionary specifying columns to include (using all True values) + or exclude (using all False values); + a string giving a single column (only returns the value, not a dictionary); + or an integer code (0 means only return the label, + 1 means return all search columns, 2 means all columns (default)). - ``offset`` -- integer. allows retrieval of a later record rather than just first. - ``sort`` -- The sort order, from which the first result is returned. - - None, Using the default sort order for the table - - a list of strings (which are interpreted as column names in the - ascending direction) or of pairs (column name, 1 or -1). - If not specified, will use the default sort order on the table. - - [] (default), unsorted, thus if there is more than one match to - the query then the choice of the result is arbitrary. + + - None, Using the default sort order for the table + - a list of strings (which are interpreted as column names in the + ascending direction) or of pairs (column name, 1 or -1). + If not specified, will use the default sort order on the table. + - [] (default), unsorted, thus if there is more than one match to + the query then the choice of the result is arbitrary. - ``raw`` -- a string, to be used as the WHERE part of the query. DO NOT USE THIS DIRECTLY FOR INPUT FROM WEBSITE. - ``raw_values`` -- a list of values to be substituted for %s entries in the raw string. Useful when strings might include quotes. - ``join`` -- a list of tuples describing other search tables to join to this one, @@ -1003,19 +1003,18 @@ def search( INPUT: - ``query`` -- a mongo-style dictionary specifying the query. - Generally, the keys will correspond to columns, - and values will either be specific numbers (specifying an equality test) - or dictionaries giving more complicated constraints. - The main exception is that "$or" can be a top level key, - specifying a list of constraints of which at least one must be true. + Generally, the keys will correspond to columns, + and values will either be specific numbers (specifying an equality test) + or dictionaries giving more complicated constraints. + The main exception is that "$or" can be a top level key, + specifying a list of constraints of which at least one must be true. - ``projection`` -- which columns are desired. This can be specified either as a list of columns to include; - a dictionary specifying columns to include (using all True values) - or exclude (using all False values); - a string giving a single column (only returns the value, not a dictionary); - or an integer code (0 means only return the label, - 1 means return all search columns (default), - 2 means all columns). + a dictionary specifying columns to include (using all True values) + or exclude (using all False values); + a string giving a single column (only returns the value, not a dictionary); + or an integer code (0 means only return the label, + 1 means return all search columns (default), 2 means all columns). - ``limit`` -- an integer or None (default), giving the maximum number of records to return. - ``offset`` -- a nonnegative integer (default 0), where to start in the list of results. - ``sort`` -- a sort order. Either None or a list of strings (which are interpreted as column names in the ascending direction) or of pairs (column name, 1 or -1). If not specified, will use the default sort order on the table. If you want the result unsorted, use []. From 06f0e85b899b7fa8007ea180c5c68609acbf2097 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 22 Jul 2026 02:12:09 -0400 Subject: [PATCH 2/4] Add Sphinx documentation: guides plus autodoc API reference Sphinx + MyST-Parser + autodoc with the furo theme, designed to be served on Read the Docs. The narrative guides are the existing root-level Markdown files, copied into docs/ at build time by conf.py so the GitHub-rendered copies stay canonical and their relative links keep working on the site; the API reference is one automodule page per module, rendering the Sage-convention docstrings as-is. A fifth CI job builds with -W --keep-going, and .readthedocs.yaml sets fail_on_warning to match, so a broken cross-reference or malformed docstring fails rather than degrading the site. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 26 ++++++++++++++- .gitignore | 8 ++++- .readthedocs.yaml | 27 ++++++++++++++++ docs/api/base.md | 5 +++ docs/api/config.md | 5 +++ docs/api/database.md | 5 +++ docs/api/dbdiff.md | 5 +++ docs/api/encoding.md | 5 +++ docs/api/index.md | 44 ++++++++++++++++++++++++++ docs/api/notifications.md | 5 +++ docs/api/searchtable.md | 5 +++ docs/api/slowlog.md | 5 +++ docs/api/statstable.md | 5 +++ docs/api/table.md | 5 +++ docs/api/utils.md | 5 +++ docs/conf.py | 66 +++++++++++++++++++++++++++++++++++++++ docs/index.md | 28 +++++++++++++++++ docs/requirements.txt | 7 +++++ 18 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/api/base.md create mode 100644 docs/api/config.md create mode 100644 docs/api/database.md create mode 100644 docs/api/dbdiff.md create mode 100644 docs/api/encoding.md create mode 100644 docs/api/index.md create mode 100644 docs/api/notifications.md create mode 100644 docs/api/searchtable.md create mode 100644 docs/api/slowlog.md create mode 100644 docs/api/statstable.md create mode 100644 docs/api/table.md create mode 100644 docs/api/utils.md create mode 100644 docs/conf.py create mode 100644 docs/index.md create mode 100644 docs/requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08b53fb..18d35ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,12 @@ # Continuous integration for psycodict. # -# Four jobs, cheapest first, so an obvious mistake fails in under a minute: +# Five jobs, cheapest first, so an obvious mistake fails in under a minute: # # lint static checks, no interpreter matrix, no database # unit the database-free tests on every supported Python # database the full suite against real PostgreSQL service containers # package sdist/wheel build, twine metadata check, install smoke tests +# docs the Sphinx build with warnings as errors, no database # # The downstream regression suites (LMFDB and seminars) live in a separate # workflow, downstream.yml, because they are an order of magnitude slower. @@ -200,3 +201,26 @@ jobs: with: name: dist path: dist/ + + docs: + name: Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + + # The package itself is installed so that autodoc can import it; no + # database is needed (importing psycodict does not connect). + - name: Install package and documentation dependencies + run: | + python -m pip install '.[pgbinary]' -r docs/requirements.txt + + # -W turns warnings into errors: a broken cross-reference or a + # malformed docstring fails here rather than silently degrading the + # site. Read the Docs builds with fail_on_warning to match. + - name: Build documentation + run: python -m sphinx -W --keep-going -b html docs docs/_build/html diff --git a/.gitignore b/.gitignore index b8bda12..f0d2d32 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,10 @@ dmypy.json *~ # Secrets file: should not be committed to repo! -secrets.ini \ No newline at end of file +secrets.ini +# Guides copied into docs/ at build time by docs/conf.py +docs/README.md +docs/QueryLanguage.md +docs/Searching.md +docs/DataManagement.md +docs/MetadataFormats.md diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..bd7cdf1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,27 @@ +# Read the Docs build configuration for the psycodict documentation. +# +# One manual step activates this: import the repository at +# https://readthedocs.org/dashboard/import/ (the psycodict GitHub App +# integration handles builds on push and pull-request previews from then +# on). Once 1.0 is tagged, activate the tag under Versions so that +# /stable/ serves the release while /latest/ tracks master. +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/conf.py + # The same contract as the docs job in ci.yml: a broken cross-reference + # or malformed docstring fails the build rather than degrading the site. + fail_on_warning: true + +python: + install: + # The package itself, so that autodoc can import it. + - method: pip + path: . + extra_requirements: [pgbinary] + - requirements: docs/requirements.txt diff --git a/docs/api/base.md b/docs/api/base.md new file mode 100644 index 0000000..91cc04b --- /dev/null +++ b/docs/api/base.md @@ -0,0 +1,5 @@ +# psycodict.base + +```{eval-rst} +.. automodule:: psycodict.base +``` diff --git a/docs/api/config.md b/docs/api/config.md new file mode 100644 index 0000000..c094562 --- /dev/null +++ b/docs/api/config.md @@ -0,0 +1,5 @@ +# psycodict.config + +```{eval-rst} +.. automodule:: psycodict.config +``` diff --git a/docs/api/database.md b/docs/api/database.md new file mode 100644 index 0000000..ac65834 --- /dev/null +++ b/docs/api/database.md @@ -0,0 +1,5 @@ +# psycodict.database + +```{eval-rst} +.. automodule:: psycodict.database +``` diff --git a/docs/api/dbdiff.md b/docs/api/dbdiff.md new file mode 100644 index 0000000..c80083e --- /dev/null +++ b/docs/api/dbdiff.md @@ -0,0 +1,5 @@ +# psycodict.dbdiff + +```{eval-rst} +.. automodule:: psycodict.dbdiff +``` diff --git a/docs/api/encoding.md b/docs/api/encoding.md new file mode 100644 index 0000000..a0c113b --- /dev/null +++ b/docs/api/encoding.md @@ -0,0 +1,5 @@ +# psycodict.encoding + +```{eval-rst} +.. automodule:: psycodict.encoding +``` diff --git a/docs/api/index.md b/docs/api/index.md new file mode 100644 index 0000000..70bfa38 --- /dev/null +++ b/docs/api/index.md @@ -0,0 +1,44 @@ +# API reference + +Generated from the docstrings. The map of the library: + +- {mod}`psycodict.database` — {class}`~psycodict.database.PostgresDatabase`, + the connection object; each table in the database is an attribute of it. +- {mod}`psycodict.searchtable` — + {class}`~psycodict.searchtable.PostgresSearchTable`, the read API + (`search`, `lucky`, `lookup`, `count`, `random`, …) driven by the query + dictionaries of [the query language](../QueryLanguage.md). +- {mod}`psycodict.table` — {class}`~psycodict.table.PostgresTable`, the + write and schema half (row writes, bulk file import/export, reloads, + indexes, columns); base class of the search table. +- {mod}`psycodict.statstable` — + {class}`~psycodict.statstable.PostgresStatsTable`, precomputed counts and + statistics, available as `table.stats`. +- {mod}`psycodict.encoding` — value conversion between Python and + PostgreSQL, including the file formats used by the bulk operations. +- {mod}`psycodict.config` — configuration discovery and parsing + (`config.ini`, command-line arguments, `$PSYCODICT_CONFIG`). +- {mod}`psycodict.utils` — {class}`~psycodict.utils.DelayCommit` and other + helpers shared across the library. +- {mod}`psycodict.notifications` — LISTEN/NOTIFY support: schema-change + announcements and a small publish/subscribe primitive. +- {mod}`psycodict.dbdiff` — comparing the contents of two databases. +- {mod}`psycodict.slowlog` — parsing and reporting on the slow-query log. +- {mod}`psycodict.base` — {class}`~psycodict.base.PostgresBase`, the shared + execution and logging plumbing underneath everything else. + +```{toctree} +:maxdepth: 1 + +database +searchtable +table +statstable +encoding +config +utils +notifications +dbdiff +slowlog +base +``` diff --git a/docs/api/notifications.md b/docs/api/notifications.md new file mode 100644 index 0000000..e3897ec --- /dev/null +++ b/docs/api/notifications.md @@ -0,0 +1,5 @@ +# psycodict.notifications + +```{eval-rst} +.. automodule:: psycodict.notifications +``` diff --git a/docs/api/searchtable.md b/docs/api/searchtable.md new file mode 100644 index 0000000..81c4e29 --- /dev/null +++ b/docs/api/searchtable.md @@ -0,0 +1,5 @@ +# psycodict.searchtable + +```{eval-rst} +.. automodule:: psycodict.searchtable +``` diff --git a/docs/api/slowlog.md b/docs/api/slowlog.md new file mode 100644 index 0000000..f853cd2 --- /dev/null +++ b/docs/api/slowlog.md @@ -0,0 +1,5 @@ +# psycodict.slowlog + +```{eval-rst} +.. automodule:: psycodict.slowlog +``` diff --git a/docs/api/statstable.md b/docs/api/statstable.md new file mode 100644 index 0000000..6223872 --- /dev/null +++ b/docs/api/statstable.md @@ -0,0 +1,5 @@ +# psycodict.statstable + +```{eval-rst} +.. automodule:: psycodict.statstable +``` diff --git a/docs/api/table.md b/docs/api/table.md new file mode 100644 index 0000000..d6cf4e1 --- /dev/null +++ b/docs/api/table.md @@ -0,0 +1,5 @@ +# psycodict.table + +```{eval-rst} +.. automodule:: psycodict.table +``` diff --git a/docs/api/utils.md b/docs/api/utils.md new file mode 100644 index 0000000..e643945 --- /dev/null +++ b/docs/api/utils.md @@ -0,0 +1,5 @@ +# psycodict.utils + +```{eval-rst} +.. automodule:: psycodict.utils +``` diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..a2efdb1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,66 @@ +# Configuration for the psycodict documentation build (Sphinx). +# +# Build locally with +# +# pip install '.[pgbinary]' -r docs/requirements.txt +# python -m sphinx -W --keep-going -b html docs docs/_build/html +# +# The same build runs in CI (the docs job of ci.yml) and on Read the Docs +# (.readthedocs.yaml), with warnings treated as errors in both. +import shutil +from importlib.metadata import version as _dist_version +from pathlib import Path + +_DOCS = Path(__file__).resolve().parent +_ROOT = _DOCS.parent + +# The canonical copies of the narrative guides live at the repository root, +# where GitHub renders them; they are copied here at build time so that the +# site includes them and their relative links to one another keep working. +# The copies are ignored by git (see .gitignore). +_GUIDES = [ + "README.md", + "QueryLanguage.md", + "Searching.md", + "DataManagement.md", + "MetadataFormats.md", +] +for _name in _GUIDES: + shutil.copyfile(_ROOT / _name, _DOCS / _name) + +project = "psycodict" +author = "David Roe and Edgar Costa" +copyright = "2019–2026, David Roe and Edgar Costa" +release = _dist_version("psycodict") +version = release + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", +] + +# Generate GitHub-style anchors for headings so that intra-page links in the +# Markdown guides ([`search`](#search) and friends) keep working on the site. +myst_heading_anchors = 4 + +# The API reference is generated from the docstrings, which follow the Sage +# documentation conventions (INPUT:/OUTPUT: bullet blocks, EXAMPLES:: with +# literal transcripts); those are plain reST, so autodoc renders them as-is. +autodoc_member_order = "bysource" +autodoc_default_options = { + "members": True, + "undoc-members": True, + "show-inheritance": True, +} + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "psycopg": ("https://www.psycopg.org/psycopg3/docs", None), +} + +exclude_patterns = ["_build"] + +html_theme = "furo" +html_title = f"psycodict {release}" diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..b8ea130 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,28 @@ +# psycodict + +psycodict is a dictionary-based Python interface to PostgreSQL databases: +queries are Python dictionaries, results come back as Python values, and the +metadata that makes this ergonomic — column types, sort orders, statistics — +is managed for you. It was extracted from the +[LMFDB](https://www.lmfdb.org) and powers several mathematical databases. + +The guides below describe the system top-down; the API reference is +generated from the docstrings. + +```{toctree} +:maxdepth: 2 +:caption: Guides + +Getting started +QueryLanguage +Searching +DataManagement +MetadataFormats +``` + +```{toctree} +:maxdepth: 2 +:caption: Reference + +api/index +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..9ae7ce7 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +# Documentation build dependencies. Used by the docs job in +# .github/workflows/ci.yml and by Read the Docs (.readthedocs.yaml); +# the package itself (with a psycopg extra) must also be installed so +# that autodoc can import it. +sphinx>=7 +myst-parser>=3 +furo>=2024.1.29 From 24a7d3e9e9c96a23831609ab03fb83280395d70b Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 22 Jul 2026 03:02:43 -0400 Subject: [PATCH 3/4] Include Versioning.md in the documentation site Versioning.md (from the versioning-policy branch, merged here so this builds standalone in either merge order) is linked from the README, so the site must carry it: without the copy and the toctree entry the combined tree failed sphinx -W with myst.xref_missing on the README link. Its own links (QueryLanguage, Searching, DataManagement, MetadataFormats) all point at guides already on the site. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + docs/conf.py | 1 + docs/index.md | 1 + 3 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f0d2d32..c2a775e 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ docs/QueryLanguage.md docs/Searching.md docs/DataManagement.md docs/MetadataFormats.md +docs/Versioning.md diff --git a/docs/conf.py b/docs/conf.py index a2efdb1..925b0e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,6 +24,7 @@ "Searching.md", "DataManagement.md", "MetadataFormats.md", + "Versioning.md", ] for _name in _GUIDES: shutil.copyfile(_ROOT / _name, _DOCS / _name) diff --git a/docs/index.md b/docs/index.md index b8ea130..7a2bd2e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,6 +18,7 @@ QueryLanguage Searching DataManagement MetadataFormats +Versioning ``` ```{toctree} From 4955ad29da5afa2685821584ec519c09613acd26 Mon Sep 17 00:00:00 2001 From: David Roe Date: Wed, 22 Jul 2026 03:08:53 -0400 Subject: [PATCH 4/4] Carry CONTRIBUTING.md and SECURITY.md on the documentation site Nothing on the site links to them yet, so this is not fixing a build break -- it is preventing the next one: the moment a copied guide gains a "see CONTRIBUTING.md" line, sphinx -W fails with myst.xref_missing exactly as the README's Versioning.md link did. Beyond the tripwire, CONTRIBUTING.md was written around this site (its Documentation section gives the sphinx -W build command) and links SECURITY.md plus all four spec guides, so on the site those links now go live. They sit under their own Project caption rather than among the user guides. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 ++ docs/conf.py | 2 ++ docs/index.md | 8 ++++++++ 3 files changed, 12 insertions(+) diff --git a/.gitignore b/.gitignore index 235ddab..caa045a 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,5 @@ docs/Searching.md docs/DataManagement.md docs/MetadataFormats.md docs/Versioning.md +docs/CONTRIBUTING.md +docs/SECURITY.md diff --git a/docs/conf.py b/docs/conf.py index 925b0e8..a0d81a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,6 +25,8 @@ "DataManagement.md", "MetadataFormats.md", "Versioning.md", + "CONTRIBUTING.md", + "SECURITY.md", ] for _name in _GUIDES: shutil.copyfile(_ROOT / _name, _DOCS / _name) diff --git a/docs/index.md b/docs/index.md index 7a2bd2e..33536dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -27,3 +27,11 @@ Versioning api/index ``` + +```{toctree} +:maxdepth: 1 +:caption: Project + +Contributing +Security policy +```