diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08b53fb..18d35ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,12 @@ # Continuous integration for psycodict. # -# Four jobs, cheapest first, so an obvious mistake fails in under a minute: +# Five jobs, cheapest first, so an obvious mistake fails in under a minute: # # lint static checks, no interpreter matrix, no database # unit the database-free tests on every supported Python # database the full suite against real PostgreSQL service containers # package sdist/wheel build, twine metadata check, install smoke tests +# docs the Sphinx build with warnings as errors, no database # # The downstream regression suites (LMFDB and seminars) live in a separate # workflow, downstream.yml, because they are an order of magnitude slower. @@ -200,3 +201,26 @@ jobs: with: name: dist path: dist/ + + docs: + name: Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + + # The package itself is installed so that autodoc can import it; no + # database is needed (importing psycodict does not connect). + - name: Install package and documentation dependencies + run: | + python -m pip install '.[pgbinary]' -r docs/requirements.txt + + # -W turns warnings into errors: a broken cross-reference or a + # malformed docstring fails here rather than silently degrading the + # site. Read the Docs builds with fail_on_warning to match. + - name: Build documentation + run: python -m sphinx -W --keep-going -b html docs docs/_build/html diff --git a/.gitignore b/.gitignore index 41c3653..caa045a 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,12 @@ secrets.ini # Local database configuration (may contain a password): copy # config.ini.example to config.ini and edit. Should not be committed. config.ini +# Guides copied into docs/ at build time by docs/conf.py +docs/README.md +docs/QueryLanguage.md +docs/Searching.md +docs/DataManagement.md +docs/MetadataFormats.md +docs/Versioning.md +docs/CONTRIBUTING.md +docs/SECURITY.md diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..bd7cdf1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,27 @@ +# Read the Docs build configuration for the psycodict documentation. +# +# One manual step activates this: import the repository at +# https://readthedocs.org/dashboard/import/ (the psycodict GitHub App +# integration handles builds on push and pull-request previews from then +# on). Once 1.0 is tagged, activate the tag under Versions so that +# /stable/ serves the release while /latest/ tracks master. +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/conf.py + # The same contract as the docs job in ci.yml: a broken cross-reference + # or malformed docstring fails the build rather than degrading the site. + fail_on_warning: true + +python: + install: + # The package itself, so that autodoc can import it. + - method: pip + path: . + extra_requirements: [pgbinary] + - requirements: docs/requirements.txt diff --git a/DataManagement.md b/DataManagement.md index 76c17ff..1999990 100644 --- a/DataManagement.md +++ b/DataManagement.md @@ -1,3 +1,4 @@ +# Data management ## Introduction diff --git a/QueryLanguage.md b/QueryLanguage.md index ea1d9fb..d865999 100644 --- a/QueryLanguage.md +++ b/QueryLanguage.md @@ -1,3 +1,4 @@ +# The query language ## Introduction diff --git a/Searching.md b/Searching.md index 391825c..2ea0fb0 100644 --- a/Searching.md +++ b/Searching.md @@ -30,7 +30,7 @@ database and a table variable such as `nf = db.nf_fields`. - [Stability](#stability) - [`search`](#search) - - [Parameters](#search-parameters) + - [Parameters](#parameters) - [Return type: iterator vs. list](#return-type-iterator-vs-list) - [Projections](#projections) - [Sorting](#sorting) diff --git a/docs/api/base.md b/docs/api/base.md new file mode 100644 index 0000000..91cc04b --- /dev/null +++ b/docs/api/base.md @@ -0,0 +1,5 @@ +# psycodict.base + +```{eval-rst} +.. automodule:: psycodict.base +``` diff --git a/docs/api/config.md b/docs/api/config.md new file mode 100644 index 0000000..c094562 --- /dev/null +++ b/docs/api/config.md @@ -0,0 +1,5 @@ +# psycodict.config + +```{eval-rst} +.. automodule:: psycodict.config +``` diff --git a/docs/api/database.md b/docs/api/database.md new file mode 100644 index 0000000..ac65834 --- /dev/null +++ b/docs/api/database.md @@ -0,0 +1,5 @@ +# psycodict.database + +```{eval-rst} +.. automodule:: psycodict.database +``` diff --git a/docs/api/dbdiff.md b/docs/api/dbdiff.md new file mode 100644 index 0000000..c80083e --- /dev/null +++ b/docs/api/dbdiff.md @@ -0,0 +1,5 @@ +# psycodict.dbdiff + +```{eval-rst} +.. automodule:: psycodict.dbdiff +``` diff --git a/docs/api/encoding.md b/docs/api/encoding.md new file mode 100644 index 0000000..a0c113b --- /dev/null +++ b/docs/api/encoding.md @@ -0,0 +1,5 @@ +# psycodict.encoding + +```{eval-rst} +.. automodule:: psycodict.encoding +``` diff --git a/docs/api/index.md b/docs/api/index.md new file mode 100644 index 0000000..70bfa38 --- /dev/null +++ b/docs/api/index.md @@ -0,0 +1,44 @@ +# API reference + +Generated from the docstrings. The map of the library: + +- {mod}`psycodict.database` — {class}`~psycodict.database.PostgresDatabase`, + the connection object; each table in the database is an attribute of it. +- {mod}`psycodict.searchtable` — + {class}`~psycodict.searchtable.PostgresSearchTable`, the read API + (`search`, `lucky`, `lookup`, `count`, `random`, …) driven by the query + dictionaries of [the query language](../QueryLanguage.md). +- {mod}`psycodict.table` — {class}`~psycodict.table.PostgresTable`, the + write and schema half (row writes, bulk file import/export, reloads, + indexes, columns); base class of the search table. +- {mod}`psycodict.statstable` — + {class}`~psycodict.statstable.PostgresStatsTable`, precomputed counts and + statistics, available as `table.stats`. +- {mod}`psycodict.encoding` — value conversion between Python and + PostgreSQL, including the file formats used by the bulk operations. +- {mod}`psycodict.config` — configuration discovery and parsing + (`config.ini`, command-line arguments, `$PSYCODICT_CONFIG`). +- {mod}`psycodict.utils` — {class}`~psycodict.utils.DelayCommit` and other + helpers shared across the library. +- {mod}`psycodict.notifications` — LISTEN/NOTIFY support: schema-change + announcements and a small publish/subscribe primitive. +- {mod}`psycodict.dbdiff` — comparing the contents of two databases. +- {mod}`psycodict.slowlog` — parsing and reporting on the slow-query log. +- {mod}`psycodict.base` — {class}`~psycodict.base.PostgresBase`, the shared + execution and logging plumbing underneath everything else. + +```{toctree} +:maxdepth: 1 + +database +searchtable +table +statstable +encoding +config +utils +notifications +dbdiff +slowlog +base +``` diff --git a/docs/api/notifications.md b/docs/api/notifications.md new file mode 100644 index 0000000..e3897ec --- /dev/null +++ b/docs/api/notifications.md @@ -0,0 +1,5 @@ +# psycodict.notifications + +```{eval-rst} +.. automodule:: psycodict.notifications +``` diff --git a/docs/api/searchtable.md b/docs/api/searchtable.md new file mode 100644 index 0000000..81c4e29 --- /dev/null +++ b/docs/api/searchtable.md @@ -0,0 +1,5 @@ +# psycodict.searchtable + +```{eval-rst} +.. automodule:: psycodict.searchtable +``` diff --git a/docs/api/slowlog.md b/docs/api/slowlog.md new file mode 100644 index 0000000..f853cd2 --- /dev/null +++ b/docs/api/slowlog.md @@ -0,0 +1,5 @@ +# psycodict.slowlog + +```{eval-rst} +.. automodule:: psycodict.slowlog +``` diff --git a/docs/api/statstable.md b/docs/api/statstable.md new file mode 100644 index 0000000..6223872 --- /dev/null +++ b/docs/api/statstable.md @@ -0,0 +1,5 @@ +# psycodict.statstable + +```{eval-rst} +.. automodule:: psycodict.statstable +``` diff --git a/docs/api/table.md b/docs/api/table.md new file mode 100644 index 0000000..d6cf4e1 --- /dev/null +++ b/docs/api/table.md @@ -0,0 +1,5 @@ +# psycodict.table + +```{eval-rst} +.. automodule:: psycodict.table +``` diff --git a/docs/api/utils.md b/docs/api/utils.md new file mode 100644 index 0000000..e643945 --- /dev/null +++ b/docs/api/utils.md @@ -0,0 +1,5 @@ +# psycodict.utils + +```{eval-rst} +.. automodule:: psycodict.utils +``` diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..a0d81a1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,69 @@ +# Configuration for the psycodict documentation build (Sphinx). +# +# Build locally with +# +# pip install '.[pgbinary]' -r docs/requirements.txt +# python -m sphinx -W --keep-going -b html docs docs/_build/html +# +# The same build runs in CI (the docs job of ci.yml) and on Read the Docs +# (.readthedocs.yaml), with warnings treated as errors in both. +import shutil +from importlib.metadata import version as _dist_version +from pathlib import Path + +_DOCS = Path(__file__).resolve().parent +_ROOT = _DOCS.parent + +# The canonical copies of the narrative guides live at the repository root, +# where GitHub renders them; they are copied here at build time so that the +# site includes them and their relative links to one another keep working. +# The copies are ignored by git (see .gitignore). +_GUIDES = [ + "README.md", + "QueryLanguage.md", + "Searching.md", + "DataManagement.md", + "MetadataFormats.md", + "Versioning.md", + "CONTRIBUTING.md", + "SECURITY.md", +] +for _name in _GUIDES: + shutil.copyfile(_ROOT / _name, _DOCS / _name) + +project = "psycodict" +author = "David Roe and Edgar Costa" +copyright = "2019–2026, David Roe and Edgar Costa" +release = _dist_version("psycodict") +version = release + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", +] + +# Generate GitHub-style anchors for headings so that intra-page links in the +# Markdown guides ([`search`](#search) and friends) keep working on the site. +myst_heading_anchors = 4 + +# The API reference is generated from the docstrings, which follow the Sage +# documentation conventions (INPUT:/OUTPUT: bullet blocks, EXAMPLES:: with +# literal transcripts); those are plain reST, so autodoc renders them as-is. +autodoc_member_order = "bysource" +autodoc_default_options = { + "members": True, + "undoc-members": True, + "show-inheritance": True, +} + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "psycopg": ("https://www.psycopg.org/psycopg3/docs", None), +} + +exclude_patterns = ["_build"] + +html_theme = "furo" +html_title = f"psycodict {release}" diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..33536dd --- /dev/null +++ b/docs/index.md @@ -0,0 +1,37 @@ +# psycodict + +psycodict is a dictionary-based Python interface to PostgreSQL databases: +queries are Python dictionaries, results come back as Python values, and the +metadata that makes this ergonomic — column types, sort orders, statistics — +is managed for you. It was extracted from the +[LMFDB](https://www.lmfdb.org) and powers several mathematical databases. + +The guides below describe the system top-down; the API reference is +generated from the docstrings. + +```{toctree} +:maxdepth: 2 +:caption: Guides + +Getting started +QueryLanguage +Searching +DataManagement +MetadataFormats +Versioning +``` + +```{toctree} +:maxdepth: 2 +:caption: Reference + +api/index +``` + +```{toctree} +:maxdepth: 1 +:caption: Project + +Contributing +Security policy +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..9ae7ce7 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +# Documentation build dependencies. Used by the docs job in +# .github/workflows/ci.yml and by Read the Docs (.readthedocs.yaml); +# the package itself (with a psycopg extra) must also be installed so +# that autodoc can import it. +sphinx>=7 +myst-parser>=3 +furo>=2024.1.29 diff --git a/psycodict/notifications.py b/psycodict/notifications.py index d5cff74..6152601 100644 --- a/psycodict/notifications.py +++ b/psycodict/notifications.py @@ -149,7 +149,8 @@ class NotificationListener: - ``config`` -- a :class:`~psycodict.config.Configuration`; the ``postgresql`` options are used to open the dedicated connection - ``channels`` -- a channel name, or an iterable of them (default: - ``("psycodict_schema",)``); each is validated and ``LISTEN``ed on + ``("psycodict_schema",)``); each is validated and subscribed with + ``LISTEN`` - ``**connect_kwargs`` -- extra keyword arguments passed on to ``psycopg.connect`` (e.g. keepalive settings), overriding the configuration where they overlap diff --git a/psycodict/searchtable.py b/psycodict/searchtable.py index a7f71b5..3c9b75a 100644 --- a/psycodict/searchtable.py +++ b/psycodict/searchtable.py @@ -906,27 +906,27 @@ def lucky(self, query={}, projection=2, offset=0, sort=[], raw=None, raw_values= INPUT: - ``query`` -- a mongo-style dictionary specifying the query. - Generally, the keys will correspond to columns, - and values will either be specific numbers (specifying an equality test) - or dictionaries giving more complicated constraints. - The main exception is that "$or" can be a top level key, - specifying a list of constraints of which at least one must be true. + Generally, the keys will correspond to columns, + and values will either be specific numbers (specifying an equality test) + or dictionaries giving more complicated constraints. + The main exception is that "$or" can be a top level key, + specifying a list of constraints of which at least one must be true. - ``projection`` -- which columns are desired. This can be specified either as a list of columns to include; - a dictionary specifying columns to include (using all True values) - or exclude (using all False values); - a string giving a single column (only returns the value, not a dictionary); - or an integer code (0 means only return the label, - 1 means return all search columns, - 2 means all columns (default)). + a dictionary specifying columns to include (using all True values) + or exclude (using all False values); + a string giving a single column (only returns the value, not a dictionary); + or an integer code (0 means only return the label, + 1 means return all search columns, 2 means all columns (default)). - ``offset`` -- integer. allows retrieval of a later record rather than just first. - ``sort`` -- The sort order, from which the first result is returned. - - None, Using the default sort order for the table - - a list of strings (which are interpreted as column names in the - ascending direction) or of pairs (column name, 1 or -1). - If not specified, will use the default sort order on the table. - - [] (default), unsorted, thus if there is more than one match to - the query then the choice of the result is arbitrary. + + - None, Using the default sort order for the table + - a list of strings (which are interpreted as column names in the + ascending direction) or of pairs (column name, 1 or -1). + If not specified, will use the default sort order on the table. + - [] (default), unsorted, thus if there is more than one match to + the query then the choice of the result is arbitrary. - ``raw`` -- a string, to be used as the WHERE part of the query. DO NOT USE THIS DIRECTLY FOR INPUT FROM WEBSITE. - ``raw_values`` -- a list of values to be substituted for %s entries in the raw string. Useful when strings might include quotes. - ``join`` -- a list of tuples describing other search tables to join to this one, @@ -1003,19 +1003,18 @@ def search( INPUT: - ``query`` -- a mongo-style dictionary specifying the query. - Generally, the keys will correspond to columns, - and values will either be specific numbers (specifying an equality test) - or dictionaries giving more complicated constraints. - The main exception is that "$or" can be a top level key, - specifying a list of constraints of which at least one must be true. + Generally, the keys will correspond to columns, + and values will either be specific numbers (specifying an equality test) + or dictionaries giving more complicated constraints. + The main exception is that "$or" can be a top level key, + specifying a list of constraints of which at least one must be true. - ``projection`` -- which columns are desired. This can be specified either as a list of columns to include; - a dictionary specifying columns to include (using all True values) - or exclude (using all False values); - a string giving a single column (only returns the value, not a dictionary); - or an integer code (0 means only return the label, - 1 means return all search columns (default), - 2 means all columns). + a dictionary specifying columns to include (using all True values) + or exclude (using all False values); + a string giving a single column (only returns the value, not a dictionary); + or an integer code (0 means only return the label, + 1 means return all search columns (default), 2 means all columns). - ``limit`` -- an integer or None (default), giving the maximum number of records to return. - ``offset`` -- a nonnegative integer (default 0), where to start in the list of results. - ``sort`` -- a sort order. Either None or a list of strings (which are interpreted as column names in the ascending direction) or of pairs (column name, 1 or -1). If not specified, will use the default sort order on the table. If you want the result unsorted, use [].