Polish docstrings for the documentation site: fix silent mis-rendering, fill missing docstrings, add module intros

psycodict/base.py

@@ -1,4 +1,16 @@
 # -*- coding: utf-8 -*-
+"""
+The shared plumbing underneath every psycodict object.
+
+:class:`PostgresBase` is the common base of the database, table and
+statistics classes; it owns statement execution through ``_execute``
+(logging, slow-query warnings, commit/rollback bookkeeping and
+reconnection) together with helpers for inspecting tables, indexes and
+constraints.  The module also defines the layout of the ``meta_*`` tables --
+the column lists, types and creation statements shared by everything that
+reads or writes them -- and the metadata format version (``META_FORMAT``)
+stamped into ``meta_format``.
+"""
 import csv
 import logging
 import re
@@ -111,6 +123,16 @@
 
 
 def jsonb_idx(cols, cols_type):
+    """
+    The positions in ``cols`` whose type is ``jsonb``, as a tuple of
+    indexes.  Used to decide which values need json decoding when reading
+    rows of the ``meta_*`` tables.
+
+    INPUT:
+
+    - ``cols`` -- a list of column names
+    - ``cols_type`` -- a dictionary mapping column names to their types
+    """
     return tuple(i for i, elt in enumerate(cols) if cols_type[elt] == "jsonb")
 
 
@@ -629,7 +651,7 @@ def _constraint_exists(self, constraintname, tablename=None):
 
     def _list_indexes(self, tablename):
         """
-        Lists built index names on the search table `tablename`
+        Lists built index names on the search table ``tablename``
         """
         cur = self._execute(
             SQL("SELECT indexname FROM pg_indexes WHERE tablename = %s"),
@@ -640,7 +662,7 @@ def _list_indexes(self, tablename):
 
     def _list_constraints(self, tablename):
         """
-        Lists constraint names on the search table `tablename`
+        Lists constraint names on the search table ``tablename``
         """
         # if we look into information_schema.table_constraints
         # we also get internal constraints, I'm not sure why

psycodict/config.py

@@ -1,4 +1,16 @@
 # -*- coding: utf-8 -*-
+"""
+Configuration handling for psycodict.
+
+:class:`Configuration` merges a ``config.ini`` file with command-line
+arguments (off by default; see ``readargs``) and hands the result to
+:class:`~psycodict.database.PostgresDatabase`.  When no file is named
+explicitly, :func:`find_config_file` discovers one: the
+``PSYCODICT_CONFIG`` environment variable, then ``config.ini`` in the
+current directory if it already exists, then ``~/.psycodict/config.ini``,
+which is created on first use.  A ``secrets.ini`` next to the configuration
+file overrides its values, so credentials can be kept out of it.
+"""
 import os
 import argparse
 from configparser import ConfigParser
@@ -362,6 +374,11 @@ def _default_log_dir(config_file):
         return None
 
     def get_postgresql_default(self):
+        """
+        The built-in default connection options (host, port, user, ...), as
+        a dictionary -- the values used before the configuration file and
+        command line are consulted.
+        """
         res = dict(self.default_args["postgresql"])
         res["port"] = int(res["port"])
         return res

psycodict/database.py

@@ -1,4 +1,17 @@
 # -*- coding: utf-8 -*-
+"""
+The connection object at the center of psycodict.
+
+:class:`PostgresDatabase` connects using a
+:class:`~psycodict.config.Configuration` and exposes each table in the
+database as an attribute (``db.nf_fields``), a
+:class:`~psycodict.searchtable.PostgresSearchTable`.  It also provides the
+database-wide operations: creating, dropping and renaming tables,
+reloading or cleaning up every table at once, schema-change notifications
+(:meth:`~PostgresDatabase.listener`), refreshing table metadata without a
+restart (:meth:`~PostgresDatabase.refresh_tables`), and the metadata
+format stamp and its migrations.
+"""
 import csv
 import logging
 from pathlib import Path
@@ -78,10 +91,20 @@ class NumericLoader(Loader):
     (Sage Integer/RealLiteral when Sage is available, int/float otherwise).
     """
     def load(self, data):
+        """
+        Convert the numeric's text representation to a Python/Sage number.
+        """
         return numeric_converter(bytes(data).decode())
 
 
 def setup_connection(conn):
+    """
+    Prepare a fresh psycopg connection for psycodict: set the client
+    encoding and register the loaders and dumpers that implement
+    psycodict's value conversion (numerics, json, arrays, and -- when Sage
+    is available -- Sage integers and reals).  Called for every connection
+    the database opens.
+    """
     # psycopg3 uses unicode (str) everywhere by default, so the psycopg2-era
     # UNICODE/UNICODEARRAY registrations are no longer needed.
     conn.execute("SET client_encoding TO 'UTF8'")
@@ -1053,7 +1076,7 @@ def create_table(
         include_nones=True,
     ):
         """
-        Add a new search table to the database.  See also `create_table_like`.
+        Add a new search table to the database.  See also :meth:`create_table_like`.
 
         INPUT:
 
@@ -1450,24 +1473,21 @@ def reload_all(
     ):
         """
         Reloads all tables from files in a given folder.  The filenames must match
-        the names of the tables, with `_counts` and `_stats` appended as appropriate.
+        the names of the tables, with ``_counts`` and ``_stats`` appended as appropriate.
 
         INPUT:
 
-            - ``data_folder`` -- the folder that contains files to be reloaded
-            - ``halt_on_errors`` -- whether to stop if a DatabaseError is
-                encountered while trying to reload one of the tables
-            - ``sequential_swap`` -- if True, then the whole transaction will not be wrapped in a DelayCommit, which can sometimes prevent deadlocks.
-
-        INPUTS passed to `reload` function in `PostgresTable`:
-
-                - ``resort``, ``restat``, ``adjust_schema``, and any extra keywords
-
-
+        - ``data_folder`` -- the folder that contains files to be reloaded
+        - ``halt_on_errors`` -- whether to stop if a DatabaseError is
+          encountered while trying to reload one of the tables
+        - ``sequential_swap`` -- if True, then the whole transaction will not
+          be wrapped in a DelayCommit, which can sometimes prevent deadlocks
+        - ``resort``, ``restat``, ``adjust_schema``, and any extra keywords
+          are passed on to the ``reload`` method of each
+          :class:`~psycodict.table.PostgresTable`
 
         Note that this function currently does not reload data that is not in a
         search table, such as knowls or user data.
-
         """
         data_folder = Path(data_folder)
 
@@ -1639,7 +1659,7 @@ def reload_all_revert(self, data_folder):
 
     def cleanup_all(self):
         """
-        Drops all `_tmp` and `_old` tables created by the reload() method.
+        Drops all ``_tmp`` and ``_old`` tables created by the reload() method.
         """
         with DelayCommit(self, silence=True):
             for tablename in self.tablenames:

psycodict/encoding.py

@@ -77,10 +77,17 @@ def __str__(self):
             return self.literal
 
     class RealEncoder():
+        """
+        A wrapper rendering a Sage RealNumber/RealLiteral as its literal
+        text (the exact decimal string it was created from, when available).
+        """
         def __init__(self, value):
             self._value = value
 
         def getquoted(self):
+            """
+            The wrapped real number's literal text.
+            """
             if isinstance(self._value, RealLiteral):
                 return self._value.literal
             else:
@@ -96,6 +103,9 @@ class RealLiteralDumper(Dumper):
         as psycopg2's client-side interpolation did.
         """
         def dump(self, obj):
+            """
+            Render the real number as its literal text.
+            """
             if isinstance(obj, RealLiteral):
                 return obj.literal.encode()
             return str(obj).encode()
@@ -105,6 +115,9 @@ class SageIntegerDumper(Dumper):
         Dumps a Sage Integer as its decimal text (replacing psycopg2's AsIs).
         """
         def dump(self, obj):
+            """
+            Render the integer as decimal text.
+            """
             return str(obj).encode()
 
 
@@ -180,6 +193,9 @@ def __init__(self, seq):
         self._seq = seq
 
     def getquoted(self):
+        """
+        The Postgres array literal for the wrapped sequence, as bytes.
+        """
         return _pg_array_literal(self._seq).encode()
 
     def __str__(self):
@@ -213,6 +229,9 @@ class ArrayDumper(Dumper):
     Dumps an Array wrapper as a Postgres array literal with unknown oid.
     """
     def dump(self, obj):
+        """
+        Render the wrapped sequence as a Postgres array literal.
+        """
         return _pg_array_literal(obj._seq).encode()
 
 
@@ -236,10 +255,18 @@ def __repr__(self):
 
     @classmethod
     def dumps(cls, obj):
+        """
+        Serialize ``obj`` to json text using the extended encoding
+        (see :meth:`prep`).
+        """
         return json.dumps(cls.prep(obj))
 
     @classmethod
     def loads(cls, s):
+        """
+        Parse json text and decode the extended encoding back to Python
+        and Sage objects (see :meth:`extract`).
+        """
         return cls.extract(json.loads(s))
 
     @classmethod
@@ -514,6 +541,9 @@ class JsonWrapperDumper(Dumper):
     context (this works for both json and jsonb columns).
     """
     def dump(self, obj):
+        """
+        Render the wrapped value as json text.
+        """
         return Json.dumps(obj.obj).encode()
 
 
@@ -523,6 +553,9 @@ class DictJsonDumper(Dumper):
     ``register_adapter(dict, Json)``).
     """
     def dump(self, obj):
+        """
+        Render the dict as json text.
+        """
         return Json.dumps(obj).encode()
 
 

psycodict/searchtable.py

@@ -1,4 +1,14 @@
 # -*- coding: utf-8 -*-
+"""
+The read half of a table: dictionary queries.
+
+:class:`PostgresSearchTable` extends
+:class:`~psycodict.table.PostgresTable` with the query API -- ``search``,
+``lucky``, ``lookup``, ``count``, ``random`` and friends -- translating
+query dictionaries into SQL SELECT statements, with projections, sorting,
+joins and the ``info`` contract used by search pages.  The query language
+is specified in QueryLanguage.md and the read API in Searching.md.
+"""
 import random
 import time
 from itertools import islice
@@ -33,6 +43,25 @@ def _qualify(frag, tablename):
 
 
 class PostgresSearchTable(PostgresTable):
+    """
+    A single search table, as returned by ``db.<tablename>``: the interface
+    for reading data with dictionary queries.
+
+    The read methods -- :meth:`search`, :meth:`lucky`, :meth:`lookup`,
+    :meth:`exists`, :meth:`count`, :meth:`random`, :meth:`distinct` and
+    friends -- accept a query dictionary such as
+    ``{"degree": 2, "disc": {"$lt": 0}}``, translate it into SQL, and
+    return rows as dictionaries (or bare values, under a string
+    projection).  The query language is specified in QueryLanguage.md and
+    the read API in Searching.md.  Writing and schema changes live on the
+    base class :class:`~psycodict.table.PostgresTable`, and precomputed
+    statistics on :class:`~psycodict.statstable.PostgresStatsTable`
+    (available as ``self.stats``).
+
+    Instances are constructed by
+    :class:`~psycodict.database.PostgresDatabase` from each table's
+    ``meta_tables`` row; they are not meant to be created directly.
+    """
     ##################################################################
     # Helper functions for querying                                  #
     ##################################################################
@@ -51,7 +80,7 @@ def _parse_projection(self, projection):
             could be split into a search table and an extras table).
           - If 3, as 1 but with id included
           - If a dictionary, can specify columns to include by giving True values, or columns to exclude by giving False values.  Entries must be plain columns; path specifiers are not accepted here (they raise the not-a-column error).
-          - If a list, specifies which columns to include.  Entries may be plain columns, array slicers (``vec[2]``) or dotted [path specifiers](#column-part-specifiers) (``data.s``, ``vec.1``), the same forms accepted by joined queries; only the base column is validated, and the entry string is used verbatim as the key of the returned result dictionaries.
+          - If a list, specifies which columns to include.  Entries may be plain columns, array slicers (``vec[2]``) or dotted path specifiers (``data.s``, ``vec.1``; see the *Column part specifiers* section of QueryLanguage.md), the same forms accepted by joined queries; only the base column is validated, and the entry string is used verbatim as the key of the returned result dictionaries.
           - If a string, projects onto just that column (which may itself be a slicer or path); searches will return the value rather than a dictionary.
 
         OUTPUT:
@@ -142,7 +171,7 @@ def _parse_projection(self, projection):
     def _create_typecast(self, key, value, col, col_type):
         """
         This method is used to add typecasts to queries when necessary.
-        It is called from `_parse_special` and `_parse_dict`; see the documentation
+        It is called from ``_parse_special`` and ``_parse_dict``; see the documentation
         of those functions for inputs.
         """
         if col_type == "smallint[]" and key in ["$contains", "$containedin"]:
@@ -1019,7 +1048,7 @@ def search(
         - ``offset`` -- a nonnegative integer (default 0), where to start in the list of results.
         - ``sort`` -- a sort order.  Either None or a list of strings (which are interpreted as column names in the ascending direction) or of pairs (column name, 1 or -1).  If not specified, will use the default sort order on the table.  If you want the result unsorted, use [].
         - ``info`` -- a dictionary, which is updated with values of 'query', 'count', 'start', 'exact_count' and 'number'.  Optional.
-        - ``split_ors`` -- a boolean.  If true, executes one query per clause in the `$or` list, combining the results.  Only used when a limit is provided.
+        - ``split_ors`` -- a boolean.  If true, executes one query per clause in the ``$or`` list, combining the results.  Only used when a limit is provided.
         - ``one_per`` -- a list of columns.  If provided, only one result will be included with each given set of values for those columns (the first according to the provided sort order).
         - ``silent`` -- a boolean.  If True, slow query warnings will be suppressed.
         - ``raw`` -- a string, to be used as the WHERE part of the query.  DO NOT USE THIS DIRECTLY FOR INPUT FROM WEBSITE.