diff --git a/psycodict/base.py b/psycodict/base.py index 76f376d..0683a40 100644 --- a/psycodict/base.py +++ b/psycodict/base.py @@ -1,4 +1,16 @@ # -*- coding: utf-8 -*- +""" +The shared plumbing underneath every psycodict object. + +:class:`PostgresBase` is the common base of the database, table and +statistics classes; it owns statement execution through ``_execute`` +(logging, slow-query warnings, commit/rollback bookkeeping and +reconnection) together with helpers for inspecting tables, indexes and +constraints. The module also defines the layout of the ``meta_*`` tables -- +the column lists, types and creation statements shared by everything that +reads or writes them -- and the metadata format version (``META_FORMAT``) +stamped into ``meta_format``. +""" import csv import logging import re @@ -111,6 +123,16 @@ def jsonb_idx(cols, cols_type): + """ + The positions in ``cols`` whose type is ``jsonb``, as a tuple of + indexes. Used to decide which values need json decoding when reading + rows of the ``meta_*`` tables. + + INPUT: + + - ``cols`` -- a list of column names + - ``cols_type`` -- a dictionary mapping column names to their types + """ return tuple(i for i, elt in enumerate(cols) if cols_type[elt] == "jsonb") @@ -629,7 +651,7 @@ def _constraint_exists(self, constraintname, tablename=None): def _list_indexes(self, tablename): """ - Lists built index names on the search table `tablename` + Lists built index names on the search table ``tablename`` """ cur = self._execute( SQL("SELECT indexname FROM pg_indexes WHERE tablename = %s"), @@ -640,7 +662,7 @@ def _list_indexes(self, tablename): def _list_constraints(self, tablename): """ - Lists constraint names on the search table `tablename` + Lists constraint names on the search table ``tablename`` """ # if we look into information_schema.table_constraints # we also get internal constraints, I'm not sure why diff --git a/psycodict/config.py b/psycodict/config.py index f13704a..9b9320c 100644 --- a/psycodict/config.py +++ b/psycodict/config.py @@ -1,4 +1,16 @@ # -*- coding: utf-8 -*- +""" +Configuration handling for psycodict. + +:class:`Configuration` merges a ``config.ini`` file with command-line +arguments (off by default; see ``readargs``) and hands the result to +:class:`~psycodict.database.PostgresDatabase`. When no file is named +explicitly, :func:`find_config_file` discovers one: the +``PSYCODICT_CONFIG`` environment variable, then ``config.ini`` in the +current directory if it already exists, then ``~/.psycodict/config.ini``, +which is created on first use. A ``secrets.ini`` next to the configuration +file overrides its values, so credentials can be kept out of it. +""" import os import argparse from configparser import ConfigParser @@ -362,6 +374,11 @@ def _default_log_dir(config_file): return None def get_postgresql_default(self): + """ + The built-in default connection options (host, port, user, ...), as + a dictionary -- the values used before the configuration file and + command line are consulted. + """ res = dict(self.default_args["postgresql"]) res["port"] = int(res["port"]) return res diff --git a/psycodict/database.py b/psycodict/database.py index 3d743bf..98997d5 100644 --- a/psycodict/database.py +++ b/psycodict/database.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +""" +The connection object at the center of psycodict. + +:class:`PostgresDatabase` connects using a +:class:`~psycodict.config.Configuration` and exposes each table in the +database as an attribute (``db.nf_fields``), a +:class:`~psycodict.searchtable.PostgresSearchTable`. It also provides the +database-wide operations: creating, dropping and renaming tables, +reloading or cleaning up every table at once, schema-change notifications +(:meth:`~PostgresDatabase.listener`), refreshing table metadata without a +restart (:meth:`~PostgresDatabase.refresh_tables`), and the metadata +format stamp and its migrations. +""" import csv import logging from pathlib import Path @@ -78,10 +91,20 @@ class NumericLoader(Loader): (Sage Integer/RealLiteral when Sage is available, int/float otherwise). """ def load(self, data): + """ + Convert the numeric's text representation to a Python/Sage number. + """ return numeric_converter(bytes(data).decode()) def setup_connection(conn): + """ + Prepare a fresh psycopg connection for psycodict: set the client + encoding and register the loaders and dumpers that implement + psycodict's value conversion (numerics, json, arrays, and -- when Sage + is available -- Sage integers and reals). Called for every connection + the database opens. + """ # psycopg3 uses unicode (str) everywhere by default, so the psycopg2-era # UNICODE/UNICODEARRAY registrations are no longer needed. conn.execute("SET client_encoding TO 'UTF8'") @@ -1053,7 +1076,7 @@ def create_table( include_nones=True, ): """ - Add a new search table to the database. See also `create_table_like`. + Add a new search table to the database. See also :meth:`create_table_like`. INPUT: @@ -1450,24 +1473,21 @@ def reload_all( ): """ Reloads all tables from files in a given folder. The filenames must match - the names of the tables, with `_counts` and `_stats` appended as appropriate. + the names of the tables, with ``_counts`` and ``_stats`` appended as appropriate. INPUT: - - ``data_folder`` -- the folder that contains files to be reloaded - - ``halt_on_errors`` -- whether to stop if a DatabaseError is - encountered while trying to reload one of the tables - - ``sequential_swap`` -- if True, then the whole transaction will not be wrapped in a DelayCommit, which can sometimes prevent deadlocks. - - INPUTS passed to `reload` function in `PostgresTable`: - - - ``resort``, ``restat``, ``adjust_schema``, and any extra keywords - - + - ``data_folder`` -- the folder that contains files to be reloaded + - ``halt_on_errors`` -- whether to stop if a DatabaseError is + encountered while trying to reload one of the tables + - ``sequential_swap`` -- if True, then the whole transaction will not + be wrapped in a DelayCommit, which can sometimes prevent deadlocks + - ``resort``, ``restat``, ``adjust_schema``, and any extra keywords + are passed on to the ``reload`` method of each + :class:`~psycodict.table.PostgresTable` Note that this function currently does not reload data that is not in a search table, such as knowls or user data. - """ data_folder = Path(data_folder) @@ -1639,7 +1659,7 @@ def reload_all_revert(self, data_folder): def cleanup_all(self): """ - Drops all `_tmp` and `_old` tables created by the reload() method. + Drops all ``_tmp`` and ``_old`` tables created by the reload() method. """ with DelayCommit(self, silence=True): for tablename in self.tablenames: diff --git a/psycodict/encoding.py b/psycodict/encoding.py index 47ed1be..0ba5eda 100644 --- a/psycodict/encoding.py +++ b/psycodict/encoding.py @@ -77,10 +77,17 @@ def __str__(self): return self.literal class RealEncoder(): + """ + A wrapper rendering a Sage RealNumber/RealLiteral as its literal + text (the exact decimal string it was created from, when available). + """ def __init__(self, value): self._value = value def getquoted(self): + """ + The wrapped real number's literal text. + """ if isinstance(self._value, RealLiteral): return self._value.literal else: @@ -96,6 +103,9 @@ class RealLiteralDumper(Dumper): as psycopg2's client-side interpolation did. """ def dump(self, obj): + """ + Render the real number as its literal text. + """ if isinstance(obj, RealLiteral): return obj.literal.encode() return str(obj).encode() @@ -105,6 +115,9 @@ class SageIntegerDumper(Dumper): Dumps a Sage Integer as its decimal text (replacing psycopg2's AsIs). """ def dump(self, obj): + """ + Render the integer as decimal text. + """ return str(obj).encode() @@ -180,6 +193,9 @@ def __init__(self, seq): self._seq = seq def getquoted(self): + """ + The Postgres array literal for the wrapped sequence, as bytes. + """ return _pg_array_literal(self._seq).encode() def __str__(self): @@ -213,6 +229,9 @@ class ArrayDumper(Dumper): Dumps an Array wrapper as a Postgres array literal with unknown oid. """ def dump(self, obj): + """ + Render the wrapped sequence as a Postgres array literal. + """ return _pg_array_literal(obj._seq).encode() @@ -236,10 +255,18 @@ def __repr__(self): @classmethod def dumps(cls, obj): + """ + Serialize ``obj`` to json text using the extended encoding + (see :meth:`prep`). + """ return json.dumps(cls.prep(obj)) @classmethod def loads(cls, s): + """ + Parse json text and decode the extended encoding back to Python + and Sage objects (see :meth:`extract`). + """ return cls.extract(json.loads(s)) @classmethod @@ -514,6 +541,9 @@ class JsonWrapperDumper(Dumper): context (this works for both json and jsonb columns). """ def dump(self, obj): + """ + Render the wrapped value as json text. + """ return Json.dumps(obj.obj).encode() @@ -523,6 +553,9 @@ class DictJsonDumper(Dumper): ``register_adapter(dict, Json)``). """ def dump(self, obj): + """ + Render the dict as json text. + """ return Json.dumps(obj).encode() diff --git a/psycodict/searchtable.py b/psycodict/searchtable.py index 3c9b75a..8b9eaff 100644 --- a/psycodict/searchtable.py +++ b/psycodict/searchtable.py @@ -1,4 +1,14 @@ # -*- coding: utf-8 -*- +""" +The read half of a table: dictionary queries. + +:class:`PostgresSearchTable` extends +:class:`~psycodict.table.PostgresTable` with the query API -- ``search``, +``lucky``, ``lookup``, ``count``, ``random`` and friends -- translating +query dictionaries into SQL SELECT statements, with projections, sorting, +joins and the ``info`` contract used by search pages. The query language +is specified in QueryLanguage.md and the read API in Searching.md. +""" import random import time from itertools import islice @@ -33,6 +43,25 @@ def _qualify(frag, tablename): class PostgresSearchTable(PostgresTable): + """ + A single search table, as returned by ``db.``: the interface + for reading data with dictionary queries. + + The read methods -- :meth:`search`, :meth:`lucky`, :meth:`lookup`, + :meth:`exists`, :meth:`count`, :meth:`random`, :meth:`distinct` and + friends -- accept a query dictionary such as + ``{"degree": 2, "disc": {"$lt": 0}}``, translate it into SQL, and + return rows as dictionaries (or bare values, under a string + projection). The query language is specified in QueryLanguage.md and + the read API in Searching.md. Writing and schema changes live on the + base class :class:`~psycodict.table.PostgresTable`, and precomputed + statistics on :class:`~psycodict.statstable.PostgresStatsTable` + (available as ``self.stats``). + + Instances are constructed by + :class:`~psycodict.database.PostgresDatabase` from each table's + ``meta_tables`` row; they are not meant to be created directly. + """ ################################################################## # Helper functions for querying # ################################################################## @@ -51,7 +80,7 @@ def _parse_projection(self, projection): could be split into a search table and an extras table). - If 3, as 1 but with id included - If a dictionary, can specify columns to include by giving True values, or columns to exclude by giving False values. Entries must be plain columns; path specifiers are not accepted here (they raise the not-a-column error). - - If a list, specifies which columns to include. Entries may be plain columns, array slicers (``vec[2]``) or dotted [path specifiers](#column-part-specifiers) (``data.s``, ``vec.1``), the same forms accepted by joined queries; only the base column is validated, and the entry string is used verbatim as the key of the returned result dictionaries. + - If a list, specifies which columns to include. Entries may be plain columns, array slicers (``vec[2]``) or dotted path specifiers (``data.s``, ``vec.1``; see the *Column part specifiers* section of QueryLanguage.md), the same forms accepted by joined queries; only the base column is validated, and the entry string is used verbatim as the key of the returned result dictionaries. - If a string, projects onto just that column (which may itself be a slicer or path); searches will return the value rather than a dictionary. OUTPUT: @@ -142,7 +171,7 @@ def _parse_projection(self, projection): def _create_typecast(self, key, value, col, col_type): """ This method is used to add typecasts to queries when necessary. - It is called from `_parse_special` and `_parse_dict`; see the documentation + It is called from ``_parse_special`` and ``_parse_dict``; see the documentation of those functions for inputs. """ if col_type == "smallint[]" and key in ["$contains", "$containedin"]: @@ -1019,7 +1048,7 @@ def search( - ``offset`` -- a nonnegative integer (default 0), where to start in the list of results. - ``sort`` -- a sort order. Either None or a list of strings (which are interpreted as column names in the ascending direction) or of pairs (column name, 1 or -1). If not specified, will use the default sort order on the table. If you want the result unsorted, use []. - ``info`` -- a dictionary, which is updated with values of 'query', 'count', 'start', 'exact_count' and 'number'. Optional. - - ``split_ors`` -- a boolean. If true, executes one query per clause in the `$or` list, combining the results. Only used when a limit is provided. + - ``split_ors`` -- a boolean. If true, executes one query per clause in the ``$or`` list, combining the results. Only used when a limit is provided. - ``one_per`` -- a list of columns. If provided, only one result will be included with each given set of values for those columns (the first according to the provided sort order). - ``silent`` -- a boolean. If True, slow query warnings will be suppressed. - ``raw`` -- a string, to be used as the WHERE part of the query. DO NOT USE THIS DIRECTLY FOR INPUT FROM WEBSITE. diff --git a/psycodict/statstable.py b/psycodict/statstable.py index 6f51ceb..7a97d12 100644 --- a/psycodict/statstable.py +++ b/psycodict/statstable.py @@ -1,4 +1,16 @@ # -*- coding: utf-8 -*- +""" +Precomputed counts and statistics for a search table. + +:class:`PostgresStatsTable`, available as ``table.stats``, records the +number of rows matching queries (in an auxiliary ``_counts`` table) and +statistics -- minimum, maximum, average, totals -- on numerical columns +(in a ``_stats`` table), so that count displays and statistics pages need +not rescan large tables. Statistics are added with ``add_stats`` and +``add_numstats``, kept current by the data management methods of +:class:`~psycodict.table.PostgresTable`, and read back with ``count``, +``quick_count``, ``max``, ``column_counts``, ``numstats`` and friends. +""" import itertools import logging import math @@ -40,7 +52,9 @@ class PostgresStatsTable(PostgresBase): """ - This object is used for storing statistics and counts for a search table. + This object is used for storing statistics and counts for a search table, + available as the ``stats`` attribute of a + :class:`~psycodict.searchtable.PostgresSearchTable`. For each search table (e.g. ec_curvedata), there are two auxiliary tables supporting statistics functionality. The counts table (e.g. ec_curvedata_counts) records @@ -67,29 +81,31 @@ class PostgresStatsTable(PostgresBase): updated. The backend functionality of this object supports the StatsDisplay object - available in `lmfdb.utils.display_stats`. See that module for more details - on making a statistics page for a section of the LMFDB. In particular, - the interface there has the capacity to automatically call ``add_stats`` so that - viewing an appropriate stats page (e.g. beta.lmfdb.org/ModularForm/GL2/Q/holomorphic/stats) - is sufficient to add the necessary statistics to the stats and counts tables. - The methods ``_get_values_counts`` and ``_get_total_avg`` exist to support - the ``StatsDisplay`` object. + available in the LMFDB (``lmfdb.utils.display_stats``). See that module + for more details on making a statistics page for a section of the LMFDB. + In particular, the interface there has the capacity to automatically call + ``add_stats`` so that viewing an appropriate stats page + (e.g. beta.lmfdb.org/ModularForm/GL2/Q/holomorphic/stats) is sufficient to + add the necessary statistics to the stats and counts tables. The methods + ``_get_values_counts`` and ``_get_total_avg`` exist to support the + ``StatsDisplay`` object. Once statistics have been added, they are accessed using the following functions: - ``quick_count`` -- count the number of rows satisfying a query, - returning None if not already cached. - - ``count`` -- count the number of rows satisfying a query, computing and storing - the result if not yet cached. - - ``max`` -- returns the maximum value attained by a column, computing and storing - the result if not yet cached. - - ``column_counts`` -- provides all counts stored for a given column or set of columns. - This will be much faster than calling ``count`` repeatedly. - If ``add_stats`` has not been called, it will do so. - - ``numstats`` -- provides numerical statistics on a single column, grouped by - the values taken on by another set of columns. - - ``extra_counts`` -- returns a dictionary giving counts that were added separately - from an ``add_stats`` call (for example, via user requests on the website) + returning None if not already cached. + - ``count`` -- count the number of rows satisfying a query, computing and + storing the result if not yet cached. + - ``max`` -- returns the maximum value attained by a column, computing and + storing the result if not yet cached. + - ``column_counts`` -- provides all counts stored for a given column or + set of columns. This will be much faster than calling ``count`` + repeatedly. If ``add_stats`` has not been called, it will do so. + - ``numstats`` -- provides numerical statistics on a single column, + grouped by the values taken on by another set of columns. + - ``extra_counts`` -- returns a dictionary giving counts that were added + separately from an ``add_stats`` call (for example, via user requests + on the website) - ``status`` -- prints a summary of the statistics currently stored. EXAMPLES: @@ -142,15 +158,15 @@ class PostgresStatsTable(PostgresBase): - ``cols`` -- these are the columns specified in the query. A list, stored as a jsonb. - ``values`` -- these could be numbers, or dictionaries giving a more complicated constraint. - A list, of the same length as ``cols``, stored as a jsonb. + A list, of the same length as ``cols``, stored as a jsonb. - ``count`` -- the number of rows in the search table where the columns take on the given values. - ``extra`` -- false if the count was added in an ``add_stats`` method, - true if it was added separately (such as by a request on a search results page). + true if it was added separately (such as by a request on a search results page). - ``split`` -- used when column values are arrays. If true, then the array is split - up before counting. For example, when counting ramified primes, - if split werefalse then [2,3,5] and [2,3,7] would count as separate values - (there are 888280 number fields in the LMFDB with ramps = [2,3,5]). - If split were true, then both [2,3,5] and [2,3,7] would contribute toward the count for 2. + up before counting. For example, when counting ramified primes, + if split were false then [2,3,5] and [2,3,7] would count as separate values + (there are 888280 number fields in the LMFDB with ramps = [2,3,5]). + If split were true, then both [2,3,5] and [2,3,7] would contribute toward the count for 2. For example, ["ramps"], [[2, 3, 5]], 888280, t, f @@ -167,16 +183,16 @@ class PostgresStatsTable(PostgresBase): The columns in a stats table are: - ``stat`` -- a text field giving the statistic type. Currently, will be one of - "max", "min", "avg", "total" (one such row for each add_stats call), - "ntotal" (one such row for each add_numstats call), "split_total" - (one such row for each add_stats call with split_list True). + "max", "min", "avg", "total" (one such row for each add_stats call), + "ntotal" (one such row for each add_numstats call), "split_total" + (one such row for each add_stats call with split_list True). - ``cols`` -- the columns for which statistics are being computed. Must have - length 1 and be numerical in order to have "max", "min" or "avg" + length 1 and be numerical in order to have "max", "min" or "avg" - ``constraint_cols`` -- columns in the constraint dictionary - ``constraint_values`` -- the values specified for the columns in ``ccols`` - ``threshold`` -- NULL or an integer. If specified, only value sets where the - row count surpasses the threshold will be added to the counts table and - counted toward min, max and avg statistics. + row count surpasses the threshold will be added to the counts table and + counted toward min, max and avg statistics. BUCKETED STATS: @@ -192,12 +208,12 @@ class PostgresStatsTable(PostgresBase): sage: db.mf_newforms.stats.add_bucketed_counts(['level', 'weight'], {'level': ['1','2-10','11-100','101-1000','1001-2000', '2001-4000','4001-6000','6001-8000','8001-10000'], 'weight': ['1','2','3','4','5-8','9-16','17-32','33-64','65-316']}) - You can now count certain ranges: + You can now count certain ranges:: sage: db.mf_newforms.stats.quick_count({'level':{'$gte':101, '$lte':1000}, 'weight':4}) 12281 - But only those specified by the buckets: + But only those specified by the buckets:: sage: db.mf_newforms.stats.quick_count({'level':{'$gte':201, '$lte':800}, 'weight':2}) is None True @@ -206,7 +222,7 @@ class PostgresStatsTable(PostgresBase): - ``table`` -- a ``PostgresTable`` object. - ``total`` -- an integer, the number of rows in the search table. If not provided, - it will be looked up or computed. + it will be looked up or computed. """ # By default we don't save counts. You can inherit from this class and change # the following value to True, then set _stats_table_class_ to your new stats class on your table class @@ -761,7 +777,7 @@ def _slow_statistic(self, col, constraint, kind="max"): def _record_statistic(self, col, ccols, cvals, m, kind="max"): """ - Store a computed statistic (referenced by `kind`) in the stats table. + Store a computed statistic (referenced by ``kind``) in the stats table. INPUT: @@ -769,7 +785,7 @@ def _record_statistic(self, col, ccols, cvals, m, kind="max"): - ``ccols`` -- the constraint columns - ``cvals`` -- the constraint values - ``m`` -- the maximum value to be stored - - ``kind`` -- the kind of statistic. Usually `min` or `max` or `sum` + - ``kind`` -- the kind of statistic. Usually ``min`` or ``max`` or ``sum`` """ try: inserter = SQL( @@ -1146,7 +1162,7 @@ def add_numstats(self, col, grouping, constraint=None, threshold=None, suffix="" INPUT: - ``col`` -- the column whose minimum, maximum and average values are to be computed. - Should be an integer or real type in order for `AVG` to function. + Should be an integer or real type in order for ``AVG`` to function. - ``grouping`` -- a list of columns. Statistics will be computed within groups defined by the values taken on by these columns. If no columns given, then the overall statistics will be computed. @@ -1284,7 +1300,7 @@ def numstats(self, col, grouping, constraint=None, threshold=None): INPUT: - ``col`` -- the column whose minimum, maximum and average values are to be computed. - Should be an integer or real type in order for `AVG` to function. + Should be an integer or real type in order for ``AVG`` to function. - ``grouping`` -- a list of columns. Statistics will be computed within groups defined by the values taken on by these columns. If no columns given, then the overall statistics will be computed. @@ -1912,7 +1928,7 @@ def extra_counts(self, include_counts=True, suffix=""): INPUT: - ``include_counts`` -- if False, will omit the counts and just give lists of values. - - ``suffix`` -- Used when dealing with `_tmp` or `_old*` tables. + - ``suffix`` -- Used when dealing with ``_tmp`` or ``_old*`` tables. """ selecter = SQL("SELECT cols, values, count FROM {0} WHERE extra ='t'").format( Identifier(self.counts + suffix) diff --git a/psycodict/table.py b/psycodict/table.py index d71b8c8..cdc2d1b 100644 --- a/psycodict/table.py +++ b/psycodict/table.py @@ -1,4 +1,16 @@ # -*- coding: utf-8 -*- +""" +The write and schema half of a table. + +:class:`PostgresTable` manages a single search table: row-level writes +(``insert_many``, ``update``, ``upsert``, ``delete``), the bulk file-based +import and export that psycodict is built around (``copy_from``, +``copy_to``, ``reload`` and its staged ``_tmp``-table machinery), and the +table's schema -- columns, indexes, constraints and the corresponding +``meta_*`` bookkeeping with its versioned history. The read interface +lives in the subclass :class:`~psycodict.searchtable.PostgresSearchTable`, +which is what ``db.`` actually returns. +""" import csv import os import tempfile @@ -303,10 +315,10 @@ def list_indexes(self, verbose=False): NOTE: - - not necessarily all built - - not necessarily a supset of all the built indexes. + - not necessarily all built + - not necessarily a superset of all the built indexes - For the current built indexes on the search table, see _list_built_indexes + For the current built indexes on the search table, see ``_list_built_indexes`` """ if self._db._meta_format >= 1: selecter = SQL("SELECT index_name, type, columns, modifiers, whereclause FROM meta_indexes WHERE table_name = %s") @@ -755,10 +767,10 @@ def list_constraints(self, verbose=False): NOTE: - - not necessarily all built - - not necessarily a supset of all the built constraints. + - not necessarily all built + - not necessarily a superset of all the built constraints - For the current built constraints on the search table, see _list_built_constraints + For the current built constraints on the search table, see ``_list_built_constraints`` """ selecter = SQL("SELECT constraint_name, type, columns, check_func FROM meta_constraints WHERE table_name = %s") cur = self._execute(selecter, [self.search_table], silent=True) @@ -946,12 +958,25 @@ def _constraints_touching(self, columns): ################################################################## def copy_to_meta(self, filename, sep="|"): + """ + Export this table's row of ``meta_tables`` to a file, in the format + accepted by :meth:`reload_meta`. + """ self._copy_to_meta("meta_tables", filename, self.search_table, sep=sep) def copy_to_indexes(self, filename, sep="|"): + """ + Export this table's index definitions (its rows of ``meta_indexes``) + to a file, in the format accepted by :meth:`reload_indexes`. + """ self._copy_to_meta("meta_indexes", filename, self.search_table, sep=sep) def copy_to_constraints(self, filename, sep="|"): + """ + Export this table's constraint definitions (its rows of + ``meta_constraints``) to a file, in the format accepted by + :meth:`reload_constraints`. + """ self._copy_to_meta("meta_constraints", filename, self.search_table, sep=sep) def _get_current_index_version(self): @@ -961,21 +986,67 @@ def _get_current_constraint_version(self): return self._get_current_meta_version("meta_constraints", self.search_table) def reload_indexes(self, filename, sep="|"): + """ + Replace this table's index definitions in ``meta_indexes`` with the + contents of the file (as written by :meth:`copy_to_indexes`). The + definitions being replaced are archived in ``meta_indexes_hist``, so + :meth:`revert_indexes` can undo this. + """ return self._reload_meta("meta_indexes", filename, self.search_table, sep=sep) def reload_meta(self, filename, sep="|"): + """ + Replace this table's row of ``meta_tables`` with the contents of the + file (as written by :meth:`copy_to_meta`). The row being replaced + is archived in ``meta_tables_hist``, so :meth:`revert_meta` can undo + this. + """ return self._reload_meta("meta_tables", filename, self.search_table, sep=sep) def reload_constraints(self, filename, sep="|"): + """ + Replace this table's constraint definitions in ``meta_constraints`` + with the contents of the file (as written by + :meth:`copy_to_constraints`). The definitions being replaced are + archived in ``meta_constraints_hist``, so :meth:`revert_constraints` + can undo this. + """ return self._reload_meta("meta_constraints", filename, self.search_table, sep=sep) def revert_indexes(self, version=None): + """ + Restore an earlier version of this table's index definitions from + ``meta_indexes_hist``. + + INPUT: + + - ``version`` -- the version to restore (default: the one before the + current version) + """ return self._revert_meta("meta_indexes", self.search_table, version) def revert_constraints(self, version=None): + """ + Restore an earlier version of this table's constraint definitions + from ``meta_constraints_hist``. + + INPUT: + + - ``version`` -- the version to restore (default: the one before the + current version) + """ return self._revert_meta("meta_constraints", self.search_table, version) def revert_meta(self, version=None): + """ + Restore an earlier version of this table's ``meta_tables`` row from + ``meta_tables_hist``. + + INPUT: + + - ``version`` -- the version to restore (default: the one before the + current version) + """ return self._revert_meta("meta_tables", self.search_table, version) ################################################################## @@ -1070,6 +1141,11 @@ def _break_order(self): self._out_of_order = True def finalize_changes(self): + """ + Intended to finish off a batch of data changes by updating the + cached total, refreshing statistics targets, and re-sorting by id. + Currently a placeholder that does nothing. + """ # TODO # Update stats.total # Refresh stats targets @@ -2097,12 +2173,12 @@ def reload( def reload_final_swap(self, tables=None, metafile=None, ordered=False, sep="|"): """ - Renames the _tmp versions of `tables` to the live versions. - and updates the corresponding meta_tables row if `metafile` is provided + Renames the ``_tmp`` versions of ``tables`` to the live versions, + and updates the corresponding meta_tables row if ``metafile`` is provided. INPUT: - - ``tables`` -- list of strings (optional), of the tables to renamed. If None is provided, renames all the tables ending in `_tmp` + - ``tables`` -- list of strings (optional), of the tables to be renamed. If None is provided, renames all the tables ending in ``_tmp`` - ``metafile`` -- a string (optional), giving a file containing the meta information for the table. - ``sep`` -- a character (default ``|``) to separate columns """ diff --git a/psycodict/utils.py b/psycodict/utils.py index 4520675..6fe519e 100644 --- a/psycodict/utils.py +++ b/psycodict/utils.py @@ -1,5 +1,11 @@ # -*- coding: utf-8 -*- - +""" +Helpers shared across psycodict: :class:`DelayCommit` for batching many +writes into one transaction, the whitelist parser that admits limited raw +arithmetic into queries (:func:`filter_sql_injection`), identifier wrapping +with array slicers (:func:`IdentifierWrapper`), the exceptions raised during +query parsing and locking, and small formatting utilities. +""" import os.path import sys import re @@ -39,14 +45,18 @@ def __init__(self, *args, **kwargs): # This function is used to support the inclusion of limited raw postgres in queries def filter_sql_injection(clause, col, col_type, op, table, join_context=None): """ + Build a comparison clause from an untrusted arithmetic expression, + admitting only whitelisted column names, numbers and the arithmetic + operators ``+-*/^()``. Returns a pair of an SQL composable and the list + of values to substitute into it. + INPUT: - ``clause`` -- a plain string, obtained from the website UI so NOT SAFE - ``col`` -- an SQL Identifier for a column in a table - ``col_type`` -- a string giving the type of the column - - ``valid_cols`` -- the column names for this table - ``op`` -- a string giving the operator to use - (`=` or one of the values in the ``postgres_infix_ops dictionary`` above) + (``=`` or one of the values in the ``postgres_infix_ops`` dictionary above) - ``table`` -- a PostgresTable object for determining which columns are valid - ``join_context`` -- when filtering an expression inside a joined query, a dictionary mapping joined table names to their table objects. Names @@ -186,6 +196,11 @@ def IdentifierWrapper(name, convert=True): class LockError(RuntimeError): + """ + Raised when a data-changing operation cannot proceed because another + operation holds a conflicting lock on the table (for example, a reload + in progress, or a staged ``_tmp`` copy awaiting its swap). + """ pass @@ -195,6 +210,9 @@ class QueryLogFilter(): """ def filter(self, record): + """ + Keep only records emitted from base.py (the query-execution path). + """ if os.path.basename(record.pathname) == "base.py": return 1 else: @@ -205,7 +223,7 @@ class DelayCommit(): """ Used to set default behavior for whether to commit changes to the database connection. - Entering this context in a with statement will cause `_execute` calls to not commit by + Entering this context in a with statement will cause ``_execute`` calls to not commit by default. When the final DelayCommit is exited, the connection will commit. Setting active=False disables the DelayCommit completely, which can be helpful since it's often used in a with context and conditionally entering that context is annoying to write with if statements. @@ -253,6 +271,11 @@ def reraise(exc_type, exc_value, exc_traceback=None): raise exc_value def range_formatter(x): + """ + Format a query-language range constraint for human display: + ``{"$gte": 2, "$lte": 7}`` becomes ``"2-7"``, one-sided constraints + become ``"2-"`` or ``"..7"``, and ``None`` becomes ``"Unknown"``. + """ if x is None: return 'Unknown' elif isinstance(x, dict):