Add an opt-in PostgreSQL backend for the webtiles user database

crawl-ref/source/webserver/CHANGELOG.md

@@ -7,6 +7,16 @@ organizing principle, though it is important to know that the webtiles server
 as normally installed (e.g on dgamelaunch-config setups) always runs trunk
 code. This file is updated at least at major releases.
 
+## Unreleased
+
+New features:
+
+- An opt-in PostgreSQL backend for the account and user settings databases:
+  set `userdb_backend = "postgresql"` and `userdb_dsn`. The default remains
+  the SQLite files (`password_db`, `settings_db`), and nothing changes for
+  servers that do not set the new options. Requires `psycopg` (see
+  `requirements/postgresql.py3.txt`).
+
 ## [0.34-a0 through 0.34-a0-958-gd0e9a9ddd5]
 
 Major changes:

crawl-ref/source/webserver/README.md

@@ -168,6 +168,10 @@ The server can be configured by modifying the file `config.py`. Most of
 the options are commented or should be self-evident. Suggestions:
 
 * Set uid and gid to a non-privileged user
+* Optionally keep accounts and user settings in PostgreSQL instead of the
+  SQLite files (`userdb_backend = "postgresql"`, `userdb_dsn`); see
+  `config.py` and `requirements/postgresql.py3.txt`. This lets several
+  webtiles hosts share one account database.
 * Enable logging to a file in `logging_config`
 * If required, write a script that initializes  user-specific data, like copying
   a default rc file if the user doesn't yet have one. You can have the script be

crawl-ref/source/webserver/config.py

@@ -98,6 +98,12 @@
 # logging.getLogger('asyncio').setLevel(logging.DEBUG)
 
 password_db = "./webserver/passwd.db3"
+# Accounts and user settings are stored in the SQLite files above by default.
+# Large or multi-host deployments can keep them in PostgreSQL instead; the
+# schema is created on first start and SQLite is not touched. Requires
+# `pip install -r requirements/postgresql.py3.txt`.
+# userdb_backend = "postgresql"
+# userdb_dsn = "postgresql://webtiles:secret@db.example.org/webtiles"
 # Uncomment and change if you want this db somewhere separate from the
 # password_db location.
 #settings_db = "./webserver/user_settings.db3"

crawl-ref/source/webserver/requirements.in/postgresql.txt

@@ -0,0 +1,2 @@
+-r base.txt
+psycopg[binary]

crawl-ref/source/webserver/requirements/postgresql.py3.txt

@@ -0,0 +1,7 @@
+#
+# Optional: only needed with `userdb_backend = "postgresql"`. Install with
+#    pip install -r requirements/postgresql.py3.txt
+# (keep in sync with requirements.in/postgresql.txt)
+#
+-r base.py3.txt
+psycopg[binary]==3.3.4

crawl-ref/source/webserver/webtiles/config.py

@@ -80,6 +80,10 @@ def do_early_logging():
     'max_passwd_length': 20,
     'allow_password_reset': False,
     'admin_password_reset': False,
+    # 'sqlite' (the password_db/settings_db files) or 'postgresql'
+    # (userdb_dsn; requires psycopg, see requirements/postgresql.py3.txt)
+    'userdb_backend': 'sqlite',
+    'userdb_dsn': None,
     'crypt_algorithm': "broken", # should this be the default??
     'crypt_salt_length': 16,
     'login_token_lifetime': 7, # Days; set to <= 0 to disable
@@ -586,6 +590,12 @@ def validate():
         raise ValueError("Webtiles config: malformed ban list ('%s')" %
                                                         repr(get('banned')))
 
+    backend = get('userdb_backend')
+    if backend not in ('sqlite', 'postgresql'):
+        raise ValueError("Webtiles config: unknown userdb_backend '%s'" % backend)
+    if backend == 'postgresql' and not get('userdb_dsn'):
+        raise ValueError("Webtiles config: userdb_backend = 'postgresql' requires userdb_dsn")
+
     # set up defaults that are conditioned on other values
     if not has_key('settings_db'):
         set('settings_db', os.path.join(os.path.dirname(get('password_db')),

crawl-ref/source/webserver/webtiles/userdb.py

@@ -61,6 +61,114 @@ def execute(self, *args, **kwargs):
             raise sqlite3.ProgrammingError("Database connection not initialized!")
         return contextlib.closing(self.conn.execute(*args, **kwargs))
 
+
+def using_postgresql():  # type: () -> bool
+    """Opt-in PostgreSQL storage for accounts and settings. Defaults to the
+    SQLite files, so existing servers are unaffected unless they set
+    `userdb_backend = "postgresql"`."""
+    return config.get('userdb_backend', 'sqlite') == 'postgresql'
+
+
+_PG_REWRITES = [
+    # userdb.py is written in SQLite's dialect; these rewrites are the whole
+    # difference for the queries it uses. Case-insensitive matching comes from
+    # citext columns instead of COLLATE NOCASE.
+    (re.compile(r"\s+COLLATE\s+(NOCASE|RTRIM)\b"), ""),
+    (re.compile(r"datetime\('now',\s*'-(\d+) hours'\)"), r"(now() - interval '\1 hours')"),
+    (re.compile(r"datetime\('now'\)"), "now()"),
+]
+
+
+def pg_sql(sql):  # type: (str) -> str
+    """Rewrite one of this module's SQLite queries for PostgreSQL."""
+    for pattern, replacement in _PG_REWRITES:
+        sql = pattern.sub(replacement, sql)
+    sql = sql.replace("?", "%s")
+    if sql.lstrip().startswith("INSERT OR REPLACE INTO mutesettings"):
+        sql = " ".join(sql.split())  # normalise whitespace so the suffix attaches cleanly
+        sql = sql.replace("INSERT OR REPLACE INTO", "INSERT INTO")
+        sql = sql.rstrip(";") + " ON CONFLICT (username) DO UPDATE SET mutelist = EXCLUDED.mutelist;"
+    return sql
+
+
+class _pg_cursor(object):
+    """Cursor wrapper that applies pg_sql() to every statement."""
+    def __init__(self, cursor):
+        self._cursor = cursor
+
+    def execute(self, sql, parameters=()):
+        return self._cursor.execute(pg_sql(sql), parameters)
+
+    def __getattr__(self, name):
+        return getattr(self._cursor, name)
+
+    def __iter__(self):
+        return iter(self._cursor)
+
+
+class crawl_pg_db(crawl_db):
+    """crawl_db over a PostgreSQL connection (psycopg 3).
+
+    Same interface as crawl_db: `execute()` returns a closing cursor, and the
+    object as a context manager wraps a transaction. The connection runs in
+    autocommit mode so single statements outside `with db:` never leave a
+    transaction open. psycopg is imported lazily; it is only required when
+    this backend is selected (see requirements/postgresql.py3.txt)."""
+    def __init__(self, dsn):  # type: (str) -> None
+        self.name = dsn
+        self.conn = None
+        self._transaction = None
+        if dsn:
+            import psycopg  # optional dependency
+            self.conn = psycopg.connect(dsn, autocommit=True)
+
+    def cursor(self):
+        if not self.conn:
+            raise sqlite3.ProgrammingError("Database connection not initialized!")
+        return contextlib.closing(_pg_cursor(self.conn.cursor()))
+
+    def __enter__(self):
+        if not self.conn:
+            raise sqlite3.ProgrammingError("Database connection not initialized!")
+        self._transaction = self.conn.transaction()
+        self._transaction.__enter__()
+        return self
+
+    def __exit__(self, *args):
+        transaction, self._transaction = self._transaction, None
+        return transaction.__exit__(*args)
+
+    def execute(self, sql, parameters=()):
+        if not self.conn:
+            raise sqlite3.ProgrammingError("Database connection not initialized!")
+        return contextlib.closing(self.conn.execute(pg_sql(sql), parameters))
+
+
+# PostgreSQL schemas. citext gives the case-insensitive matching and unique
+# username that the SQLite schema gets from COLLATE NOCASE.
+pg_settings_schema = """
+    CREATE TABLE IF NOT EXISTS mutesettings (
+        username CITEXT PRIMARY KEY NOT NULL,
+        mutelist TEXT DEFAULT ''
+    );
+"""
+pg_user_schema = """
+    CREATE EXTENSION IF NOT EXISTS citext;
+    CREATE TABLE IF NOT EXISTS dglusers (
+        id SERIAL PRIMARY KEY,
+        username CITEXT UNIQUE,
+        email CITEXT,
+        env TEXT,
+        password TEXT,
+        flags INTEGER
+    );
+    CREATE TABLE IF NOT EXISTS recovery_tokens (
+        token TEXT PRIMARY KEY,
+        token_time TIMESTAMPTZ,
+        user_id INTEGER NOT NULL REFERENCES dglusers(id)
+    );
+"""
+
 def create_settings_db(filename):  # type: () -> None
     # note: when this behavior was converted from muting to blocking, the db
     # names were left as-is
@@ -76,6 +184,11 @@ def create_settings_db(filename):  # type: () -> None
 
 
 def ensure_settings_db_exists(quiet=False):
+    if using_postgresql():
+        db = crawl_pg_db(config.get('userdb_dsn'))
+        with db:
+            db.execute(pg_settings_schema)
+        return db
     dbname = config.get('settings_db')
     if not os.path.exists(dbname):
         if not quiet:
@@ -120,6 +233,11 @@ def create_user_db(filename):
 
 
 def ensure_user_db_exists(quiet=False):  # type: () -> None
+    if using_postgresql():
+        db = crawl_pg_db(config.get('userdb_dsn'))
+        with db:
+            db.execute(pg_user_schema)
+        return db
     dbname = config.get('password_db')
     if not os.path.exists(dbname):
         if not quiet:
@@ -135,6 +253,8 @@ def ensure_user_db_exists(quiet=False):  # type: () -> None
 def upgrade_user_db():  # type: () -> None
     """Automatically upgrades the database."""
     global user_db, recovery_schema
+    if using_postgresql():
+        return  # the schema is created with IF NOT EXISTS
     # possibly CREATE .. IF NOT EXISTS would be more idiomatic sql...
     with user_db.cursor() as c:
         query = "SELECT name FROM sqlite_master WHERE type='table' or type='index';"

crawl-ref/source/webserver/webtiles/userdb_test.py

@@ -0,0 +1,100 @@
+"""Tests for the user database backends.
+
+The SQLite suite (`UserDBTest`, defined next to the code in userdb.py) always
+runs. The PostgreSQL suite reruns the same tests against a live server and is
+skipped unless `WEBTILES_TEST_POSTGRESQL_DSN` is set, e.g.
+
+    WEBTILES_TEST_POSTGRESQL_DSN=postgresql://user:pw@localhost/webtiles pytest
+
+It drops and recreates the webtiles tables in that database.
+"""
+import os
+import unittest
+
+from webtiles import config, userdb
+from webtiles.userdb import UserDBTest  # noqa: F401 -- collected by pytest
+
+PG_DSN = os.environ.get("WEBTILES_TEST_POSTGRESQL_DSN")
+
+
+class PostgresSQLTranslationTest(unittest.TestCase):
+    """The SQLite dialect used throughout userdb.py is rewritten for
+    PostgreSQL at execute time; these pin the rewrite rules."""
+
+    def test_placeholders(self):
+        self.assertEqual(userdb.pg_sql("SELECT 1 FROM t WHERE a=? AND b=?"),
+                         "SELECT 1 FROM t WHERE a=%s AND b=%s")
+
+    def test_collations_are_dropped(self):
+        # username/email/token columns are citext on PostgreSQL, so the
+        # SQLite collation clauses have no equivalent and must vanish.
+        self.assertEqual(userdb.pg_sql("WHERE username=?\n        COLLATE NOCASE"),
+                         "WHERE username=%s")
+        self.assertEqual(userdb.pg_sql("WHERE t.token = ? COLLATE RTRIM"),
+                         "WHERE t.token = %s")
+
+    def test_datetime_functions(self):
+        self.assertEqual(userdb.pg_sql("VALUES (?,datetime('now'),?)"),
+                         "VALUES (%s,now(),%s)")
+        self.assertEqual(
+            userdb.pg_sql("t.token_time > datetime('now','-12 hours')"),
+            "t.token_time > (now() - interval '12 hours')")
+        self.assertEqual(
+            userdb.pg_sql("SET token_time=datetime('now', '-2 hours')"),
+            "SET token_time=(now() - interval '2 hours')")
+
+    def test_upsert(self):
+        sql = userdb.pg_sql("INSERT OR REPLACE INTO mutesettings (username, mutelist) "
+                            "VALUES (?,?);")
+        self.assertEqual(sql, "INSERT INTO mutesettings (username, mutelist) VALUES (%s,%s) "
+                              "ON CONFLICT (username) DO UPDATE SET mutelist = EXCLUDED.mutelist;")
+
+
+@unittest.skipUnless(PG_DSN, "WEBTILES_TEST_POSTGRESQL_DSN not set")
+class PostgresUserDBTest(UserDBTest):
+    """Exactly the SQLite test suite, against PostgreSQL."""
+
+    def setUp(self):
+        if not self.logging_init:
+            import webtiles.server as server
+            server.init_logging(config.get('logging_config'))
+            self.logging_init = True
+        self.config_shim = dict(
+            userdb_backend="postgresql",
+            userdb_dsn=PG_DSN,
+            # unused on this backend; set so tearDown of the base class is inert
+            settings_db="./unittest_pg_unused_settings.db3",
+            password_db="./unittest_pg_unused_passwd.db3",
+            dgl_mode=True)
+        config.server_config = self.config_shim
+        self._drop_tables()
+        userdb.init_db_connections(quiet=True)
+
+    def tearDown(self):
+        self._drop_tables()
+        userdb.user_db.close()
+        userdb.settings_db.close()
+        super().tearDown()
+
+    def _drop_tables(self):
+        db = userdb.crawl_pg_db(PG_DSN)
+        try:
+            with db:
+                db.execute("DROP TABLE IF EXISTS recovery_tokens, mutesettings, dglusers")
+        finally:
+            db.close()
+
+    def test_backend_selected(self):
+        self.assertIsInstance(userdb.user_db, userdb.crawl_pg_db)
+        self.assertIsInstance(userdb.settings_db, userdb.crawl_pg_db)
+
+    def test_case_insensitive_username_is_unique(self):
+        self.assertIsNone(userdb.register_user("Test", "hunter2", "a@example.com"))
+        self.assertEqual(userdb.register_user("tEsT", "hunter2", "b@example.com"),
+                         "User already exists!")
+        self.assertEqual(userdb.get_user_info("TEST").username, "Test")
+
+    def test_blocklist_upsert(self):
+        userdb.set_blocklist("test", "alice")
+        userdb.set_blocklist("TEST", "alice bob")
+        self.assertEqual(userdb.get_blocklist("test"), "alice bob")