diff --git a/crawl-ref/source/webserver/CHANGELOG.md b/crawl-ref/source/webserver/CHANGELOG.md index 32c4788b98d..35de01b46b1 100644 --- a/crawl-ref/source/webserver/CHANGELOG.md +++ b/crawl-ref/source/webserver/CHANGELOG.md @@ -7,6 +7,16 @@ organizing principle, though it is important to know that the webtiles server as normally installed (e.g on dgamelaunch-config setups) always runs trunk code. This file is updated at least at major releases. +## Unreleased + +New features: + +- An opt-in PostgreSQL backend for the account and user settings databases: + set `userdb_backend = "postgresql"` and `userdb_dsn`. The default remains + the SQLite files (`password_db`, `settings_db`), and nothing changes for + servers that do not set the new options. Requires `psycopg` (see + `requirements/postgresql.py3.txt`). + ## [0.34-a0 through 0.34-a0-958-gd0e9a9ddd5] Major changes: diff --git a/crawl-ref/source/webserver/README.md b/crawl-ref/source/webserver/README.md index e95c119b370..9f746f4fc7c 100644 --- a/crawl-ref/source/webserver/README.md +++ b/crawl-ref/source/webserver/README.md @@ -168,6 +168,10 @@ The server can be configured by modifying the file `config.py`. Most of the options are commented or should be self-evident. Suggestions: * Set uid and gid to a non-privileged user +* Optionally keep accounts and user settings in PostgreSQL instead of the + SQLite files (`userdb_backend = "postgresql"`, `userdb_dsn`); see + `config.py` and `requirements/postgresql.py3.txt`. This lets several + webtiles hosts share one account database. * Enable logging to a file in `logging_config` * If required, write a script that initializes user-specific data, like copying a default rc file if the user doesn't yet have one. You can have the script be diff --git a/crawl-ref/source/webserver/config.py b/crawl-ref/source/webserver/config.py index fd2d137dc16..7e969ae9ce9 100644 --- a/crawl-ref/source/webserver/config.py +++ b/crawl-ref/source/webserver/config.py @@ -98,6 +98,12 @@ # logging.getLogger('asyncio').setLevel(logging.DEBUG) password_db = "./webserver/passwd.db3" +# Accounts and user settings are stored in the SQLite files above by default. +# Large or multi-host deployments can keep them in PostgreSQL instead; the +# schema is created on first start and SQLite is not touched. Requires +# `pip install -r requirements/postgresql.py3.txt`. +# userdb_backend = "postgresql" +# userdb_dsn = "postgresql://webtiles:secret@db.example.org/webtiles" # Uncomment and change if you want this db somewhere separate from the # password_db location. #settings_db = "./webserver/user_settings.db3" diff --git a/crawl-ref/source/webserver/requirements.in/postgresql.txt b/crawl-ref/source/webserver/requirements.in/postgresql.txt new file mode 100644 index 00000000000..c37f3d81dd0 --- /dev/null +++ b/crawl-ref/source/webserver/requirements.in/postgresql.txt @@ -0,0 +1,2 @@ +-r base.txt +psycopg[binary] diff --git a/crawl-ref/source/webserver/requirements/postgresql.py3.txt b/crawl-ref/source/webserver/requirements/postgresql.py3.txt new file mode 100644 index 00000000000..ac9da8073bf --- /dev/null +++ b/crawl-ref/source/webserver/requirements/postgresql.py3.txt @@ -0,0 +1,7 @@ +# +# Optional: only needed with `userdb_backend = "postgresql"`. Install with +# pip install -r requirements/postgresql.py3.txt +# (keep in sync with requirements.in/postgresql.txt) +# +-r base.py3.txt +psycopg[binary]==3.3.4 diff --git a/crawl-ref/source/webserver/webtiles/config.py b/crawl-ref/source/webserver/webtiles/config.py index 26701674093..f8ff4336fd3 100644 --- a/crawl-ref/source/webserver/webtiles/config.py +++ b/crawl-ref/source/webserver/webtiles/config.py @@ -80,6 +80,10 @@ def do_early_logging(): 'max_passwd_length': 20, 'allow_password_reset': False, 'admin_password_reset': False, + # 'sqlite' (the password_db/settings_db files) or 'postgresql' + # (userdb_dsn; requires psycopg, see requirements/postgresql.py3.txt) + 'userdb_backend': 'sqlite', + 'userdb_dsn': None, 'crypt_algorithm': "broken", # should this be the default?? 'crypt_salt_length': 16, 'login_token_lifetime': 7, # Days; set to <= 0 to disable @@ -586,6 +590,12 @@ def validate(): raise ValueError("Webtiles config: malformed ban list ('%s')" % repr(get('banned'))) + backend = get('userdb_backend') + if backend not in ('sqlite', 'postgresql'): + raise ValueError("Webtiles config: unknown userdb_backend '%s'" % backend) + if backend == 'postgresql' and not get('userdb_dsn'): + raise ValueError("Webtiles config: userdb_backend = 'postgresql' requires userdb_dsn") + # set up defaults that are conditioned on other values if not has_key('settings_db'): set('settings_db', os.path.join(os.path.dirname(get('password_db')), diff --git a/crawl-ref/source/webserver/webtiles/userdb.py b/crawl-ref/source/webserver/webtiles/userdb.py index cfa8b04dcdc..b22751b7452 100644 --- a/crawl-ref/source/webserver/webtiles/userdb.py +++ b/crawl-ref/source/webserver/webtiles/userdb.py @@ -61,6 +61,114 @@ def execute(self, *args, **kwargs): raise sqlite3.ProgrammingError("Database connection not initialized!") return contextlib.closing(self.conn.execute(*args, **kwargs)) + +def using_postgresql(): # type: () -> bool + """Opt-in PostgreSQL storage for accounts and settings. Defaults to the + SQLite files, so existing servers are unaffected unless they set + `userdb_backend = "postgresql"`.""" + return config.get('userdb_backend', 'sqlite') == 'postgresql' + + +_PG_REWRITES = [ + # userdb.py is written in SQLite's dialect; these rewrites are the whole + # difference for the queries it uses. Case-insensitive matching comes from + # citext columns instead of COLLATE NOCASE. + (re.compile(r"\s+COLLATE\s+(NOCASE|RTRIM)\b"), ""), + (re.compile(r"datetime\('now',\s*'-(\d+) hours'\)"), r"(now() - interval '\1 hours')"), + (re.compile(r"datetime\('now'\)"), "now()"), +] + + +def pg_sql(sql): # type: (str) -> str + """Rewrite one of this module's SQLite queries for PostgreSQL.""" + for pattern, replacement in _PG_REWRITES: + sql = pattern.sub(replacement, sql) + sql = sql.replace("?", "%s") + if sql.lstrip().startswith("INSERT OR REPLACE INTO mutesettings"): + sql = " ".join(sql.split()) # normalise whitespace so the suffix attaches cleanly + sql = sql.replace("INSERT OR REPLACE INTO", "INSERT INTO") + sql = sql.rstrip(";") + " ON CONFLICT (username) DO UPDATE SET mutelist = EXCLUDED.mutelist;" + return sql + + +class _pg_cursor(object): + """Cursor wrapper that applies pg_sql() to every statement.""" + def __init__(self, cursor): + self._cursor = cursor + + def execute(self, sql, parameters=()): + return self._cursor.execute(pg_sql(sql), parameters) + + def __getattr__(self, name): + return getattr(self._cursor, name) + + def __iter__(self): + return iter(self._cursor) + + +class crawl_pg_db(crawl_db): + """crawl_db over a PostgreSQL connection (psycopg 3). + + Same interface as crawl_db: `execute()` returns a closing cursor, and the + object as a context manager wraps a transaction. The connection runs in + autocommit mode so single statements outside `with db:` never leave a + transaction open. psycopg is imported lazily; it is only required when + this backend is selected (see requirements/postgresql.py3.txt).""" + def __init__(self, dsn): # type: (str) -> None + self.name = dsn + self.conn = None + self._transaction = None + if dsn: + import psycopg # optional dependency + self.conn = psycopg.connect(dsn, autocommit=True) + + def cursor(self): + if not self.conn: + raise sqlite3.ProgrammingError("Database connection not initialized!") + return contextlib.closing(_pg_cursor(self.conn.cursor())) + + def __enter__(self): + if not self.conn: + raise sqlite3.ProgrammingError("Database connection not initialized!") + self._transaction = self.conn.transaction() + self._transaction.__enter__() + return self + + def __exit__(self, *args): + transaction, self._transaction = self._transaction, None + return transaction.__exit__(*args) + + def execute(self, sql, parameters=()): + if not self.conn: + raise sqlite3.ProgrammingError("Database connection not initialized!") + return contextlib.closing(self.conn.execute(pg_sql(sql), parameters)) + + +# PostgreSQL schemas. citext gives the case-insensitive matching and unique +# username that the SQLite schema gets from COLLATE NOCASE. +pg_settings_schema = """ + CREATE TABLE IF NOT EXISTS mutesettings ( + username CITEXT PRIMARY KEY NOT NULL, + mutelist TEXT DEFAULT '' + ); +""" +pg_user_schema = """ + CREATE EXTENSION IF NOT EXISTS citext; + CREATE TABLE IF NOT EXISTS dglusers ( + id SERIAL PRIMARY KEY, + username CITEXT UNIQUE, + email CITEXT, + env TEXT, + password TEXT, + flags INTEGER + ); + CREATE TABLE IF NOT EXISTS recovery_tokens ( + token TEXT PRIMARY KEY, + token_time TIMESTAMPTZ, + user_id INTEGER NOT NULL REFERENCES dglusers(id) + ); +""" + def create_settings_db(filename): # type: () -> None # note: when this behavior was converted from muting to blocking, the db # names were left as-is @@ -76,6 +184,11 @@ def create_settings_db(filename): # type: () -> None def ensure_settings_db_exists(quiet=False): + if using_postgresql(): + db = crawl_pg_db(config.get('userdb_dsn')) + with db: + db.execute(pg_settings_schema) + return db dbname = config.get('settings_db') if not os.path.exists(dbname): if not quiet: @@ -120,6 +233,11 @@ def create_user_db(filename): def ensure_user_db_exists(quiet=False): # type: () -> None + if using_postgresql(): + db = crawl_pg_db(config.get('userdb_dsn')) + with db: + db.execute(pg_user_schema) + return db dbname = config.get('password_db') if not os.path.exists(dbname): if not quiet: @@ -135,6 +253,8 @@ def ensure_user_db_exists(quiet=False): # type: () -> None def upgrade_user_db(): # type: () -> None """Automatically upgrades the database.""" global user_db, recovery_schema + if using_postgresql(): + return # the schema is created with IF NOT EXISTS # possibly CREATE .. IF NOT EXISTS would be more idiomatic sql... with user_db.cursor() as c: query = "SELECT name FROM sqlite_master WHERE type='table' or type='index';" diff --git a/crawl-ref/source/webserver/webtiles/userdb_test.py b/crawl-ref/source/webserver/webtiles/userdb_test.py new file mode 100644 index 00000000000..169f0042c0e --- /dev/null +++ b/crawl-ref/source/webserver/webtiles/userdb_test.py @@ -0,0 +1,100 @@ +"""Tests for the user database backends. + +The SQLite suite (`UserDBTest`, defined next to the code in userdb.py) always +runs. The PostgreSQL suite reruns the same tests against a live server and is +skipped unless `WEBTILES_TEST_POSTGRESQL_DSN` is set, e.g. + + WEBTILES_TEST_POSTGRESQL_DSN=postgresql://user:pw@localhost/webtiles pytest + +It drops and recreates the webtiles tables in that database. +""" +import os +import unittest + +from webtiles import config, userdb +from webtiles.userdb import UserDBTest # noqa: F401 -- collected by pytest + +PG_DSN = os.environ.get("WEBTILES_TEST_POSTGRESQL_DSN") + + +class PostgresSQLTranslationTest(unittest.TestCase): + """The SQLite dialect used throughout userdb.py is rewritten for + PostgreSQL at execute time; these pin the rewrite rules.""" + + def test_placeholders(self): + self.assertEqual(userdb.pg_sql("SELECT 1 FROM t WHERE a=? AND b=?"), + "SELECT 1 FROM t WHERE a=%s AND b=%s") + + def test_collations_are_dropped(self): + # username/email/token columns are citext on PostgreSQL, so the + # SQLite collation clauses have no equivalent and must vanish. + self.assertEqual(userdb.pg_sql("WHERE username=?\n COLLATE NOCASE"), + "WHERE username=%s") + self.assertEqual(userdb.pg_sql("WHERE t.token = ? COLLATE RTRIM"), + "WHERE t.token = %s") + + def test_datetime_functions(self): + self.assertEqual(userdb.pg_sql("VALUES (?,datetime('now'),?)"), + "VALUES (%s,now(),%s)") + self.assertEqual( + userdb.pg_sql("t.token_time > datetime('now','-12 hours')"), + "t.token_time > (now() - interval '12 hours')") + self.assertEqual( + userdb.pg_sql("SET token_time=datetime('now', '-2 hours')"), + "SET token_time=(now() - interval '2 hours')") + + def test_upsert(self): + sql = userdb.pg_sql("INSERT OR REPLACE INTO mutesettings (username, mutelist) " + "VALUES (?,?);") + self.assertEqual(sql, "INSERT INTO mutesettings (username, mutelist) VALUES (%s,%s) " + "ON CONFLICT (username) DO UPDATE SET mutelist = EXCLUDED.mutelist;") + + +@unittest.skipUnless(PG_DSN, "WEBTILES_TEST_POSTGRESQL_DSN not set") +class PostgresUserDBTest(UserDBTest): + """Exactly the SQLite test suite, against PostgreSQL.""" + + def setUp(self): + if not self.logging_init: + import webtiles.server as server + server.init_logging(config.get('logging_config')) + self.logging_init = True + self.config_shim = dict( + userdb_backend="postgresql", + userdb_dsn=PG_DSN, + # unused on this backend; set so tearDown of the base class is inert + settings_db="./unittest_pg_unused_settings.db3", + password_db="./unittest_pg_unused_passwd.db3", + dgl_mode=True) + config.server_config = self.config_shim + self._drop_tables() + userdb.init_db_connections(quiet=True) + + def tearDown(self): + self._drop_tables() + userdb.user_db.close() + userdb.settings_db.close() + super().tearDown() + + def _drop_tables(self): + db = userdb.crawl_pg_db(PG_DSN) + try: + with db: + db.execute("DROP TABLE IF EXISTS recovery_tokens, mutesettings, dglusers") + finally: + db.close() + + def test_backend_selected(self): + self.assertIsInstance(userdb.user_db, userdb.crawl_pg_db) + self.assertIsInstance(userdb.settings_db, userdb.crawl_pg_db) + + def test_case_insensitive_username_is_unique(self): + self.assertIsNone(userdb.register_user("Test", "hunter2", "a@example.com")) + self.assertEqual(userdb.register_user("tEsT", "hunter2", "b@example.com"), + "User already exists!") + self.assertEqual(userdb.get_user_info("TEST").username, "Test") + + def test_blocklist_upsert(self): + userdb.set_blocklist("test", "alice") + userdb.set_blocklist("TEST", "alice bob") + self.assertEqual(userdb.get_blocklist("test"), "alice bob")