From ff1b70cfeaedb1fec7d9dca5f4c6a26a79535513 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Wed, 29 Jul 2026 01:09:56 -0700 Subject: [PATCH] feat: coordinate process-local invalidation --- README.md | 159 +++++- package.json | 1 + scripts/benchmark-invalidation.mjs | 334 ++++++++++++ scripts/test-package.mjs | 287 +++++++++- src/config.ts | 11 +- src/dialcache.ts | 204 ++++++- src/index.ts | 17 +- src/internal/invalidation-coordinator.ts | 120 +++++ src/internal/invalidation-event.ts | 187 +++++++ src/internal/local-cache.ts | 45 +- src/internal/local-invalidation.ts | 123 +++++ src/internal/redis-cache.ts | 92 +++- src/internal/redis-scripts.ts | 66 +++ src/invalidation.ts | 58 ++ src/node-redis.ts | 278 +++++++++- src/redis-client.ts | 29 + src/redis-protocol.ts | 7 + src/valkey-glide.ts | 40 +- test/dialcache-local-invalidation.test.ts | 619 ++++++++++++++++++++++ test/fake-redis.ts | 34 +- test/invalidation-event.test.ts | 356 +++++++++++++ test/local-invalidation.test.ts | 267 ++++++++++ test/node-redis-coordination.test.ts | 437 +++++++++++++++ test/redis-cluster.integration.test.ts | 116 +++- test/redis-real.integration.test.ts | 437 ++++++++++++++- test/valkey-glide.test.ts | 125 +++++ 26 files changed, 4386 insertions(+), 63 deletions(-) create mode 100644 scripts/benchmark-invalidation.mjs create mode 100644 src/internal/invalidation-coordinator.ts create mode 100644 src/internal/invalidation-event.ts create mode 100644 src/internal/local-invalidation.ts create mode 100644 src/invalidation.ts create mode 100644 test/dialcache-local-invalidation.test.ts create mode 100644 test/invalidation-event.test.ts create mode 100644 test/local-invalidation.test.ts create mode 100644 test/node-redis-coordination.test.ts diff --git a/README.md b/README.md index 7c692db..609308f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Codecov](https://codecov.io/gh/lan17/DialCache/branch/main/graph/badge.svg)](https://codecov.io/gh/lan17/DialCache) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/lan17/DialCache/badge)](https://scorecard.dev/viewer/?uri=github.com/lan17/DialCache) -Fine-grained TypeScript caching with explicit enabled contexts, request-local memoization, process-local and Redis TTL caching, stable key construction, runtime rollout controls, request coalescing, adapter-based observability, and Redis watermark-based targeted invalidation. +Fine-grained TypeScript caching with explicit enabled contexts, request-local memoization, process-local and Redis TTL caching, stable key construction, runtime rollout controls, request coalescing, adapter-based observability, and Redis watermark-based targeted invalidation with optional process-local coordination. ## Contents @@ -20,6 +20,7 @@ Fine-grained TypeScript caching with explicit enabled contexts, request-local me - [Request-local cache](#request-local-cache) · [Process-local cache](#process-local-cache) · [Redis-backed TTL cache](#redis-backed-ttl-cache) · [Remote read deadlines](#remote-read-deadlines-and-async-liveness) · [Serialization](#serialization) - [Cached-value ownership](#cached-value-ownership) - [Targeted invalidation and watermarks](#targeted-invalidation-and-watermarks) + - [Coordinating process-local invalidation](#coordinating-process-local-invalidation) - [Request coalescing](#request-coalescing) - [Fallback deadlines](#fallback-deadlines) · [Coalescing state](#coalescing-state) - [Metrics](#metrics) @@ -120,7 +121,7 @@ Use `cached(fn, options)` for an extracted, reusable function. The wrapped calla | `cacheKey` | yes | Selector over `fn`'s parameters; returns a bare id or `{ id, args }`. | | `defaultConfig` | no | `DialCacheKeyConfig` baseline policy that runtime config overlays field by field (see [Runtime config](#runtime-config-and-ramp-controls)). | | `serializer` | when the return type is not statically JSON-compatible | Per-function `Serializer` for Redis values (see [Serialization](#serialization)). | -| `trackForInvalidation` | no (default `false`) | Opts this use case's Redis entries into watermark-based targeted invalidation. | +| `trackForInvalidation` | no (default `false`) | Opts this use case into Redis watermark-based targeted invalidation and, when configured, coordinated process-local invalidation. | | `fallbackTimeoutMs` | no (default `60_000`) | Fallback deadline in milliseconds, at most 2,147,483,647; `null` disables it (see [Fallback deadlines](#fallback-deadlines)). | `cached()` validates `useCase` at registration: a duplicate within one `DialCache` instance throws `UseCaseIsAlreadyRegisteredError`. Both APIs reject the internal name `watermark` with `UseCaseNameIsReservedError`. @@ -183,7 +184,7 @@ const dialcache = new DialCache({ That produces Redis keys beginning with `users-api:...`, or `{users-api:...}` for invalidation-tracked values. `namespace` is DialCache's single cache-identity and key-partitioning setting: it participates in request-local, process-local, Redis, coalescing, deterministic ramp, invalidation, and metrics. It may not contain `{` or `}` because DialCache reserves those characters for Redis Cluster hash tags. Use a namespace to express any required application or environment separation, such as `production-users-api`. -- **`keyType` + `id` is the invalidation unit for tracked Redis entries.** `dialcache.invalidateRemote("user_id", "123", futureBufferMs)` writes one watermark for that user; any `trackForInvalidation` Redis entry with the same `keyType` and `id` is refreshed across all `args` variants when Redis is read. `invalidateRemote` does not evict existing request-local or process-local entries (see [Targeted invalidation](#targeted-invalidation-and-watermarks)), and untracked Redis entries do not consult the watermark. `useCase` identifies the individual cache (it's the metrics label and part of the stored key). +- **`keyType` + `id` is the invalidation unit for tracked entries.** `dialcache.invalidateRemote("user_id", "123", futureBufferMs)` writes one Redis watermark for that user; any `trackForInvalidation` Redis entry with the same `keyType` and `id` is refreshed across all `args` variants when Redis is read. Process-local entries remain unchanged by default, or can use the [optional coordinator](#coordinating-process-local-invalidation) to evict the same identity across processes. Request-local and untracked entries are never evicted. `useCase` identifies the individual cache (it's the metrics label and part of the stored key). - **`args` are part of the cache key** — different `args` produce different entries — but invalidation is by `id` only. - **Scalar key equality is string-based.** Runtime type is not an identity dimension: for matching surrounding dimensions, numeric `1`, string `"1"`, and bigint `1n` identify the same key; argument values `null` and `"null"` also match. `-0` matches `0`, and an `undefined` argument is omitted. If a deployment changes the logical meaning represented by a scalar, change an explicit identity dimension such as `keyType`, `useCase`, or an argument name/value. - **Non-key inputs** (for example a db handle) are parameters ignored by a `cacheKey` selector or values captured by a `getOrLoad()` loader. They still reach non-coalesced executions, but concurrent same-key cache misses share the leader's execution, so do not omit values like auth context, locale, or cancellation behavior unless sharing one result is correct. @@ -198,7 +199,7 @@ Instance-wide behavior is set through the `DialCache` constructor: | `DialCacheConfig` option | Default | Description | | --- | --- | --- | | `namespace` | `"urn"` | Logical cache namespace and first key component (see [Keys, ids, and extra dimensions](#keys-ids-and-extra-dimensions)). | -| `redis` | none | `{ client: DialCacheRedisClient, readTimeoutMs?: number }`; enables the Redis layer with a 50 ms default read deadline (see [Redis-backed TTL cache](#redis-backed-ttl-cache)). | +| `redis` | none | `{ client: DialCacheRedisClient, readTimeoutMs?: number }`; enables the Redis layer with a 50 ms default read deadline. A coordinated client may also provide `coordinator` for tracked process-local invalidation (see [Redis-backed TTL cache](#redis-backed-ttl-cache)). | | `localMaxSize` | `10_000` | Global process-local entry cap; `0` disables process-local storage. Nonnegative safe integer. | | `cacheConfigProvider` | none | Resolves runtime config per enabled invocation as a sparse overlay on the function's `defaultConfig`; `null` applies no overrides. | | `metrics` | disabled | A `DialCacheMetricsAdapter` (see [Metrics](#metrics)). | @@ -307,6 +308,8 @@ const dialcache = new DialCache({ localMaxSize: 25_000 }); The limit counts entries rather than estimating JavaScript object memory. Recently read entries stay resident ahead of less recently used entries when the limit is reached. +Process-local entries are not shared between instances and, by default, are not changed by `invalidateRemote`. Applications that need targeted invalidation without giving up this layer can opt into [coordinated process-local invalidation](#coordinating-process-local-invalidation). + ### Redis-backed TTL cache The Redis layer supports standalone Redis, Valkey, and Redis Cluster. Register DialCache's native node-redis scripts when creating the client, then pass that client to DialCache: @@ -373,9 +376,9 @@ Pass the same module namespace that created the client. DialCache uses its itself, so linked workspaces and applications with another installed GLIDE version cannot accidentally mix native script handles. -The application owns the complete Redis lifecycle. It creates and connects the underlying client and passes the semantic adapter to DialCache. During shutdown, stop starting DialCache-backed work and await every promise returned by a cached function, `getOrLoad()`, or `invalidateRemote()`, including calls still running fallbacks that may later write Redis. A read that crossed DialCache's wait deadline may still be active inside the client, so use client-native telemetry and shutdown controls to drain or terminate that work before disposing adapter-owned resources and closing the connection. DialCache only borrows `redis.client`; it has no close or drain method and never disposes or closes caller resources. +The application owns the complete Redis lifecycle. It creates and connects the underlying client and passes the semantic adapter to DialCache. During shutdown, stop starting DialCache-backed work and await every promise returned by a cached function, `getOrLoad()`, or `invalidateRemote()`, including calls still running fallbacks that may later write Redis. A read that crossed DialCache's wait deadline may still be active inside the client, so use client-native telemetry and shutdown controls to drain or terminate that work before disposing adapter-owned resources and closing the connection. DialCache only borrows `redis.client`; its `dispose()` method only detaches process-local invalidation coordination and never drains, disposes, or closes caller resources. -The node-redis adapter owns no additional resources, so the application closes the underlying node-redis client after draining work. The GLIDE adapter owns five native `Script` handles but not the wrapped connection. After outstanding operations finish, call its idempotent `dispose()` before closing GLIDE as shown above; disposal while an adapter operation is in flight throws rather than releasing a live script. +The node-redis command adapter owns no additional resources, so the application closes the underlying node-redis client after draining work. The optional node-redis invalidation coordinator owns its subscription and event listeners, but not its dedicated subscriber connection; see its [shutdown order](#coordinating-process-local-invalidation). The GLIDE adapter owns five eagerly-created native `Script` handles, plus a sixth created lazily if coordinated invalidation is used, but not the wrapped connection. After outstanding operations finish, call its idempotent `dispose()` before closing GLIDE as shown above; disposal while an adapter operation is in flight throws rather than releasing a live script. Node-redis computes each script's SHA, uses `EVALSHA`, and retries with `EVAL` after `NOSCRIPT`. Its cluster client routes scripts by their first key and performs that fallback on the selected shard. The GLIDE adapter uses GLIDE's native `Script` lifecycle and byte decoder; GLIDE routes scripts from their declared keys. Tracked reads are deliberately routed to primaries so a lagging replica cannot hide an invalidation watermark. @@ -393,7 +396,7 @@ Writes, invalidations, async `cacheConfigProvider` calls, and custom serializer #### Serialization -The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. It exchanges serialized values as `string | Buffer` and does not expose client commands or wire encodings. Distinct untracked/tracked read and write Lua sources, the invalidation source, and wire constants are available from `dialcache/redis-protocol`. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed payloads, unsupported encodings, and Lua reply-domain violations in logs. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. +The core Redis boundary is the client-agnostic `DialCacheRedisClient` interface. It exchanges serialized values as `string | Buffer` and does not expose client commands or wire encodings. Distinct untracked/tracked read and write Lua sources, legacy and coordinated invalidation sources, and wire constants are available from `dialcache/redis-protocol`. Custom adapters can throw the root-exported `DialCacheRedisPayloadError`, `DialCacheRedisPayloadEncodingError`, and `DialCacheRedisProtocolError` classes to distinguish malformed payloads, unsupported encodings, and Lua reply-domain violations in logs. DialCache records bounded `cache_read`, `cache_write`, or `invalidation` metrics by failure site. Redis values use a compact binary frame: @@ -477,7 +480,7 @@ const getUser = dialcache.cached( useCase: "GetMutableUser", cacheKey: (userId) => userId, trackForInvalidation: true, - // Strongly invalidated mutable data should disable request-local and process-local caching. + // Without coordinated local invalidation, keep mutable data out of both in-memory layers. defaultConfig: new DialCacheKeyConfig({ ttlSec: { [CacheLayer.REMOTE]: 300 }, ramp: { [CacheLayer.REMOTE]: 100 }, @@ -507,7 +510,130 @@ Size the buffer to cover the maximum expected negative clock skew between promot This is a timing contract rather than a cancellation or acquisition fence: the buffer prevents stale fallback results from passing that tracked Redis write only while the configured window remains active, and it does not force a fallback to read from an authoritative source. -Targeted invalidation is remote-only and enforced by Redis watermarks. `invalidateRemote` does not evict existing request-local or process-local entries. Strongly invalidated mutable data should disable request-local and process-local caching (or use a very short process-local TTL only when stale reads are acceptable). +Without a coordinator, targeted invalidation remains remote-only: `invalidateRemote` uses the legacy Redis script and leaves every in-memory entry alone. Strongly invalidated mutable data should therefore disable request-local and process-local caching, or use a very short process-local TTL only when stale reads are acceptable. + +### Coordinating process-local invalidation + +Coordinated invalidation is an opt-in extension of the same `trackForInvalidation` identity and `invalidateRemote(keyType, id, futureBufferMs)` call. It does not add a generation to cache keys or make local hits consult Redis. Instead, one process-wide coordinator receives Redis Pub/Sub events and synchronously fans them out to its `DialCache` instances: + +1. The process calling `invalidateRemote` first installs a provisional monotonic fence and scans each attached local LRU, evicting tracked entries whose namespace, `keyType`, and `id` match. This includes every `useCase` and `args` variant for that identity, but not untracked entries or neighboring ids. +2. One Redis Lua command advances the watermark, derives its effective remaining window from Redis time, and publishes a versioned event. The returned event immediately extends the origin process's fence, and the origin's subscription normally receives the published echo as well. Subscribed peer processes apply the same Redis-derived duration to their monotonic clocks. +3. A tracked local miss captures a publication permit. Immediately before an async result enters the LRU, DialCache checks that no matching fence or coordinator health transition occurred since the miss. This prevents a Redis hit, fallback, or serializer that was already in flight from restoring an invalidated local entry during the buffer. + +The local key format and normal lookup path are unchanged, so a healthy steady-state local hit has no Pub/Sub, Redis, or fence-map work. The invalidation path does the more expensive work. A peer normally scans its current LRU once for the Pub/Sub event. The origin normally scans three times: once for the provisional signal, once for the authoritative event returned by Redis, and once for its Pub/Sub echo. The returned event and echo can arrive in either order. + +Every signal scans deliberately, even when two events carry an equal watermark. Equal watermarks can represent separate mutations, and an entry could have been inserted between them. Suppressing the later scan would leave that entry readable. Each scan is bounded by `localMaxSize`; multiply scan capacity by roughly three for origin-side sizing. Fences are bounded by the same limit; if it is exhausted, DialCache conservatively replaces them with one global deadline rather than growing memory without limit. + +The versioned Pub/Sub payload has a hard limit of `MAX_REDIS_INVALIDATION_EVENT_BYTES` (16 KiB), exported from `dialcache/redis-protocol`. The limit applies to the UTF-8 byte length of the complete JSON event, including `namespace`, `keyType`, `id`, timestamps, field names, escaping, and framing. The bundled coordinated Lua script constructs and checks the event before changing the watermark or publishing, so an oversized identity rejects the invalidation without a Redis mutation. Custom protocol adapters must enforce the same pre-mutation limit. + +Use a caller-owned, connected node-redis command client and a separate, dedicated standalone subscriber. One coordinator can be shared by all `DialCache` instances with the same namespace in a process: + +```ts +import { createClient } from "redis"; +import { CacheLayer, DialCache, DialCacheKeyConfig } from "dialcache"; +import { + createNodeRedisDialCacheClient, + createNodeRedisDialCacheInvalidationCoordinator, + dialcacheRedisScripts, +} from "dialcache/node-redis"; + +const namespace = "users-api"; +const commandClient = createClient({ + url: process.env.REDIS_URL, + scripts: dialcacheRedisScripts, + disableOfflineQueue: true, +}); +const subscriber = createClient({ + url: process.env.REDIS_URL, + disableOfflineQueue: true, +}); +const reportRedisError = (error: Error) => { + logger.error("Redis client error", error); +}; +commandClient.on("error", reportRedisError); +subscriber.on("error", reportRedisError); +await Promise.all([commandClient.connect(), subscriber.connect()]); + +const coordinator = await createNodeRedisDialCacheInvalidationCoordinator( + subscriber, + { namespace }, +); +const dialcache = new DialCache({ + namespace, + redis: { + client: createNodeRedisDialCacheClient(commandClient), + coordinator, + }, +}); +const USER_INVALIDATION_BUFFER_MS = 5_000; + +const getUserLocally = dialcache.cached( + (userId: string) => db.fetchUser(userId), + { + keyType: "user_id", + useCase: "GetUserLocally", + cacheKey: (userId) => userId, + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.LOCAL]: 60, [CacheLayer.REMOTE]: 300 }, + ramp: { [CacheLayer.LOCAL]: 100, [CacheLayer.REMOTE]: 100 }, + }), + }, +); + +await updateUser("123", patch); +await dialcache.invalidateRemote("user_id", "123", USER_INVALIDATION_BUFFER_MS); +``` + +The helper requires the subscriber to be connected, ready, dedicated, and not already in Pub/Sub mode. It owns only its subscription and event listeners; it never creates, connects, or closes the subscriber. Node-redis requires the application to install error listeners before `connect()`; retain those application-owned listeners until after each client closes because coordinator disposal removes only the helper's listener. The namespace must exactly match the `DialCache` namespace. Budget one extra long-lived Redis connection for each process-wide coordinator (normally one per process and namespace). + +During shutdown, stop new work and drain all cached calls and invalidations first. Then call `dispose()` on every attached `DialCache`, dispose the shared coordinator, close the subscriber, and finally close the command connection: + +```ts +for (const cache of dialcacheInstances) { + cache.dispose(); +} +await coordinator.dispose(); +await subscriber.quit(); +await commandClient.quit(); +``` + +`DialCache.dispose()` is synchronous and permanently detaches only that instance; it does not close Redis or the shared coordinator. Coordinator disposal is idempotent, unsubscribes, and detaches its health listeners without closing the subscriber. + +#### Delivery and concurrency guarantees + +[Redis Pub/Sub is at-most-once](https://redis.io/docs/latest/develop/pubsub/), best-effort notification, not a durable invalidation log. There is no replay, acknowledgement, subscriber quorum, or sequence/generation check. A publish succeeds even when Redis reports zero subscribers. A silently missed event or an invalidation sent by an uncoordinated/older publisher can therefore leave a local entry readable until its TTL expires. + +The coordinator treats observable uncertainty conservatively. It starts unavailable, becomes ready only after the subscription acknowledgement, and transitions to unavailable on subscriber errors, reconnects, connection end, a wrong channel, or a malformed event. Each health transition clears tracked local entries and changes the publication epoch; while unavailable, tracked values cannot be inserted into the local LRU. A post-resubscription `ready` transition clears again before allowing new tracked publications. This closes known disconnect gaps, but it cannot turn best-effort Pub/Sub into a strict delivery guarantee. Keep process-local caching disabled when the application requires strict post-invalidation freshness. + +Invalidation does not cancel a value already returned to a caller, revoke a request-local memo, or cancel an active process flight. Because process coalescing still happens before the cache lookup, a same-key caller arriving after invalidation can join a leader that started earlier; it and followers already sharing that leader receive the leader's result. The final publication check prevents that result from repopulating the process-local cache while the fence is active, and the Redis watermark independently guards its tracked Redis write. Size `futureBufferMs` to cover the complete stale-work interval; work that finishes after an undersized fence can still publish. + +If the Redis operation rejects, `invalidateRemote` rejects as before, while the origin process conservatively retains its provisional local eviction and fence. Treat the remote outcome as ambiguous: Redis Lua does not roll back commands completed before a later runtime error, and a client failure after dispatch also cannot prove whether the script ran. In particular, a watermark can advance even if `PUBLISH` is then denied. Peers may not receive that event, so alert and retry according to the application's mutation protocol rather than treating rejection as “nothing changed.” + +Request-local memoization is deliberately excluded because it is an already-observed snapshot owned by the outermost `enable()` scope. If a mutation must be visible later in the same scope, disable request-local caching for that use case or cross a new request boundary. Untracked process-local values are also intentionally unaffected. + +#### Redis Cluster, clients, and ACLs + +The coordinated Lua script has one watermark key, so a node-redis or GLIDE cluster command client routes it by the same hash tag used for tracked values. The event uses ordinary `PUBLISH`/`SUBSCRIBE`, not sharded Pub/Sub. For node-redis, keep the cluster command client for cache commands and connect the dedicated standalone subscriber to a cluster node or stable endpoint that can reconnect across topology changes. The node-redis Cluster facade is not accepted as the subscriber because it does not expose the ready/reconnect lifecycle required by the fail-safe health contract. Ordinary cluster Pub/Sub propagates across nodes, but its cluster-wide fan-out makes invalidation traffic a capacity input; verify the behavior and reconnect path of any managed Redis proxy or service. + +The GLIDE adapter supports the publisher side and lazily allocates the coordinated script when first used. DialCache does not provide a first-party GLIDE subscriber because its available callback/lifecycle surface cannot establish the required health transitions. A GLIDE command client can be paired with the node-redis coordinator above, or with an application-owned implementation of `DialCacheInvalidationCoordinator` that satisfies the same synchronous fan-out and health contract. + +Feature-specific ACL requirements are: + +- The command connection keeps the existing key/script permissions and adds permission to `PUBLISH` on the deterministic channel returned by `redisInvalidationChannel(namespace)` from `dialcache/redis-protocol` (`dialcache:invalidation:v1:users-api` in the example above). +- The subscriber needs `SUBSCRIBE` and `UNSUBSCRIBE` plus permission for that exact channel where [channel ACLs](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) are enforced. +- Both connections still need the authentication, handshake, client-metadata, and health commands required by the selected Redis client. Redis versions and managed services differ, so validate the least-privilege rules in the target environment. + +#### Safe rollout + +Do not enable tracked process-local caching in a mixed fleet: older publishers use the legacy watermark-only script, and older readers cannot receive events. A safe rollout is: + +1. Ramp the process-local layer to zero for mutable tracked use cases across the fleet. Restart pods or wait out the longest old local TTL before relying on that state. +2. Before enabling coordinated publishing, validate the fleet's worst-case identity sizes against the 16 KiB full-event limit. Count UTF-8 JSON bytes rather than characters; an identity that is valid in a Redis key can still make the event too large. +3. Deploy the coordinated script adapter, dedicated subscribers, and coordinators everywhere while local caching remains off. Verify ready/unavailable logs, reconnect behavior, Pub/Sub traffic, fallback load, and every mutation path's awaited `invalidateRemote` call. +4. After every publisher and reader is coordinated, ramp the process-local layer up gradually and watch invalidation latency, LRU scan cost, Redis/cluster Pub/Sub traffic, and source fallback load. + +For rollback, ramp tracked local caching back to zero first and leave it off until old entries have expired or pods have restarted; only then remove coordinators or roll back publishers. Omitting `redis.coordinator` always preserves the original uncoordinated behavior and public invalidation call. ## Request coalescing @@ -624,7 +750,7 @@ The Prometheus adapter emits: | `dialcache_miss_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache misses | | `dialcache_disabled_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `reason` | Cache skips (`context`, `policy_disabled`, `invalid_ttl`, `invalid_ramp`, `ramped_down`, `config_error`) | | `dialcache_error_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `layer`, `error`, `in_fallback` | Cache/fallback errors classified by a bounded failure site | -| `dialcache_invalidation_counter` | Counter | `cache_namespace`, `key_type`, `layer` | Invalidation calls for the layers touched | +| `dialcache_invalidation_counter` | Counter | `cache_namespace`, `key_type`, `layer` | Remote invalidation calls and coordinated local event applications | | `dialcache_coalesced_counter` | Counter | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests split by `request_local` or `process` scope | | `dialcache_get_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache get latency in seconds | | `dialcache_fallback_timer` | Histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Elapsed time until the underlying function settles or timeout rejection is delivered | @@ -635,6 +761,8 @@ The Prometheus adapter emits: Every metric carries `cache_namespace`, including disabled-context, key-construction, coalescing, and invalidation paths that do not have a constructed key. Its value is `DialCacheConfig.namespace`, defaulting to `urn`. The `layer` label is `request_local`, `local` (process-local), or `remote`. Disabled-context, key-construction, and config-provider failures use `noop` because no cache layer was reached. The bounded `scope` label on `dialcache_coalesced_counter` distinguishes request-local from instance-scoped single-flight work. `scope="process"` coordinates calls only within one `DialCache` instance; separate instances in the same process do not share in-flight state. +For coordinated invalidation, `dialcache_invalidation_counter{layer="local"}` counts authoritative event applications per attached `DialCache` instance, not logical mutations or provisional origin fences. A healthy origin commonly increments it twice for one `invalidateRemote` call—once for the event returned by Redis and once for its Pub/Sub echo—while a peer commonly increments it once. Use the remote-layer series to count API calls; interpret the local-layer series as fan-out work. Datadog's `dialcache.invalidation.count` has the same semantics. + ### Datadog Install `hot-shots` separately, create the DogStatsD client your application owns, and pass it to the Datadog adapter: @@ -681,7 +809,7 @@ The Datadog adapter emits exact increments of `1` for counters and preserves sec | `dialcache.miss.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache misses | | `dialcache.disabled.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `reason` | Cache skips by bounded reason | | `dialcache.error.count` | Count | `cache_namespace`, `use_case`, `key_type`, `layer`, `error`, `in_fallback` | Cache/fallback errors by bounded failure site | -| `dialcache.invalidation.count` | Count | `cache_namespace`, `key_type`, `layer` | Invalidation calls for the layers touched | +| `dialcache.invalidation.count` | Count | `cache_namespace`, `key_type`, `layer` | Remote invalidation calls and coordinated local event applications | | `dialcache.coalesced.count` | Count | `cache_namespace`, `use_case`, `key_type`, `scope` | Coalesced requests by sharing scope | | `dialcache.get.duration` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Cache get latency in seconds | | `dialcache.fallback.duration` | Distribution or histogram | `cache_namespace`, `use_case`, `key_type`, `layer` | Elapsed time until the underlying function settles or timeout rejection is delivered | @@ -703,7 +831,7 @@ The `error` label reports where an operation failed rather than copying the thro | `cache_write` | A local-cache or Redis write failed | | `serialization_load` | Deserializing a Redis payload failed | | `serialization_dump` | Serializing a value for Redis failed | -| `invalidation` | Writing an invalidation watermark failed | +| `invalidation` | Writing or publishing an invalidation watermark failed | | `fallback` | The wrapped application function failed or exceeded its DialCache deadline | | `unknown` | Reserved for an otherwise unclassified future failure site | @@ -721,9 +849,14 @@ From a repository checkout, run the semantic microbenchmark after installing dep ```bash pnpm benchmark:request-local +pnpm benchmark:invalidation ``` -The command builds `dist` before reporting six scenarios: sequential request-local hits, sequential process-local hits, enabled bounded fallbacks, request-local coalescing fan-out, process coalescing fan-out, and remote-read-deadline coalescing fan-out. The benchmark is a maintainer tool and is not included in the published package. It asserts fallback counts, coalescing state, timer cleanup, and returned values but deliberately applies no timing threshold. Override its work sizes with `DIALCACHE_BENCH_ITERATIONS` and `DIALCACHE_BENCH_FANOUT`. +The request-local command reports six scenarios: sequential request-local hits, sequential process-local hits, enabled bounded fallbacks, request-local coalescing fan-out, process coalescing fan-out, and remote-read-deadline coalescing fan-out. Override its work sizes with `DIALCACHE_BENCH_ITERATIONS` and `DIALCACHE_BENCH_FANOUT`. + +The invalidation command compares healthy coordinated and unconfigured tracked process-local hits, measures one peer-event LRU scan at 1,000, 10,000, and 50,000 entries, and measures the origin's three-signal invalidation path at 10,000 entries. Override its work sizes with `DIALCACHE_BENCH_ITERATIONS`, `DIALCACHE_BENCH_ROUNDS`, `DIALCACHE_BENCH_SCAN_ITERATIONS`, and `DIALCACHE_BENCH_LARGE_OCCUPANCY`. + +Both commands build `dist` first. They are maintainer tools and are not included in the published package. They assert semantic results and fallback counts but deliberately apply no timing threshold. ### Releasing diff --git a/package.json b/package.json index c5cea76..5e6fa1c 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "LICENSE" ], "scripts": { + "benchmark:invalidation": "pnpm build && node scripts/benchmark-invalidation.mjs", "benchmark:request-local": "pnpm build && node scripts/benchmark-request-local.mjs", "build": "tsup src/index.ts src/datadog.ts src/node-redis.ts src/prometheus.ts src/redis-protocol.ts src/valkey-glide.ts --format esm,cjs --dts --clean", "check": "pnpm typecheck && pnpm test && pnpm build && pnpm test:package", diff --git a/scripts/benchmark-invalidation.mjs b/scripts/benchmark-invalidation.mjs new file mode 100644 index 0000000..28fb5e7 --- /dev/null +++ b/scripts/benchmark-invalidation.mjs @@ -0,0 +1,334 @@ +import assert from "node:assert/strict"; +import { performance } from "node:perf_hooks"; + +import { CacheLayer, DialCache, DialCacheKeyConfig } from "../dist/index.js"; + +class BenchmarkCoordinator { + state = "ready"; + #listeners = new Set(); + + constructor(namespace) { + this.namespace = namespace; + } + + addListener(listener) { + this.#listeners.add(listener); + listener.onStateChange(this.state); + return () => this.#listeners.delete(listener); + } + + invalidate(invalidation) { + for (const listener of this.#listeners) { + try { + listener.onInvalidation(invalidation); + } catch { + // Match the coordinator contract: one listener must not break fan-out. + } + } + } +} + +const silentLogger = { + debug: () => undefined, + error: () => undefined, + warn: () => undefined, +}; +const hitIterations = readPositiveInteger("DIALCACHE_BENCH_ITERATIONS", 50_000); +const hitRounds = readPositiveInteger("DIALCACHE_BENCH_ROUNDS", 7); +const scanIterations = readPositiveInteger("DIALCACHE_BENCH_SCAN_ITERATIONS", 200); +const scanOccupancies = [ + ...new Set([ + 1_000, + 10_000, + readPositiveInteger("DIALCACHE_BENCH_LARGE_OCCUPANCY", 50_000), + ]), +].sort((left, right) => left - right); + +const unconfiguredHits = await prepareLocalHitBenchmark(false); +const coordinatedHits = await prepareLocalHitBenchmark(true); +await unconfiguredHits.run(Math.min(hitIterations, 10_000)); +await coordinatedHits.run(Math.min(hitIterations, 10_000)); + +const hitSamples = new Map([ + [unconfiguredHits, []], + [coordinatedHits, []], +]); +for (let round = 0; round < hitRounds; round += 1) { + const order = round % 2 === 0 + ? [coordinatedHits, unconfiguredHits] + : [unconfiguredHits, coordinatedHits]; + for (const benchmark of order) { + hitSamples.get(benchmark).push(await benchmark.run(hitIterations)); + } +} +const hitResults = [unconfiguredHits, coordinatedHits].map((benchmark) => { + const samples = hitSamples.get(benchmark); + return { + scenario: benchmark.scenario, + operations: hitIterations, + samples: samples.length, + elapsedMs: median(samples), + fallbackCalls: benchmark.fallbackCalls(), + }; +}); +unconfiguredHits.dispose(); +coordinatedHits.dispose(); + +console.log("\nTracked process-local hit throughput"); +console.table( + hitResults.map(({ scenario, operations, samples, elapsedMs, fallbackCalls }) => ({ + scenario, + operations, + samples, + "median elapsed (ms)": elapsedMs.toFixed(2), + "median operations/sec": Math.round((operations / elapsedMs) * 1_000).toLocaleString("en-US"), + "fallback calls": fallbackCalls, + })), +); +const [unconfiguredResult, coordinatedResult] = hitResults; +console.log( + `Coordinated median elapsed-time delta: ${ + (((coordinatedResult.elapsedMs / unconfiguredResult.elapsedMs) - 1) * 100).toFixed(2) + }% (informational).`, +); + +const scanResults = []; +for (const occupancy of scanOccupancies) { + scanResults.push(await benchmarkInvalidationScan({ + scenario: "peer Pub/Sub event", + occupancy, + iterations: scanIterations, + signalSources: ["event"], + })); +} +scanResults.push(await benchmarkInvalidationScan({ + scenario: "origin provisional + returned + echo", + occupancy: 10_000, + iterations: scanIterations, + signalSources: ["provisional", "event", "event"], +})); + +console.log("\nTracked process-local invalidation scan cost"); +console.table( + scanResults.map(({ + scenario, + occupancy, + operations, + signalsPerOperation, + scans, + elapsedMs, + fallbackCalls, + }) => ({ + scenario, + "local entries": occupancy, + "logical invalidations": operations, + "signals / invalidation": signalsPerOperation, + "LRU scans": scans, + "elapsed (ms)": elapsedMs.toFixed(2), + "average invalidation (ms)": (elapsedMs / operations).toFixed(3), + "average scan (ms)": (elapsedMs / scans).toFixed(3), + "entries examined/sec": Math.round((occupancy * scans / elapsedMs) * 1_000) + .toLocaleString("en-US"), + "fallback calls": fallbackCalls, + })), +); + +console.log("Semantic assertions passed; elapsed times are informational and have no pass/fail threshold."); + +async function prepareLocalHitBenchmark(coordinated) { + // Keep every key/config input identical so this isolates coordinator state + // rather than string hashing, ramp assignment, or cache-key construction. + const namespace = "benchmark-local-hit"; + const coordinator = coordinated ? new BenchmarkCoordinator(namespace) : null; + const redisClient = coordinated ? createUnusedCoordinatedRedisClient() : null; + const dialcache = new DialCache({ + namespace, + logger: silentLogger, + ...(coordinator === null || redisClient === null + ? {} + : { redis: { client: redisClient, coordinator } }), + }); + let fallbackCalls = 0; + const getValue = dialcache.cached( + async (id) => { + fallbackCalls += 1; + return { id }; + }, + { + keyType: "benchmark_id", + useCase: "TrackedLocalHit", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnlyConfig(), + }, + ); + + const expected = await dialcache.enable(async () => await getValue("shared")); + + return { + scenario: coordinated ? "coordinated healthy local hits" : "unconfigured local hits", + async run(iterations) { + let elapsedMs = 0; + await dialcache.enable(async () => { + let actual = expected; + const start = performance.now(); + for (let index = 0; index < iterations; index += 1) { + actual = await getValue("shared"); + } + elapsedMs = performance.now() - start; + assert.strictEqual(actual, expected); + }); + assert.equal(fallbackCalls, 1, "process-local hits should execute the fallback once"); + assert.equal(redisClient?.calls ?? 0, 0, "local-only operations must not reach Redis"); + return elapsedMs; + }, + fallbackCalls: () => fallbackCalls, + dispose: () => dialcache.dispose(), + }; +} + +async function benchmarkInvalidationScan({ + scenario, + occupancy, + iterations, + signalSources, +}) { + const signalsPerOperation = signalSources.length; + const namespace = `benchmark-scan-${occupancy}-${signalsPerOperation}`; + const coordinator = new BenchmarkCoordinator(namespace); + const redisClient = createUnusedCoordinatedRedisClient(); + const dialcache = new DialCache({ + namespace, + logger: silentLogger, + localMaxSize: occupancy, + redis: { client: redisClient, coordinator }, + }); + let version = 1; + let fallbackCalls = 0; + const getValue = dialcache.cached( + async (id) => { + fallbackCalls += 1; + return { id, version }; + }, + { + keyType: "benchmark_id", + useCase: `InvalidationScan${occupancy}x${signalsPerOperation}`, + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnlyConfig(), + }, + ); + + await dialcache.enable(async () => { + for (let index = 0; index < occupancy; index += 1) { + await getValue(String(index)); + } + }); + assert.equal(fallbackCalls, occupancy, "benchmark setup should populate every local entry"); + + const missingInvalidation = { + namespace, + keyType: "benchmark_id", + id: "not-present", + remainingMs: 60_000, + source: "event", + }; + const invalidationSignals = signalSources.map((source) => ({ + ...missingInvalidation, + source, + })); + const start = performance.now(); + for (let index = 0; index < iterations; index += 1) { + for (const signal of invalidationSignals) { + coordinator.invalidate(signal); + } + } + const elapsedMs = performance.now() - start; + + version = 2; + const targetId = String(Math.floor(occupancy / 2)); + coordinator.invalidate({ + ...missingInvalidation, + id: targetId, + }); + const [firstAfterInvalidation, secondAfterInvalidation] = await dialcache.enable(async () => [ + await getValue(targetId), + await getValue(targetId), + ]); + assert.equal(firstAfterInvalidation.version, 2, "invalidation should evict the matching local value"); + assert.equal(secondAfterInvalidation.version, 2, "the local fence should prevent immediate republication"); + assert.equal( + fallbackCalls, + occupancy + 2, + "both calls inside an active local fence should execute the fallback", + ); + assert.equal(redisClient.calls, 0, "direct local invalidation fan-out must not reach Redis"); + + dialcache.dispose(); + return { + scenario, + occupancy, + operations: iterations, + signalsPerOperation, + scans: iterations * signalsPerOperation, + elapsedMs, + fallbackCalls, + }; +} + +function localOnlyConfig() { + return new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.LOCAL]: 60 }, + ramp: { [CacheLayer.LOCAL]: 100 }, + }); +} + +function createUnusedCoordinatedRedisClient() { + return { + calls: 0, + async read() { + this.calls += 1; + return null; + }, + async write() { + this.calls += 1; + return true; + }, + async invalidate() { + this.calls += 1; + }, + async invalidateAndPublish({ namespace, keyType, id, futureBufferMs }) { + this.calls += 1; + const redisNowMs = Date.now(); + return { + version: 1, + namespace, + keyType, + id, + effectiveWatermarkMs: String(redisNowMs + futureBufferMs), + redisNowMs: String(redisNowMs), + }; + }, + }; +} + +function readPositiveInteger(name, fallback) { + const raw = process.env[name]; + if (raw === undefined) { + return fallback; + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive safe integer`); + } + return value; +} + +function median(values) { + assert(values.length > 0); + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 + ? (sorted[middle - 1] + sorted[middle]) / 2 + : sorted[middle]; +} diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 4b9bf34..1a640d3 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -24,9 +24,19 @@ const rootConsumer = `import { type CoalescedMetricLabels, type CoalescingScope, type CoalescingState, + type CoordinatedRedisConfig, type DialCacheConfig, + type DialCacheCoordinatedRedisClient, + type DialCacheInvalidationCoordinator, + type DialCacheInvalidationCoordinatorListener, + type DialCacheInvalidationCoordinatorState, + type DialCacheInvalidationEventV1, + type DialCacheInvalidationIdentity, type DialCacheKeyInit, + type DialCacheLocalInvalidation, + type DialCacheLocalInvalidationSource, type DialCacheMetricsAdapter, + type DialCacheRedisConfig, type DialCacheRedisClient, type DisabledReason, type GetOrLoadOptions, @@ -34,6 +44,7 @@ const rootConsumer = `import { type MetricErrorKind, type ProcessCoalescingState, type RedisConfig, + type RedisCoordinatedInvalidationRequest, type RedisInvalidationRequest, type RedisReadContext, type RedisWriteRequest, @@ -41,8 +52,22 @@ const rootConsumer = `import { } from "dialcache"; // @ts-expect-error The unused MissingKeyConfigError class was removed instead of deprecated. import { MissingKeyConfigError } from "dialcache"; -import { createNodeRedisDialCacheClient } from "dialcache/node-redis"; -import { READ_CACHE_SCRIPT } from "dialcache/redis-protocol"; +import { createClient } from "redis"; +import { + createNodeRedisDialCacheClient, + createNodeRedisDialCacheInvalidationCoordinator, + dialcacheRedisScripts, + type DialCacheNodeRedisInvalidationCoordinator, + type DialCacheNodeRedisSubscriberClient, +} from "dialcache/node-redis"; +import { + INVALIDATE_AND_PUBLISH_CACHE_SCRIPT, + MAX_REDIS_INVALIDATION_EVENT_BYTES, + READ_CACHE_SCRIPT, + REDIS_INVALIDATION_EVENT_VERSION, + decodeRedisInvalidationEvent, + redisInvalidationChannel, +} from "dialcache/redis-protocol"; import { DatadogDialCacheMetrics, createDatadogDialCacheMetrics, @@ -259,8 +284,106 @@ const customRedisClient: DialCacheRedisClient = { write: async ({ value }) => typeof value === "string" || Buffer.isBuffer(value), invalidate: async () => undefined, }; +const coordinatedIdentity: DialCacheInvalidationIdentity = { + namespace: "consumer-cache", + keyType: "id", + id: "123", +}; +const coordinatedSource: DialCacheLocalInvalidationSource = "event"; +const coordinatedLocalInvalidation: DialCacheLocalInvalidation = { + ...coordinatedIdentity, + remainingMs: 1_000, + source: coordinatedSource, +}; +const coordinatedEvent: DialCacheInvalidationEventV1 = { + version: 1, + ...coordinatedIdentity, + effectiveWatermarkMs: String(Date.now() + 1_000), + redisNowMs: String(Date.now()), +}; +const coordinatedState: DialCacheInvalidationCoordinatorState = "ready"; +const coordinatorListeners = new Set(); +const coordinatedInvalidationCoordinator: DialCacheInvalidationCoordinator = { + namespace: coordinatedIdentity.namespace, + state: coordinatedState, + addListener(listener) { + coordinatorListeners.add(listener); + listener.onStateChange(coordinatedState); + return () => coordinatorListeners.delete(listener); + }, + invalidate(invalidation) { + for (const listener of coordinatorListeners) { + listener.onInvalidation(invalidation); + } + }, +}; +const coordinatedRedisClient: DialCacheCoordinatedRedisClient = { + ...customRedisClient, + async invalidateAndPublish(request: RedisCoordinatedInvalidationRequest) { + return { + version: REDIS_INVALIDATION_EVENT_VERSION, + namespace: request.namespace, + keyType: request.keyType, + id: request.id, + effectiveWatermarkMs: String(Date.now() + request.futureBufferMs), + redisNowMs: String(Date.now()), + }; + }, +}; +const coordinatedRedisConfig: CoordinatedRedisConfig = { + client: coordinatedRedisClient, + coordinator: coordinatedInvalidationCoordinator, +}; +const dialcacheRedisConfig: DialCacheRedisConfig = coordinatedRedisConfig; +// A coordinated-capable client remains usable without opting into local coordination. +const coordinatedClientWithoutCoordinator: RedisConfig = { client: coordinatedRedisClient }; +// @ts-expect-error A coordinator requires the atomic invalidate-and-publish client extension. +const legacyClientWithCoordinator: DialCacheRedisConfig = { + client: customRedisClient, + coordinator: coordinatedInvalidationCoordinator, +}; +interface ExtendedRedisConfig extends RedisConfig { + readonly applicationName: string; +} +class ImplementedRedisConfig implements RedisConfig { + readonly client = customRedisClient; +} +const extendedRedisConfig: ExtendedRedisConfig = { + client: customRedisClient, + applicationName: "consumer", +}; +const implementedRedisConfig: RedisConfig = new ImplementedRedisConfig(); +const coordinatedInvalidationRequest: RedisCoordinatedInvalidationRequest = { + ...coordinatedIdentity, + watermarkKey: "{consumer-cache:id:123}#watermark", + futureBufferMs: 1_000, + channel: redisInvalidationChannel(coordinatedIdentity.namespace), +}; +const decodedCoordinatedEvent: DialCacheInvalidationEventV1 = decodeRedisInvalidationEvent( + JSON.stringify(coordinatedEvent), + coordinatedIdentity, +); +const nodeRedisCommandClient = createClient({ scripts: dialcacheRedisScripts }); +const nodeRedisCoordinatedAdapter: DialCacheCoordinatedRedisClient = + createNodeRedisDialCacheClient(nodeRedisCommandClient); +const nodeRedisSubscriber = createClient(); +const nodeRedisSubscriberSurface: DialCacheNodeRedisSubscriberClient = nodeRedisSubscriber; +const nodeRedisCoordinatorPromise: Promise = + createNodeRedisDialCacheInvalidationCoordinator(nodeRedisSubscriber, { + namespace: coordinatedIdentity.namespace, + }); +const legacyStructuralNodeScriptClient = { + dialcacheRead: async () => null, + dialcacheReadTracked: async () => null, + dialcacheWrite: async () => 1, + dialcacheWriteTracked: async () => 1, + dialcacheInvalidate: async () => 1, +}; +const legacyStructuralNodeAdapter: DialCacheRedisClient = + createNodeRedisDialCacheClient(legacyStructuralNodeScriptClient); const cacheHasNoFlushAll: "flushAll" extends keyof DialCache ? false : true = true; const cacheHasNoClose: "close" extends keyof DialCache ? false : true = true; +const cacheHasDispose: "dispose" extends keyof DialCache ? true : false = true; const clientHasNoFlushAll: "flushAll" extends keyof DialCacheRedisClient ? false : true = true; type TrackedRedisWriteRequest = Extract; const trackedWriteHasNoWatermarkTtlFloor: "watermarkTtlFloorMs" extends keyof TrackedRedisWriteRequest @@ -361,7 +484,28 @@ void disabledOverlay; void metricErrorKinds; void unboundedErrorKind; void createNodeRedisDialCacheClient; +void createNodeRedisDialCacheInvalidationCoordinator; +void dialcacheRedisScripts.dialcacheInvalidateAndPublish; +void nodeRedisCommandClient; +void nodeRedisCoordinatedAdapter.invalidateAndPublish; +void nodeRedisSubscriberSurface.isReady; +void nodeRedisCoordinatorPromise; +void legacyStructuralNodeAdapter.invalidate; void READ_CACHE_SCRIPT; +void INVALIDATE_AND_PUBLISH_CACHE_SCRIPT; +void MAX_REDIS_INVALIDATION_EVENT_BYTES; +void coordinatedIdentity; +void coordinatedLocalInvalidation; +void coordinatedEvent; +void coordinatedInvalidationCoordinator; +void coordinatedRedisConfig; +void dialcacheRedisConfig; +void coordinatedClientWithoutCoordinator; +void legacyClientWithCoordinator; +void extendedRedisConfig; +void implementedRedisConfig; +void coordinatedInvalidationRequest; +void decodedCoordinatedEvent; void customRedisClient; const globalSerializer: Serializer = { dump: () => "global", @@ -376,6 +520,7 @@ cacheWithGlobalSerializer.cached(async (_id: string) => new Date(0), optionsFor( cacheWithGlobalSerializer.getOrLoad(async () => new Date(0), inlineOptionsFor("GlobalSerializerNeedsInlineTypedOverride")); void cacheHasNoFlushAll; void cacheHasNoClose; +void cacheHasDispose; void clientHasNoFlushAll; void trackedWriteHasNoWatermarkTtlFloor; void invalidationHasNoWatermarkTtlFloor; @@ -520,7 +665,7 @@ try { const nodeRedis = await import("dialcache/node-redis"); await import("dialcache/valkey-glide"); await import("dialcache/datadog"); -await import("dialcache/redis-protocol"); +const redisProtocol = await import("dialcache/redis-protocol"); const fallbackTimeoutError = new root.FallbackTimeoutError("PackageRuntime", 1000); if (!(fallbackTimeoutError instanceof root.DialCacheError) || fallbackTimeoutError.timeoutMs !== 1000) { throw new Error("The root ESM fallback-timeout error export is invalid"); @@ -559,6 +704,73 @@ try { throw new Error("The node-redis protocol error does not match the root ESM export"); } } +if ( + typeof nodeRedis.createNodeRedisDialCacheInvalidationCoordinator !== "function" + || typeof nodeRedis.dialcacheRedisScripts.dialcacheInvalidateAndPublish?.SCRIPT !== "string" + || redisProtocol.INVALIDATE_AND_PUBLISH_CACHE_SCRIPT + !== nodeRedis.dialcacheRedisScripts.dialcacheInvalidateAndPublish.SCRIPT + || redisProtocol.REDIS_INVALIDATION_EVENT_VERSION !== 1 + || redisProtocol.MAX_REDIS_INVALIDATION_EVENT_BYTES !== 16 * 1024 +) { + throw new Error("The packed ESM coordinated invalidation exports are invalid"); +} +const protocolNamespace = "packed-esm"; +const protocolEvent = redisProtocol.decodeRedisInvalidationEvent(JSON.stringify({ + version: redisProtocol.REDIS_INVALIDATION_EVENT_VERSION, + namespace: protocolNamespace, + keyType: "id", + id: "123", + effectiveWatermarkMs: "1001", + redisNowMs: "1", +})); +if ( + protocolEvent.id !== "123" + || redisProtocol.redisInvalidationChannel(protocolNamespace) + !== "dialcache:invalidation:v1:packed-esm" +) { + throw new Error("The packed ESM invalidation event helpers are invalid"); +} +let coordinatedListenerRemoved = false; +const packageCoordinator = { + namespace: protocolNamespace, + state: "ready", + addListener(listener) { + listener.onStateChange("ready"); + return () => { + coordinatedListenerRemoved = true; + }; + }, + invalidate() {}, +}; +const packageLegacyClient = { + async read() { return null; }, + async write() { return true; }, + async invalidate() {}, +}; +new root.DialCache({ namespace: "packed-legacy", redis: { client: packageLegacyClient } }); +const packageCoordinatedCache = new root.DialCache({ + namespace: protocolNamespace, + redis: { + client: { + ...packageLegacyClient, + async invalidateAndPublish(request) { + return { + version: 1, + namespace: request.namespace, + keyType: request.keyType, + id: request.id, + effectiveWatermarkMs: "1", + redisNowMs: "1", + }; + }, + }, + coordinator: packageCoordinator, + }, +}); +packageCoordinatedCache.dispose(); +if (!coordinatedListenerRemoved) { + throw new Error("The packed ESM DialCache did not detach from its invalidation coordinator"); +} if ("MissingKeyConfigError" in root) { throw new Error("The removed MissingKeyConfigError class must not be exported from the root ESM entry"); } @@ -616,7 +828,7 @@ if (inlineCalls !== 1 || inlineSecond !== inlineFirst) { const nodeRedis = require("dialcache/node-redis"); require("dialcache/valkey-glide"); require("dialcache/datadog"); -require("dialcache/redis-protocol"); +const redisProtocol = require("dialcache/redis-protocol"); const fallbackTimeoutError = new root.FallbackTimeoutError("PackageRuntime", 1000); if (!(fallbackTimeoutError instanceof root.DialCacheError) || fallbackTimeoutError.timeoutMs !== 1000) { throw new Error("The root CommonJS fallback-timeout error export is invalid"); @@ -657,6 +869,73 @@ try { throw new Error("The node-redis protocol error does not match the root CommonJS export"); } } +if ( + typeof nodeRedis.createNodeRedisDialCacheInvalidationCoordinator !== "function" + || typeof nodeRedis.dialcacheRedisScripts.dialcacheInvalidateAndPublish?.SCRIPT !== "string" + || redisProtocol.INVALIDATE_AND_PUBLISH_CACHE_SCRIPT + !== nodeRedis.dialcacheRedisScripts.dialcacheInvalidateAndPublish.SCRIPT + || redisProtocol.REDIS_INVALIDATION_EVENT_VERSION !== 1 + || redisProtocol.MAX_REDIS_INVALIDATION_EVENT_BYTES !== 16 * 1024 +) { + throw new Error("The packed CommonJS coordinated invalidation exports are invalid"); +} +const protocolNamespace = "packed-cjs"; +const protocolEvent = redisProtocol.decodeRedisInvalidationEvent(JSON.stringify({ + version: redisProtocol.REDIS_INVALIDATION_EVENT_VERSION, + namespace: protocolNamespace, + keyType: "id", + id: "123", + effectiveWatermarkMs: "1001", + redisNowMs: "1", +})); +if ( + protocolEvent.id !== "123" + || redisProtocol.redisInvalidationChannel(protocolNamespace) + !== "dialcache:invalidation:v1:packed-cjs" +) { + throw new Error("The packed CommonJS invalidation event helpers are invalid"); +} +let coordinatedListenerRemoved = false; +const packageCoordinator = { + namespace: protocolNamespace, + state: "ready", + addListener(listener) { + listener.onStateChange("ready"); + return () => { + coordinatedListenerRemoved = true; + }; + }, + invalidate() {}, +}; +const packageLegacyClient = { + async read() { return null; }, + async write() { return true; }, + async invalidate() {}, +}; +new root.DialCache({ namespace: "packed-legacy", redis: { client: packageLegacyClient } }); +const packageCoordinatedCache = new root.DialCache({ + namespace: protocolNamespace, + redis: { + client: { + ...packageLegacyClient, + async invalidateAndPublish(request) { + return { + version: 1, + namespace: request.namespace, + keyType: request.keyType, + id: request.id, + effectiveWatermarkMs: "1", + redisNowMs: "1", + }; + }, + }, + coordinator: packageCoordinator, + }, +}); +packageCoordinatedCache.dispose(); +if (!coordinatedListenerRemoved) { + throw new Error("The packed CommonJS DialCache did not detach from its invalidation coordinator"); +} if ("MissingKeyConfigError" in root) { throw new Error("The removed MissingKeyConfigError class must not be exported from the root CommonJS entry"); } diff --git a/src/config.ts b/src/config.ts index aaf3a4a..c6c1ff5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,6 +1,6 @@ import type { DialCacheKey } from "./key.js"; import type { DialCacheMetricsAdapter } from "./metrics.js"; -import type { RedisConfig } from "./internal/redis-cache.js"; +import type { DialCacheRedisConfig } from "./internal/redis-cache.js"; import { assertValidDeadlineMs } from "./internal/deadline.js"; export enum CacheLayer { @@ -110,6 +110,13 @@ export interface DialCacheConfig { * Zero disables local storage. Defaults to 10,000. */ readonly localMaxSize?: number; - readonly redis?: RedisConfig; + /** + * Optional caller-owned Redis command client and process-local invalidation + * coordinator. Without a coordinator, tracked invalidation remains + * remote-only. With one, the client must support the coordinated + * invalidate-and-publish operation and the coordinator namespace must match + * this configuration's namespace. + */ + readonly redis?: DialCacheRedisConfig; readonly metrics?: DialCacheMetricsAdapter; } diff --git a/src/dialcache.ts b/src/dialcache.ts index af92c4f..c297763 100644 --- a/src/dialcache.ts +++ b/src/dialcache.ts @@ -10,7 +10,12 @@ import { } from "./config.js"; import { DialCacheContext, getOrCreateRequestLocalCache, type RequestLocalCache } from "./context.js"; import { FallbackTimeoutError, UseCaseIsAlreadyRegisteredError, UseCaseNameIsReservedError } from "./errors.js"; -import { DialCacheKey, assertValidNamespace, normalizeArgs } from "./key.js"; +import type { + DialCacheInvalidationCoordinator, + DialCacheInvalidationCoordinatorState, + DialCacheLocalInvalidation, +} from "./invalidation.js"; +import { DialCacheKey, assertValidNamespace, invalidationPrefix, normalizeArgs } from "./key.js"; import { NO_CACHE_LAYER, REQUEST_LOCAL_CACHE_LAYER, @@ -21,6 +26,7 @@ import { type MetricErrorKind, type MetricLayer, } from "./metrics.js"; +import { DialCacheRedisProtocolError } from "./redis-client.js"; import type { Serializer } from "./serializer.js"; import type { CacheGetResult, RemoteCacheGetResult } from "./internal/cache-result.js"; import { MAX_TIMER_DELAY_MS, withMonotonicDeadline } from "./internal/deadline.js"; @@ -29,7 +35,15 @@ import { isSupportedCacheTtlSec, MAX_CACHE_TTL_SEC, } from "./internal/duration.js"; +import { + isValidLocalInvalidation, + localInvalidationFromEvent, +} from "./internal/invalidation-event.js"; import { LocalCache } from "./internal/local-cache.js"; +import { + LocalInvalidationState, + type LocalPublicationPermit, +} from "./internal/local-invalidation.js"; import { RedisCache } from "./internal/redis-cache.js"; import { fetchKeyConfig, @@ -200,6 +214,9 @@ export class DialCache { private readonly logger: Logger; private readonly redisCache: RedisCache | null; private readonly metrics: DialCacheMetricsAdapter | null; + private readonly invalidationCoordinator: DialCacheInvalidationCoordinator | null; + private readonly localInvalidation: LocalInvalidationState | null; + private removeInvalidationListener: (() => void) | null = null; private readonly processFlights = new Map(); private activeProcessFollowers = 0; @@ -226,6 +243,16 @@ export class DialCache { this.logger = safeLogger(config.logger ?? defaultLogger); this.metrics = safeMetrics(config.metrics ?? null); this.localCache = new LocalCache(this.configProvider, localMaxSize); + const invalidationCoordinator = config.redis?.coordinator ?? null; + if ( + invalidationCoordinator !== null + && invalidationCoordinator.namespace !== namespace + ) { + throw new TypeError( + "Redis invalidation coordinator namespace must match DialCache namespace", + ); + } + this.invalidationCoordinator = invalidationCoordinator; this.redisCache = config.redis === undefined ? null @@ -234,6 +261,16 @@ export class DialCache { redis: config.redis, metrics: this.metrics, }); + this.localInvalidation = + invalidationCoordinator === null || localMaxSize === 0 + ? null + : new LocalInvalidationState(this.localCache, localMaxSize); + if (this.localInvalidation !== null) { + this.removeInvalidationListener = invalidationCoordinator?.addListener({ + onInvalidation: (invalidation) => this.applyLocalInvalidation(invalidation), + onStateChange: (state, error) => this.applyInvalidationState(state, error), + }) ?? null; + } } enable(fn: () => Awaitable): Promise { @@ -256,6 +293,25 @@ export class DialCache { return this.context.isEnabled(); } + /** + * Permanently detach this instance from its invalidation coordinator. + * Coordinated tracked process-local storage remains disabled afterward; + * Redis, request-local, and untracked behavior remain available. This does + * not dispose or close caller-owned Redis clients or the shared coordinator. + */ + dispose(): void { + const removeListener = this.removeInvalidationListener; + if (removeListener === null) { + return; + } + this.removeInvalidationListener = null; + // Establish the terminal safety state before invoking caller-owned cleanup. + // A broken removal callback may leak a listener, but it cannot re-enable + // tracked local storage or allow late events to mutate this instance. + this.applyInvalidationState("disposed"); + removeListener(); + } + /** Returns exact process-scoped single-flight state for this instance. */ getCoalescingState(): CoalescingState { const oldestFlight = this.processFlights.values().next().value as ProcessFlight | undefined; @@ -371,8 +427,19 @@ export class DialCache { /** * Writes a remote invalidation watermark for Redis-tracked entries. * - * This does not synchronously evict local cache hits or untracked Redis values. - * Call it only after the source mutation commits. + * With an optional Redis invalidation coordinator, this also synchronously + * evicts and fences matching tracked process-local entries in this process, + * then publishes the authoritative Redis timing to peer processes. Without a + * coordinator, process-local behavior is unchanged. Request-local and + * untracked values are never evicted. Call only after the source mutation + * commits. + * + * Peer delivery uses at-most-once Redis Pub/Sub and is not part of this + * method's completion barrier. Detected subscription uncertainty clears and + * disables coordinated tracked local storage until a newly acknowledged + * healthy epoch. An undetected delivery gap can still leave a stale peer + * entry until its TTL; strict all-pod read-after-invalidation requires + * disabling process-local caching or a durable validation/barrier design. * * `futureBufferMs` is an application-owned safety window. When using the * bundled timestamp protocol, every Redis node eligible for primary promotion @@ -399,10 +466,13 @@ export class DialCache { * converts more tracked Redis reads into misses and rejects their tracked * writes, but does not delay or suppress returning fallback values. * - * The watermark fences only invocations that reach the tracked Redis write. - * A rejected write also suppresses the corresponding process-local population. - * Request-local memoization remains unconditional, and invocations whose - * remote layer is disabled or ramped out are not fenced by the watermark. + * The Redis watermark fences only invocations that reach the tracked Redis + * write. A rejected write also suppresses the corresponding process-local + * population. The optional local coordinator separately guards every tracked + * local publication path, including remote-disabled or ramped-out fallbacks, + * for its known future window. After reconnect, such a path cannot prove it + * observed an event missed during the gap. Request-local memoization remains + * an unconditional outer-request snapshot. * * @param futureBufferMs Nonnegative safe integer no greater than * 31,536,000,000 (365 days); defaults to zero for backward compatibility. @@ -410,13 +480,28 @@ export class DialCache { async invalidateRemote(keyType: string, id: Id, futureBufferMs = 0): Promise { assertSupportedFutureBufferMs(futureBufferMs); - if (this.redisCache === null) { + const redisCache = this.redisCache; + if (redisCache === null) { return; } + const stringId = String(id); this.metrics?.invalidation({ cacheNamespace: this.namespace, keyType, layer: CacheLayer.REMOTE }); try { - await this.redisCache.invalidate(keyType, String(id), futureBufferMs, this.namespace); + // Validate before provisional fan-out. Invalid caller input must not + // degrade a healthy process-wide coordinator. + invalidationPrefix(this.namespace, keyType, stringId); + this.invalidationCoordinator?.invalidate({ + namespace: this.namespace, + keyType, + id: stringId, + remainingMs: futureBufferMs, + source: "provisional", + }); + const event = await redisCache.invalidate(keyType, stringId, futureBufferMs, this.namespace); + if (event !== null) { + this.invalidationCoordinator?.invalidate(localInvalidationFromEvent(event)); + } } catch (error) { this.logger.warn("Error writing DialCache invalidation watermark", error); this.metrics?.error({ @@ -498,15 +583,24 @@ export class DialCache { if (local.status === "hit") { return local.value; } + const publicationPermit = this.captureLocalPublicationPermit(key); const redisCache = this.redisCache; if (redisCache === null) { - return await this.finishLocalOnly(key, local, fallback); + return await this.finishLocalOnly(key, local, fallback, publicationPermit); } const remoteLayer = await this.resolveRemoteLayerConfig(key, keyConfig); if (remoteLayer.status === "disabled") { - return await this.finishRedisChain(redisCache, key, local, remoteLayer, fallback); + return await this.finishRedisChain( + redisCache, + key, + local, + remoteLayer, + fallback, + undefined, + publicationPermit, + ); } const remote = await this.readRemoteWithResolvedConfig( @@ -515,13 +609,26 @@ export class DialCache { remoteLayer.config, keyConfig?.remoteReadTimeoutMs ?? redisCache.readTimeoutMs, ); - return await this.finishRedisChain(redisCache, key, local, remote, fallback, remoteLayer.config); + return await this.finishRedisChain( + redisCache, + key, + local, + remote, + fallback, + remoteLayer.config, + publicationPermit, + ); } - private async finishLocalOnly(key: DialCacheKey, local: CacheGetResult, fallback: () => Promise): Promise { + private async finishLocalOnly( + key: DialCacheKey, + local: CacheGetResult, + fallback: () => Promise, + publicationPermit: LocalPublicationPermit | null, + ): Promise { const value = await this.callFallback(labelsFor(key, CacheLayer.LOCAL), fallback); if (local.status === "miss") { - await this.putLocalFailOpen(key, value, local.config); + await this.putLocalFailOpen(key, value, local.config, publicationPermit); } return value; } @@ -533,10 +640,11 @@ export class DialCache { remote: RemoteCacheGetResult, fallback: () => Promise, resolvedRemoteConfig?: ResolvedLayerConfig, + publicationPermit: LocalPublicationPermit | null = null, ): Promise { if (remote.status === "hit") { if (local.status === "miss") { - await this.putLocalFailOpen(key, remote.value, local.config); + await this.putLocalFailOpen(key, remote.value, local.config, publicationPermit); } return remote.value; } @@ -565,7 +673,7 @@ export class DialCache { } } if (!suppressCacheWrite && local.status === "miss") { - await this.putLocalFailOpen(key, value, local.config); + await this.putLocalFailOpen(key, value, local.config, publicationPermit); } return value; } @@ -649,15 +757,75 @@ export class DialCache { } } - private async putLocalFailOpen(key: DialCacheKey, value: T, config?: { readonly ttlSec: number }): Promise { + private async putLocalFailOpen( + key: DialCacheKey, + value: T, + config?: { readonly ttlSec: number }, + publicationPermit: LocalPublicationPermit | null = null, + ): Promise { try { - await this.localCache.put(key, value, config); + const localInvalidation = this.localInvalidation; + const canPublish = + localInvalidation === null || publicationPermit === null + ? undefined + : () => localInvalidation.canPublish(key, publicationPermit); + await this.localCache.put(key, value, config, canPublish); } catch (error) { this.logger.warn("Error putting value in local cache", error); this.recordError(key, CacheLayer.LOCAL, "cache_write"); } } + private captureLocalPublicationPermit(key: DialCacheKey): LocalPublicationPermit | null { + return key.trackForInvalidation + ? this.localInvalidation?.capturePublicationPermit() ?? null + : null; + } + + private applyLocalInvalidation(invalidation: DialCacheLocalInvalidation): void { + const localInvalidation = this.localInvalidation; + if (localInvalidation === null || localInvalidation.disposed) { + return; + } + if (!isValidLocalInvalidation(invalidation, this.namespace)) { + this.applyInvalidationState( + "unavailable", + new DialCacheRedisProtocolError("Invalid DialCache local invalidation"), + ); + return; + } + localInvalidation.apply(invalidation); + if (invalidation.source === "event") { + this.metrics?.invalidation({ + cacheNamespace: this.namespace, + keyType: invalidation.keyType, + layer: CacheLayer.LOCAL, + }); + } + } + + private applyInvalidationState( + state: DialCacheInvalidationCoordinatorState, + error?: unknown, + ): void { + const transition = this.localInvalidation?.transition(state); + if (transition?.changed !== true) { + return; + } + + if (state === "ready") { + this.logger.debug("DialCache local invalidation coordinator ready"); + } else if (state === "unavailable") { + if (error === undefined) { + this.logger.warn("DialCache local invalidation coordinator unavailable"); + } else { + this.logger.warn("DialCache local invalidation coordinator unavailable", error); + } + } else { + this.logger.debug("DialCache local invalidation coordinator disposed"); + } + } + private async callFallback(labels: CacheMetricLabels, fallback: () => Promise): Promise { const start = performance.now(); try { diff --git a/src/index.ts b/src/index.ts index d26f22b..44b6af6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,15 @@ export { CacheLayer, DialCacheKeyConfig } from "./config.js"; export type { CacheConfigProvider, DialCacheConfig, LayerConfig, Logger } from "./config.js"; export { DialCacheContext } from "./context.js"; +export type { + DialCacheInvalidationCoordinator, + DialCacheInvalidationCoordinatorListener, + DialCacheInvalidationCoordinatorState, + DialCacheInvalidationEventV1, + DialCacheInvalidationIdentity, + DialCacheLocalInvalidation, + DialCacheLocalInvalidationSource, +} from "./invalidation.js"; export type { CacheMetricLabels, CoalescedMetricLabels, @@ -38,10 +47,16 @@ export { DialCacheRedisPayloadError, DialCacheRedisProtocolError, } from "./redis-client.js"; -export type { RedisConfig } from "./internal/redis-cache.js"; export type { + CoordinatedRedisConfig, + DialCacheRedisConfig, + RedisConfig, +} from "./internal/redis-cache.js"; +export type { + DialCacheCoordinatedRedisClient, DialCacheRedisClient, RedisCachePayload, + RedisCoordinatedInvalidationRequest, RedisInvalidationRequest, RedisReadContext, RedisReadRequest, diff --git a/src/internal/invalidation-coordinator.ts b/src/internal/invalidation-coordinator.ts new file mode 100644 index 0000000..72d879c --- /dev/null +++ b/src/internal/invalidation-coordinator.ts @@ -0,0 +1,120 @@ +import type { + DialCacheInvalidationCoordinator, + DialCacheInvalidationCoordinatorListener, + DialCacheInvalidationCoordinatorState, + DialCacheLocalInvalidation, +} from "../invalidation.js"; +import { DialCacheRedisProtocolError } from "../redis-client.js"; +import { + decodeRedisInvalidationEvent, + isValidLocalInvalidation, + localInvalidationFromEvent, + redisInvalidationChannel, +} from "./invalidation-event.js"; + +/** + * Shared implementation used by first-party transports. It deliberately owns + * no Redis resources; adapters drive health and payload delivery. + */ +export class InvalidationCoordinator implements DialCacheInvalidationCoordinator { + readonly channel: string; + private readonly listeners = new Set(); + private currentState: DialCacheInvalidationCoordinatorState = "unavailable"; + + constructor(readonly namespace: string) { + this.channel = redisInvalidationChannel(namespace); + } + + get state(): DialCacheInvalidationCoordinatorState { + return this.currentState; + } + + addListener(listener: DialCacheInvalidationCoordinatorListener): () => void { + if (this.currentState === "disposed") { + callListener(() => listener.onStateChange("disposed")); + return () => undefined; + } + + this.listeners.add(listener); + callListener(() => listener.onStateChange(this.currentState)); + let listening = true; + return () => { + if (!listening) { + return; + } + listening = false; + this.listeners.delete(listener); + }; + } + + invalidate(invalidation: DialCacheLocalInvalidation): boolean { + if (this.currentState === "disposed") { + return false; + } + if (!isValidLocalInvalidation(invalidation, this.namespace)) { + this.unavailable(new DialCacheRedisProtocolError("Invalid DialCache local invalidation")); + return false; + } + + for (const listener of [...this.listeners]) { + if (this.state === "disposed") { + break; + } + callListener(() => listener.onInvalidation(invalidation)); + } + return true; + } + + receive(payload: string | Buffer, channel = this.channel): boolean { + if (this.currentState === "disposed") { + return false; + } + try { + if (channel !== this.channel) { + throw new DialCacheRedisProtocolError("Invalid DialCache invalidation channel"); + } + const event = decodeRedisInvalidationEvent(payload, { namespace: this.namespace }); + return this.invalidate(localInvalidationFromEvent(event)); + } catch (error) { + this.unavailable(error); + return false; + } + } + + ready(): void { + this.transition("ready"); + } + + unavailable(error?: unknown): void { + this.transition("unavailable", error); + } + + dispose(): void { + if (this.currentState === "disposed") { + return; + } + this.transition("disposed"); + this.listeners.clear(); + } + + private transition(state: DialCacheInvalidationCoordinatorState, error?: unknown): void { + if (this.currentState === "disposed" || state === this.currentState) { + return; + } + this.currentState = state; + for (const listener of [...this.listeners]) { + if (this.currentState !== state) { + break; + } + callListener(() => listener.onStateChange(state, error)); + } + } +} + +function callListener(call: () => void): void { + try { + call(); + } catch { + // One DialCache instance must not prevent sibling instances from converging. + } +} diff --git a/src/internal/invalidation-event.ts b/src/internal/invalidation-event.ts new file mode 100644 index 0000000..9a8d61c --- /dev/null +++ b/src/internal/invalidation-event.ts @@ -0,0 +1,187 @@ +import { TextDecoder } from "node:util"; + +import { + type DialCacheInvalidationEventV1, + type DialCacheInvalidationIdentity, + type DialCacheLocalInvalidation, + type DialCacheLocalInvalidationSource, +} from "../invalidation.js"; +import { invalidationPrefix } from "../key.js"; +import { DialCacheRedisProtocolError } from "../redis-client.js"; +import { MAX_SUPPORTED_DURATION_MS } from "./duration.js"; + +export const REDIS_INVALIDATION_EVENT_VERSION = 1; +export const MAX_REDIS_INVALIDATION_EVENT_BYTES = 16 * 1024; + +const EVENT_FIELDS = new Set([ + "version", + "namespace", + "keyType", + "id", + "effectiveWatermarkMs", + "redisNowMs", +]); +const CANONICAL_DECIMAL = /^(0|[1-9]\d*)$/; +const utf8Decoder = new TextDecoder("utf-8", { fatal: true }); + +export function redisInvalidationChannel(namespace: string): string { + // Reuse key validation so channel and cache identity cannot disagree. + invalidationPrefix(namespace, "", ""); + return `dialcache:invalidation:v${REDIS_INVALIDATION_EVENT_VERSION}:${ + encodeChannelComponent(namespace) + }`; +} + +export function decodeRedisInvalidationEvent( + payload: string | Buffer, + expected?: Partial, +): DialCacheInvalidationEventV1 { + const byteLength = Buffer.isBuffer(payload) ? payload.byteLength : Buffer.byteLength(payload); + if (byteLength > MAX_REDIS_INVALIDATION_EVENT_BYTES) { + throw protocolError("Invalid DialCache invalidation event; payload is too large"); + } + + let text: string; + if (Buffer.isBuffer(payload)) { + try { + text = utf8Decoder.decode(payload); + } catch { + throw protocolError("Invalid DialCache invalidation event; payload is not valid UTF-8"); + } + } else { + text = payload; + } + + let value: unknown; + try { + value = JSON.parse(text); + } catch { + throw protocolError("Invalid DialCache invalidation event; payload is not valid JSON"); + } + return validateRedisInvalidationEvent(value, expected); +} + +export function validateRedisInvalidationEvent( + value: unknown, + expected?: Partial, +): DialCacheInvalidationEventV1 { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw protocolError("Invalid DialCache invalidation event; expected an object"); + } + + const record = value as Record; + const fields = Object.keys(record); + if (fields.length !== EVENT_FIELDS.size || fields.some((field) => !EVENT_FIELDS.has(field))) { + throw protocolError("Invalid DialCache invalidation event; unexpected fields"); + } + if (record.version !== REDIS_INVALIDATION_EVENT_VERSION) { + throw protocolError("Invalid DialCache invalidation event version"); + } + if ( + typeof record.namespace !== "string" + || typeof record.keyType !== "string" + || typeof record.id !== "string" + ) { + throw protocolError("Invalid DialCache invalidation event identity"); + } + + try { + invalidationPrefix(record.namespace, record.keyType, record.id); + } catch { + throw protocolError("Invalid DialCache invalidation event identity"); + } + + if ( + expected !== undefined + && ( + (expected.namespace !== undefined && record.namespace !== expected.namespace) + || (expected.keyType !== undefined && record.keyType !== expected.keyType) + || (expected.id !== undefined && record.id !== expected.id) + ) + ) { + throw protocolError("Invalid DialCache invalidation event identity"); + } + + const effectiveWatermarkMs = parseCanonicalTimestamp(record.effectiveWatermarkMs); + const redisNowMs = parseCanonicalTimestamp(record.redisNowMs); + const remainingMs = effectiveWatermarkMs - redisNowMs; + if (remainingMs < 0 || remainingMs > MAX_SUPPORTED_DURATION_MS) { + throw protocolError("Invalid DialCache invalidation event timing"); + } + + return { + version: REDIS_INVALIDATION_EVENT_VERSION, + namespace: record.namespace, + keyType: record.keyType, + id: record.id, + effectiveWatermarkMs: record.effectiveWatermarkMs as string, + redisNowMs: record.redisNowMs as string, + }; +} + +export function localInvalidationFromEvent( + event: DialCacheInvalidationEventV1, + source: DialCacheLocalInvalidationSource = "event", +): DialCacheLocalInvalidation { + const validated = validateRedisInvalidationEvent(event); + return { + namespace: validated.namespace, + keyType: validated.keyType, + id: validated.id, + remainingMs: Number(validated.effectiveWatermarkMs) - Number(validated.redisNowMs), + source, + }; +} + +export function isValidLocalInvalidation( + invalidation: unknown, + namespace: string, +): invalidation is DialCacheLocalInvalidation { + if (invalidation === null || typeof invalidation !== "object" || Array.isArray(invalidation)) { + return false; + } + + const record = invalidation as Record; + if ( + record.namespace !== namespace + || typeof record.keyType !== "string" + || typeof record.id !== "string" + || (record.source !== "provisional" && record.source !== "event") + || !Number.isSafeInteger(record.remainingMs) + || (record.remainingMs as number) < 0 + || (record.remainingMs as number) > MAX_SUPPORTED_DURATION_MS + ) { + return false; + } + + try { + invalidationPrefix(namespace, record.keyType, record.id); + return true; + } catch { + return false; + } +} + +function parseCanonicalTimestamp(value: unknown): number { + if (typeof value !== "string" || !CANONICAL_DECIMAL.test(value)) { + throw protocolError("Invalid DialCache invalidation event timing"); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw protocolError("Invalid DialCache invalidation event timing"); + } + return parsed; +} + +function protocolError(message: string): DialCacheRedisProtocolError { + return new DialCacheRedisProtocolError(message); +} + +function encodeChannelComponent(value: string): string { + // encodeURIComponent leaves Redis ACL glob metacharacter "*" unescaped. + // Strict RFC 3986 encoding makes a returned channel safe to grant exactly. + return encodeURIComponent(value).replace( + /[!'()*]/g, + (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`, + ); +} diff --git a/src/internal/local-cache.ts b/src/internal/local-cache.ts index dd12443..da4616b 100644 --- a/src/internal/local-cache.ts +++ b/src/internal/local-cache.ts @@ -90,16 +90,57 @@ export class LocalCache { }); } - async put(key: DialCacheKey, value: T, config?: { readonly ttlSec: number }): Promise { + async put( + key: DialCacheKey, + value: T, + config?: { readonly ttlSec: number }, + canPublish?: () => boolean, + ): Promise { + if (this.cache === null) { + return; + } const ttlSec = config?.ttlSec ?? await this.resolveLocalTtlSec(key); if (ttlSec === null) { return; } + if (canPublish !== undefined && !canPublish()) { + return; + } // lru-cache expires when age > ttl, while DialCache historically expired // when its integer-millisecond clock reached the configured boundary. const ttlMs = cacheTtlSecToMs(ttlSec) - 1; - this.cache?.set(key.urn, { value }, { size: 1, ttl: ttlMs }); + // Keep the publication guard and set synchronous so an invalidation either + // deletes an already-published value or is observed before this write. + this.cache.set(key.urn, { value }, { size: 1, ttl: ttlMs }); + } + + deleteTrackedPrefix(prefix: string): number { + if (this.cache === null) { + return 0; + } + let deleted = 0; + for (const urn of this.cache.keys()) { + if (urn.startsWith(prefix) && this.cache.delete(urn)) { + deleted += 1; + } + } + return deleted; + } + + clearTracked(): number { + if (this.cache === null) { + return 0; + } + let deleted = 0; + for (const urn of this.cache.keys()) { + // Tracked URNs begin with DialCache's reserved Redis hash tag. Namespace + // and untracked key components cannot contain a literal opening brace. + if (urn.startsWith("{") && this.cache.delete(urn)) { + deleted += 1; + } + } + return deleted; } private async resolveLocalTtlSec(key: DialCacheKey): Promise { diff --git a/src/internal/local-invalidation.ts b/src/internal/local-invalidation.ts new file mode 100644 index 0000000..15fdbc7 --- /dev/null +++ b/src/internal/local-invalidation.ts @@ -0,0 +1,123 @@ +import { performance } from "node:perf_hooks"; + +import type { + DialCacheInvalidationCoordinatorState, + DialCacheLocalInvalidation, +} from "../invalidation.js"; +import { invalidationPrefix, redisClusterHashTag, type DialCacheKey } from "../key.js"; +import type { LocalCache } from "./local-cache.js"; + +export type LocalPublicationPermit = number; + +export interface LocalInvalidationTransition { + readonly changed: boolean; + readonly evicted: number; +} + +export class LocalInvalidationState { + private readonly fences = new Map(); + private health: DialCacheInvalidationCoordinatorState = "unavailable"; + private healthEpoch = 0; + private globalDeadlineMs = -1; + + constructor( + private readonly localCache: LocalCache, + private readonly maxFences: number, + ) {} + + capturePublicationPermit(): LocalPublicationPermit { + return this.healthEpoch; + } + + get disposed(): boolean { + return this.health === "disposed"; + } + + canPublish(key: DialCacheKey, permit: LocalPublicationPermit): boolean { + if (this.health !== "ready" || permit !== this.healthEpoch) { + return false; + } + + const now = performance.now(); + if (this.globalDeadlineMs >= now) { + return false; + } + this.globalDeadlineMs = -1; + + const deadline = this.fences.get(key.prefix); + if (deadline === undefined) { + return true; + } + if (deadline >= now) { + return false; + } + this.fences.delete(key.prefix); + return true; + } + + apply(invalidation: DialCacheLocalInvalidation): number { + if (this.maxFences === 0 || this.health === "disposed") { + return 0; + } + + const prefix = redisClusterHashTag( + invalidationPrefix(invalidation.namespace, invalidation.keyType, invalidation.id), + ); + const now = performance.now(); + const deadline = Math.min(now + invalidation.remainingMs, Number.MAX_SAFE_INTEGER); + this.extendFence(prefix, deadline, now); + return this.localCache.deleteTrackedPrefix(prefix); + } + + transition(state: DialCacheInvalidationCoordinatorState): LocalInvalidationTransition { + if (this.health === "disposed" || state === this.health) { + return { changed: false, evicted: 0 }; + } + + this.health = state; + this.healthEpoch += 1; + const evicted = this.localCache.clearTracked(); + if (state === "disposed") { + this.fences.clear(); + this.globalDeadlineMs = -1; + } + return { changed: true, evicted }; + } + + private extendFence(prefix: string, deadline: number, now: number): void { + if (this.globalDeadlineMs >= now) { + this.globalDeadlineMs = Math.max(this.globalDeadlineMs, deadline); + return; + } + this.globalDeadlineMs = -1; + + const current = this.fences.get(prefix); + if (current !== undefined) { + this.fences.set(prefix, Math.max(current, deadline)); + return; + } + + if (this.fences.size >= this.maxFences) { + this.pruneExpired(now); + } + if (this.fences.size < this.maxFences) { + this.fences.set(prefix, deadline); + return; + } + + let furthestDeadline = deadline; + for (const activeDeadline of this.fences.values()) { + furthestDeadline = Math.max(furthestDeadline, activeDeadline); + } + this.fences.clear(); + this.globalDeadlineMs = furthestDeadline; + } + + private pruneExpired(now: number): void { + for (const [prefix, deadline] of this.fences) { + if (deadline < now) { + this.fences.delete(prefix); + } + } + } +} diff --git a/src/internal/redis-cache.ts b/src/internal/redis-cache.ts index e2d4689..ba77a17 100644 --- a/src/internal/redis-cache.ts +++ b/src/internal/redis-cache.ts @@ -2,32 +2,70 @@ import { performance } from "node:perf_hooks"; import { CacheLayer, type CacheConfigProvider, type DialCacheKeyConfig } from "../config.js"; import { RedisReadTimeoutError } from "../errors.js"; +import type { + DialCacheInvalidationCoordinator, + DialCacheInvalidationEventV1, +} from "../invalidation.js"; import { invalidationPrefix, redisClusterHashTag, type DialCacheKey } from "../key.js"; import { labelsFor, type DialCacheMetricsAdapter, type MetricErrorKind } from "../metrics.js"; -import type { DialCacheRedisClient, RedisCachePayload } from "../redis-client.js"; +import type { + DialCacheCoordinatedRedisClient, + DialCacheRedisClient, + RedisCachePayload, +} from "../redis-client.js"; import { JsonSerializer, type Serializer } from "../serializer.js"; import type { CacheGetResult } from "./cache-result.js"; import { assertValidDeadlineMs, withMonotonicDeadline } from "./deadline.js"; import { cacheTtlSecToMs } from "./duration.js"; +import { + redisInvalidationChannel, + validateRedisInvalidationEvent, +} from "./invalidation-event.js"; import { fetchKeyConfig, resolveLayerConfigResult, type ResolvedLayerConfig } from "./runtime-config.js"; -export interface RedisConfig { +interface RedisConfigBase { + /** + * Instance-level remote-read deadline in milliseconds. Per-use-case runtime + * config can override it. Defaults to 50 ms. + */ + readonly readTimeoutMs?: number; + readonly serializer?: Serializer; +} + +/** + * Backward-compatible remote-only Redis configuration. + * + * This remains an interface so existing application interfaces and classes can + * continue to extend or implement it. + */ +export interface RedisConfig extends RedisConfigBase { /** * Caller-created, connected, and lifecycle-owned semantic Redis client. * DialCache borrows it and never drains, disposes, or closes it. */ readonly client: DialCacheRedisClient; + readonly coordinator?: undefined; +} + +export interface CoordinatedRedisConfig extends RedisConfigBase { /** - * Instance-level remote-read deadline in milliseconds. Per-use-case runtime - * config can override it. Defaults to 50 ms. + * Caller-owned command client with the atomic invalidate-and-publish + * capability. DialCache never creates, connects, drains, or closes it. */ - readonly readTimeoutMs?: number; - readonly serializer?: Serializer; + readonly client: DialCacheCoordinatedRedisClient; + /** + * Caller-owned, already-established notification coordinator. It may be + * shared by several DialCache instances in the same process. + */ + readonly coordinator: DialCacheInvalidationCoordinator; } +/** Redis configuration accepted by DialCache. */ +export type DialCacheRedisConfig = RedisConfig | CoordinatedRedisConfig; + interface RedisCacheOptions { readonly configProvider: CacheConfigProvider; - readonly redis: RedisConfig; + readonly redis: DialCacheRedisConfig; readonly metrics: DialCacheMetricsAdapter | null; } @@ -39,6 +77,7 @@ export class RedisCache { private readonly configProvider: CacheConfigProvider; private readonly defaultSerializer: Serializer; private readonly client: DialCacheRedisClient; + private readonly coordinatedClient: DialCacheCoordinatedRedisClient | null; private readonly metrics: DialCacheMetricsAdapter | null; readonly readTimeoutMs: number; @@ -68,6 +107,23 @@ export class RedisCache { } this.client = options.redis.client; + if (options.redis.coordinator === undefined) { + this.coordinatedClient = null; + } else { + if ( + options.redis.coordinator === null + || typeof options.redis.coordinator !== "object" + ) { + throw new TypeError("Redis invalidation coordinator must be an object"); + } + const coordinatedClient = options.redis.client as Partial; + if (typeof coordinatedClient.invalidateAndPublish !== "function") { + throw new TypeError( + "Redis config with coordinator requires a client that supports invalidateAndPublish", + ); + } + this.coordinatedClient = options.redis.client; + } } async get(key: DialCacheKey): Promise { @@ -166,11 +222,27 @@ export class RedisCache { } } - async invalidate(keyType: string, id: string, futureBufferMs = 0, namespace = "urn"): Promise { - await this.client.invalidate({ - watermarkKey: this.redisWatermarkKey(namespace, keyType, id), + async invalidate( + keyType: string, + id: string, + futureBufferMs = 0, + namespace = "urn", + ): Promise { + const watermarkKey = this.redisWatermarkKey(namespace, keyType, id); + if (this.coordinatedClient === null) { + await this.client.invalidate({ watermarkKey, futureBufferMs }); + return null; + } + + const event = await this.coordinatedClient.invalidateAndPublish({ + watermarkKey, futureBufferMs, + channel: redisInvalidationChannel(namespace), + namespace, + keyType, + id, }); + return validateRedisInvalidationEvent(event, { namespace, keyType, id }); } redisKey(key: DialCacheKey): string { diff --git a/src/internal/redis-scripts.ts b/src/internal/redis-scripts.ts index 891c0ac..384d955 100644 --- a/src/internal/redis-scripts.ts +++ b/src/internal/redis-scripts.ts @@ -1,4 +1,8 @@ import { MAX_SUPPORTED_DURATION_MS } from "./duration.js"; +import { + MAX_REDIS_INVALIDATION_EVENT_BYTES, + REDIS_INVALIDATION_EVENT_VERSION, +} from "./invalidation-event.js"; export const REDIS_FRAME_VERSION = 1; export const REDIS_ENCODING_UTF8 = 0; @@ -156,3 +160,65 @@ else end`, "return 1", ].join("\n\n"); + +/** + * Coordinated invalidation protocol. This is parallel to the legacy script so + * custom adapters and raw protocol consumers retain the exact integer-1 reply. + */ +export const INVALIDATE_AND_PUBLISH_CACHE_SCRIPT = [ + PARSE_WATERMARK_LUA, + CEIL_FINITE_NUMBER_LUA, + String.raw`local future_buffer_ms = ceil_finite_number(ARGV[1]) +if not future_buffer_ms or future_buffer_ms < 0 or future_buffer_ms > ${MAX_SUPPORTED_DURATION_MS} then + return redis.error_reply("ERR invalid DialCache future buffer") +end`, + REDIS_TIME_LUA, + String.raw`local proposed_watermark = now_ms + future_buffer_ms +local raw_watermark = redis.call("GET", KEYS[1]) +local current_watermark = 0 + +if raw_watermark then + local parsed_watermark = parse_watermark(raw_watermark) + if parsed_watermark then + if parsed_watermark - now_ms > ${MAX_SUPPORTED_DURATION_MS} then + return redis.error_reply("ERR invalid DialCache watermark") + end + current_watermark = parsed_watermark + end +end + +local watermark = math.ceil(math.max(current_watermark, proposed_watermark)) +local current_ttl_ms = -2 +if raw_watermark then + current_ttl_ms = redis.call("PTTL", KEYS[1]) +end +local desired_ttl_ms = math.max( + future_buffer_ms + ${WATERMARK_TTL_MARGIN_MS}, + watermark - now_ms + ${WATERMARK_TTL_MARGIN_MS} +) +if current_ttl_ms > desired_ttl_ms then + desired_ttl_ms = current_ttl_ms +end + +local encoded_watermark = string.format("%.0f", watermark) +local encoded_now = string.format("%.0f", now_ms) +local event = cjson.encode({ + version = ${REDIS_INVALIDATION_EVENT_VERSION}, + namespace = ARGV[3], + keyType = ARGV[4], + id = ARGV[5], + effectiveWatermarkMs = encoded_watermark, + redisNowMs = encoded_now +}) +if string.len(event) > ${MAX_REDIS_INVALIDATION_EVENT_BYTES} then + return redis.error_reply("ERR DialCache invalidation event is too large") +end + +if current_ttl_ms == -1 then + redis.call("SET", KEYS[1], encoded_watermark) +else + redis.call("SET", KEYS[1], encoded_watermark, "PX", desired_ttl_ms) +end +redis.call("PUBLISH", ARGV[2], event) +return event`, +].join("\n\n"); diff --git a/src/invalidation.ts b/src/invalidation.ts new file mode 100644 index 0000000..71bdcbc --- /dev/null +++ b/src/invalidation.ts @@ -0,0 +1,58 @@ +/** + * Logical identity shared by every invalidation-tracked cache entry for one + * namespace, key type, and id. + */ +export interface DialCacheInvalidationIdentity { + readonly namespace: string; + readonly keyType: string; + readonly id: string; +} + +/** + * Versioned event published by the coordinated Redis invalidation protocol. + * Redis timestamps remain canonical decimal strings on the wire and are + * validated as safe integers before any local timing is derived. + */ +export interface DialCacheInvalidationEventV1 extends DialCacheInvalidationIdentity { + readonly version: 1; + readonly effectiveWatermarkMs: string; + readonly redisNowMs: string; +} + +export type DialCacheInvalidationCoordinatorState = "ready" | "unavailable" | "disposed"; +export type DialCacheLocalInvalidationSource = "provisional" | "event"; + +/** + * Process-local invalidation delivered synchronously to every registered + * DialCache instance. `remainingMs` is translated onto each process's + * monotonic clock by the listener. + */ +export interface DialCacheLocalInvalidation extends DialCacheInvalidationIdentity { + readonly remainingMs: number; + readonly source: DialCacheLocalInvalidationSource; +} + +export interface DialCacheInvalidationCoordinatorListener { + onInvalidation(invalidation: DialCacheLocalInvalidation): void; + onStateChange(state: DialCacheInvalidationCoordinatorState, error?: unknown): void; +} + +/** + * Caller-owned, process-wide fan-out boundary for coordinated invalidation. + * + * Implementations must invoke listeners synchronously, isolate listener + * failures, and deliver the current state synchronously from `addListener` + * before it returns. `invalidate` must likewise complete local fan-out before + * returning. The returned function removes only that listener. + * + * Coordinators are notification transports, not durable invalidation stores. + * Implementations must enter `unavailable` whenever subscription continuity is + * uncertain; DialCache then clears and bypasses coordinated tracked local + * state until a newly acknowledged `ready` transition. + */ +export interface DialCacheInvalidationCoordinator { + readonly namespace: string; + readonly state: DialCacheInvalidationCoordinatorState; + addListener(listener: DialCacheInvalidationCoordinatorListener): () => void; + invalidate(invalidation: DialCacheLocalInvalidation): void; +} diff --git a/src/node-redis.ts b/src/node-redis.ts index 2093b8f..ee28510 100644 --- a/src/node-redis.ts +++ b/src/node-redis.ts @@ -1,6 +1,13 @@ import { commandOptions, defineScript } from "redis"; +import type { + DialCacheInvalidationCoordinator, + DialCacheInvalidationCoordinatorListener, +} from "./invalidation.js"; +import { InvalidationCoordinator } from "./internal/invalidation-coordinator.js"; +import { decodeRedisInvalidationEvent } from "./internal/invalidation-event.js"; import { + INVALIDATE_AND_PUBLISH_CACHE_SCRIPT, INVALIDATE_CACHE_SCRIPT, READ_CACHE_SCRIPT, READ_TRACKED_CACHE_SCRIPT, @@ -12,7 +19,12 @@ import { validateRedisScriptInvalidationReply, validateRedisScriptWriteReply, } from "./internal/redis-script-reply.js"; -import type { DialCacheRedisClient } from "./redis-client.js"; +import { DialCacheRedisProtocolError } from "./redis-client.js"; +import type { + DialCacheCoordinatedRedisClient, + DialCacheRedisClient, + RedisCoordinatedInvalidationRequest, +} from "./redis-client.js"; type BufferReplyOptions = ReturnType< typeof commandOptions<{ @@ -25,6 +37,15 @@ const bufferReplyOptions: BufferReplyOptions = commandOptions({ returnBuffers: t const readReply = (reply: string | null): string | null => reply; const writeReply = (reply: number): number => validateRedisScriptWriteReply(reply); const invalidationReply = (reply: number): number => validateRedisScriptInvalidationReply(reply); +const coordinatedInvalidationReply = (reply: string): string => { + if (typeof reply !== "string") { + throw new DialCacheRedisProtocolError( + "Invalid DialCache Redis coordinated invalidation reply; expected an event payload", + ); + } + decodeRedisInvalidationEvent(reply); + return reply; +}; type NodeRedisArgument = string | Buffer; interface NodeRedisScript, Reply> { @@ -66,6 +87,17 @@ export type DialCacheNodeRedisScripts = { [watermarkKey: string, futureBufferMs: number], number >; + readonly dialcacheInvalidateAndPublish: NodeRedisScript< + [ + watermarkKey: string, + futureBufferMs: number, + channel: string, + namespace: string, + keyType: string, + id: string, + ], + string + >; }; export const dialcacheRedisScripts: DialCacheNodeRedisScripts = { @@ -131,6 +163,23 @@ export const dialcacheRedisScripts: DialCacheNodeRedisScripts = { }, transformReply: invalidationReply, }), + dialcacheInvalidateAndPublish: defineDialCacheScript({ + SCRIPT: INVALIDATE_AND_PUBLISH_CACHE_SCRIPT, + NUMBER_OF_KEYS: 1, + FIRST_KEY_INDEX: 0, + IS_READ_ONLY: false, + transformArguments( + watermarkKey: string, + futureBufferMs: number, + channel: string, + namespace: string, + keyType: string, + id: string, + ): Array { + return [watermarkKey, String(futureBufferMs), channel, namespace, keyType, id]; + }, + transformReply: coordinatedInvalidationReply, + }), }; interface NodeRedisScriptClient { @@ -151,6 +200,17 @@ interface NodeRedisScriptClient { dialcacheInvalidate(watermarkKey: string, futureBufferMs: number): Promise; } +interface CoordinatedNodeRedisScriptClient extends NodeRedisScriptClient { + dialcacheInvalidateAndPublish( + watermarkKey: string, + futureBufferMs: number, + channel: string, + namespace: string, + keyType: string, + id: string, + ): Promise; +} + /** * Create a resource-free semantic view over a caller-owned node-redis client. * Read signals are passed to node-redis so queued commands can be removed when @@ -158,8 +218,14 @@ interface NodeRedisScriptClient { * server stopped executing it. The caller remains responsible for finite * native command budgets, draining work, and closing the client. */ -export function createNodeRedisDialCacheClient(client: NodeRedisScriptClient): DialCacheRedisClient { - return { +export function createNodeRedisDialCacheClient( + client: CoordinatedNodeRedisScriptClient, +): DialCacheCoordinatedRedisClient; +export function createNodeRedisDialCacheClient(client: NodeRedisScriptClient): DialCacheRedisClient; +export function createNodeRedisDialCacheClient( + client: NodeRedisScriptClient, +): DialCacheRedisClient | DialCacheCoordinatedRedisClient { + const adapter: DialCacheRedisClient = { async read({ valueKey, watermarkKey }, context) { const options: BufferReplyOptions = context === undefined ? bufferReplyOptions @@ -188,4 +254,210 @@ export function createNodeRedisDialCacheClient(client: NodeRedisScriptClient): D validateRedisScriptInvalidationReply(result); }, }; + if (!isCoordinatedScriptClient(client)) { + return adapter; + } + + return { + ...adapter, + async invalidateAndPublish(request: RedisCoordinatedInvalidationRequest) { + const payload = await client.dialcacheInvalidateAndPublish( + request.watermarkKey, + request.futureBufferMs, + request.channel, + request.namespace, + request.keyType, + request.id, + ); + return decodeRedisInvalidationEvent(payload, request); + }, + }; +} + +function isCoordinatedScriptClient( + client: NodeRedisScriptClient, +): client is CoordinatedNodeRedisScriptClient { + return typeof (client as Partial).dialcacheInvalidateAndPublish + === "function"; +} + +type NodeRedisBufferListener = (message: Buffer, channel: Buffer) => unknown; + +/** + * Minimal node-redis standalone subscriber surface. Redis Cluster command + * clients may still publish; use a dedicated standalone subscriber connected + * to a cluster node because the node-redis Cluster facade does not expose the + * reconnect/ready lifecycle needed by this contract. + */ +export interface DialCacheNodeRedisSubscriberClient { + readonly isOpen: boolean; + readonly isReady: boolean; + readonly isPubSubActive: boolean; + subscribe(channel: string, listener: NodeRedisBufferListener, bufferMode: true): Promise; + unsubscribe(channel: string, listener: NodeRedisBufferListener, bufferMode: true): Promise; + on(event: "error", listener: (error: Error) => void): unknown; + on(event: "reconnecting" | "ready" | "end", listener: () => void): unknown; + off(event: "error", listener: (error: Error) => void): unknown; + off(event: "reconnecting" | "ready" | "end", listener: () => void): unknown; +} + +export interface DialCacheNodeRedisInvalidationCoordinator + extends DialCacheInvalidationCoordinator { + readonly channel: string; + dispose(): Promise; +} + +/** + * Attach coordinated invalidation to a caller-created, connected, dedicated + * node-redis standalone subscriber. The helper owns only its subscription and + * EventEmitter listeners; it never creates, connects, or closes the client. + * The returned promise settles only after the initial exact-channel + * subscription acknowledgement. Dispose DialCache listeners first, then this + * coordinator, then close the caller-owned subscriber. The caller must install + * and retain its own node-redis `error` listener before connecting; the helper + * removes its internal listener during disposal. + */ +export async function createNodeRedisDialCacheInvalidationCoordinator( + subscriber: DialCacheNodeRedisSubscriberClient, + options: { readonly namespace?: string } = {}, +): Promise { + if (!subscriber.isOpen || !subscriber.isReady) { + throw new TypeError("DialCache node-redis subscriber must already be connected and ready"); + } + if (subscriber.isPubSubActive) { + throw new TypeError("DialCache node-redis subscriber must be dedicated and unsubscribed"); + } + + const coordinator = new InvalidationCoordinator(options.namespace ?? "urn"); + const expectedChannel = Buffer.from(coordinator.channel); + // A ready acknowledgement may recover transport loss, but it must not erase + // a malformed event observed after that subscribe/resubscribe attempt began. + let protocolFailureEpoch = 0; + let recoveryProtocolFailureEpoch = protocolFailureEpoch; + const markProtocolFailure = (error: unknown): void => { + protocolFailureEpoch += 1; + coordinator.unavailable(error); + }; + const beginTransportRecovery = (error: unknown): void => { + recoveryProtocolFailureEpoch = protocolFailureEpoch; + coordinator.unavailable(error); + }; + const onMessage: NodeRedisBufferListener = (message, channel) => { + try { + if (!channel.equals(expectedChannel)) { + markProtocolFailure( + new DialCacheRedisProtocolError("Invalid DialCache invalidation channel"), + ); + return; + } + if (!coordinator.receive(message)) { + protocolFailureEpoch += 1; + } + } catch (error) { + // node-redis invokes Pub/Sub listeners synchronously without containing + // callback exceptions. + markProtocolFailure(error); + } + }; + const onError = (error: Error): void => beginTransportRecovery(error); + const onReconnecting = (): void => { + beginTransportRecovery(new Error("DialCache node-redis subscriber reconnecting")); + }; + // node-redis 4.7 emits ready only after its automatic Pub/Sub resubscribe + // acknowledgement has completed. + const onReady = (): void => { + if (protocolFailureEpoch === recoveryProtocolFailureEpoch) { + coordinator.ready(); + } + }; + const onEnd = (): void => { + beginTransportRecovery(new Error("DialCache node-redis subscriber ended")); + }; + + subscriber.on("error", onError); + subscriber.on("reconnecting", onReconnecting); + subscriber.on("ready", onReady); + subscriber.on("end", onEnd); + + const initialProtocolFailureEpoch = protocolFailureEpoch; + try { + await subscriber.subscribe(coordinator.channel, onMessage, true); + if (protocolFailureEpoch === initialProtocolFailureEpoch) { + coordinator.ready(); + } + } catch (error) { + coordinator.dispose(); + detachNodeRedisListeners(subscriber, { onError, onReconnecting, onReady, onEnd }); + if (subscriber.isOpen) { + try { + await subscriber.unsubscribe(coordinator.channel, onMessage, true); + } catch { + // Preserve the subscription failure that prevented factory completion. + } + } + throw error; + } + + let disposePromise: Promise | null = null; + return { + get namespace() { + return coordinator.namespace; + }, + get channel() { + return coordinator.channel; + }, + get state() { + return coordinator.state; + }, + addListener(listener: DialCacheInvalidationCoordinatorListener) { + return coordinator.addListener(listener); + }, + invalidate(invalidation) { + coordinator.invalidate(invalidation); + }, + dispose() { + if (disposePromise !== null) { + return disposePromise; + } + + let resolveDispose!: () => void; + let rejectDispose!: (error: unknown) => void; + disposePromise = new Promise((resolve, reject) => { + resolveDispose = resolve; + rejectDispose = reject; + }); + + try { + coordinator.dispose(); + detachNodeRedisListeners(subscriber, { onError, onReconnecting, onReady, onEnd }); + } catch (error) { + rejectDispose(error); + return disposePromise; + } + + void (async () => { + if (subscriber.isOpen) { + await subscriber.unsubscribe(coordinator.channel, onMessage, true); + } + })().then(resolveDispose, rejectDispose); + return disposePromise; + }, + }; +} + +interface NodeRedisHealthListeners { + readonly onError: (error: Error) => void; + readonly onReconnecting: () => void; + readonly onReady: () => void; + readonly onEnd: () => void; +} + +function detachNodeRedisListeners( + subscriber: DialCacheNodeRedisSubscriberClient, + listeners: NodeRedisHealthListeners, +): void { + subscriber.off("error", listeners.onError); + subscriber.off("reconnecting", listeners.onReconnecting); + subscriber.off("ready", listeners.onReady); + subscriber.off("end", listeners.onEnd); } diff --git a/src/redis-client.ts b/src/redis-client.ts index eeff309..0e7bee5 100644 --- a/src/redis-client.ts +++ b/src/redis-client.ts @@ -1,4 +1,8 @@ import type { Awaitable } from "./config.js"; +import type { + DialCacheInvalidationEventV1, + DialCacheInvalidationIdentity, +} from "./invalidation.js"; const redisProtocolErrorBrand = Symbol.for("dialcache.DialCacheRedisProtocolError"); @@ -77,6 +81,12 @@ export interface RedisInvalidationRequest { readonly futureBufferMs: number; } +export interface RedisCoordinatedInvalidationRequest + extends RedisInvalidationRequest, DialCacheInvalidationIdentity { + /** Exact namespace-scoped Pub/Sub channel on which the event is published. */ + readonly channel: string; +} + /** * Caller-owned semantic Redis boundary. DialCache borrows this client and does * not create, connect, drain, dispose, or close it. @@ -103,3 +113,22 @@ export interface DialCacheRedisClient { */ invalidate(request: RedisInvalidationRequest): Awaitable; } + +/** + * Optional extension selected only when process-local invalidation + * coordination is configured. Legacy clients remain three-method structural + * implementations of `DialCacheRedisClient`. + */ +export interface DialCacheCoordinatedRedisClient extends DialCacheRedisClient { + /** + * Atomically advance the watermark and publish one versioned event. + * + * The result is the authoritative event produced by the same Redis + * invocation. A rejected command is an ambiguous write: the watermark may + * already have advanced before a later publication or response failure. + * Callers must retain their provisional local fence and may retry safely. + */ + invalidateAndPublish( + request: RedisCoordinatedInvalidationRequest, + ): Awaitable; +} diff --git a/src/redis-protocol.ts b/src/redis-protocol.ts index 5a9c23b..699b2ce 100644 --- a/src/redis-protocol.ts +++ b/src/redis-protocol.ts @@ -1,4 +1,5 @@ export { + INVALIDATE_AND_PUBLISH_CACHE_SCRIPT, INVALIDATE_CACHE_SCRIPT, READ_CACHE_SCRIPT, READ_TRACKED_CACHE_SCRIPT, @@ -8,3 +9,9 @@ export { WRITE_CACHE_SCRIPT, WRITE_TRACKED_CACHE_SCRIPT, } from "./internal/redis-scripts.js"; +export { + MAX_REDIS_INVALIDATION_EVENT_BYTES, + REDIS_INVALIDATION_EVENT_VERSION, + decodeRedisInvalidationEvent, + redisInvalidationChannel, +} from "./internal/invalidation-event.js"; diff --git a/src/valkey-glide.ts b/src/valkey-glide.ts index c63485c..c89cdf2 100644 --- a/src/valkey-glide.ts +++ b/src/valkey-glide.ts @@ -1,5 +1,7 @@ +import { decodeRedisInvalidationEvent } from "./internal/invalidation-event.js"; import { decodeRedisPayload, redisPayloadEncoding } from "./internal/redis-payload.js"; import { + INVALIDATE_AND_PUBLISH_CACHE_SCRIPT, INVALIDATE_CACHE_SCRIPT, READ_CACHE_SCRIPT, READ_TRACKED_CACHE_SCRIPT, @@ -10,7 +12,10 @@ import { validateRedisScriptInvalidationReply, validateRedisScriptWriteReply, } from "./internal/redis-script-reply.js"; -import { DialCacheRedisPayloadError, type DialCacheRedisClient } from "./redis-client.js"; +import { + DialCacheRedisPayloadError, + type DialCacheCoordinatedRedisClient, +} from "./redis-client.js"; type ValkeyGlideString = string | Buffer; @@ -47,7 +52,7 @@ interface DialCacheGlideScripts { readonly invalidate: TScript; } -export interface ValkeyGlideDialCacheClient extends DialCacheRedisClient { +export interface ValkeyGlideDialCacheClient extends DialCacheCoordinatedRedisClient { /** Release the adapter-owned GLIDE Script handles. Does not close the wrapped GLIDE client. */ dispose(): void; } @@ -61,6 +66,12 @@ export interface ValkeyGlideDialCacheClient extends DialCacheRedisClient { * client waiting but is not server-side command cancellation. GLIDE's current * script API has no per-invocation signal, so DialCache's core read deadline * may return before this adapter's invocation settles. + * + * This adapter supports coordinated publication. It does not create a + * first-party GLIDE invalidation coordinator because the pinned GLIDE API does + * not expose the subscription-health lifecycle required to detect every known + * gap. Applications may supply their own backend-neutral coordinator only when + * they can provide that health contract. */ export function createValkeyGlideDialCacheClient( client: ValkeyGlideScriptingClient, @@ -73,6 +84,7 @@ export function createValkeyGlideDialCacheClient { + beforeEach(() => { + vi.useFakeTimers(); + const clockOriginMs = Date.parse("2026-07-29T00:00:00.000Z"); + vi.setSystemTime(new Date(clockOriginMs)); + vi.spyOn(performance, "now").mockImplementation(() => Date.now() - clockOriginMs); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("evicts one identity across instances, use cases, and args without touching neighbors", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const first = coordinatedCache(redis, coordinator); + const second = coordinatedCache(redis, coordinator); + let version = 1; + const profile = first.cached(async (id: string, locale: string) => ({ id, locale, version }), { + keyType: "user_id", + useCase: "CoordinatedProfile", + cacheKey: (id, locale) => ({ id, args: { locale } }), + trackForInvalidation: true, + defaultConfig: localOnly, + }); + const permissions = first.cached(async (id: string) => ({ id, version }), { + keyType: "user_id", + useCase: "CoordinatedPermissions", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + const peerProfile = second.cached(async (id: string) => ({ id, version }), { + keyType: "user_id", + useCase: "CoordinatedPeerProfile", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + const adjacent = first.cached(async (id: string) => ({ id, version }), { + keyType: "user_id", + useCase: "CoordinatedAdjacent", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + const untracked = first.cached(async (id: string) => ({ id, version }), { + keyType: "user_id", + useCase: "CoordinatedUntracked", + cacheKey: (id) => id, + defaultConfig: localOnly, + }); + + await first.enable(async () => { + await profile("123", "en"); + await profile("123", "fr"); + await permissions("123"); + await adjacent("1234"); + await untracked("123"); + }); + await second.enable(async () => await peerProfile("123")); + + version = 2; + await first.invalidateRemote("user_id", "123"); + + await expect(first.enable(async () => profile("123", "en"))).resolves.toMatchObject({ + version: 2, + }); + await expect(first.enable(async () => profile("123", "fr"))).resolves.toMatchObject({ + version: 2, + }); + await expect(first.enable(async () => permissions("123"))).resolves.toMatchObject({ + version: 2, + }); + await expect(second.enable(async () => peerProfile("123"))).resolves.toMatchObject({ + version: 2, + }); + await expect(first.enable(async () => adjacent("1234"))).resolves.toMatchObject({ + version: 1, + }); + await expect(first.enable(async () => untracked("123"))).resolves.toMatchObject({ + version: 1, + }); + }); + + it("applies provisional eviction synchronously before Redis completes", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + const gate = deferred(); + redis.coordinatedInvalidationGate = gate.promise; + let version = 1; + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, version, call: ++calls }), { + keyType: "user_id", + useCase: "ProvisionalOriginEviction", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + await cache.enable(async () => await load("123")); + + version = 2; + const invalidation = cache.invalidateRemote("user_id", "123", 1_000); + expect(redis.coordinatedInvalidations).toHaveLength(1); + + await expect(cache.enable(async () => await load("123"))).resolves.toMatchObject({ + version: 2, + call: 2, + }); + await expect(cache.enable(async () => await load("123"))).resolves.toMatchObject({ + version: 2, + call: 3, + }); + + gate.resolve(); + await invalidation; + }); + + it("uses the furthest effective Redis watermark rather than the shorter request", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + let version = 1; + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, version, call: ++calls }), { + keyType: "user_id", + useCase: "EffectiveLocalFence", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + await cache.enable(async () => await load("123")); + await redis.invalidate({ watermarkKey, futureBufferMs: 5_000 }); + + vi.advanceTimersByTime(100); + version = 2; + await cache.invalidateRemote("user_id", "123", 100); + vi.advanceTimersByTime(200); + + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + expect(calls).toBe(3); + + vi.advanceTimersByTime(4_701); + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + expect(calls).toBe(4); + }); + + it("preserves process-flight results while suppressing stale publication", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + const started = deferred(); + const release = deferred(); + let version = 1; + let calls = 0; + const load = cache.cached(async (id: string) => { + calls += 1; + const observedVersion = version; + started.resolve(); + await release.promise; + return { id, observedVersion, call: calls }; + }, { + keyType: "user_id", + useCase: "CoordinatedProcessFlight", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + const leader = cache.enable(async () => await load("123")); + await started.promise; + version = 2; + await cache.invalidateRemote("user_id", "123", 1_000); + const follower = cache.enable(async () => await load("123")); + release.resolve(); + + const [leaderValue, followerValue] = await Promise.all([leader, follower]); + expect(followerValue).toBe(leaderValue); + expect(leaderValue.observedVersion).toBe(1); + expect(calls).toBe(1); + + await expect(cache.enable(async () => await load("123"))).resolves.toMatchObject({ + observedVersion: 2, + call: 2, + }); + }); + + it("blocks a late Redis-hit promotion after an invalidation event", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const writer = coordinatedCache(redis, coordinator); + let version = 1; + const write = writer.cached(async (id: string) => ({ id, version, fallback: 0 }), { + keyType: "user_id", + useCase: "LateRedisPromotion", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: remoteOnly, + }); + await writer.enable(async () => await write("123")); + + const loadStarted = deferred(); + const releaseLoad = deferred(); + const serializer: Serializer<{ id: string; version: number; fallback: number }> = { + dump: JSON.stringify, + load: async (payload) => { + const parsed = JSON.parse(Buffer.isBuffer(payload) ? payload.toString() : payload) as { + id: string; + version: number; + fallback: number; + }; + loadStarted.resolve(); + await releaseLoad.promise; + return parsed; + }, + }; + const reader = coordinatedCache(redis, coordinator); + let fallbackCalls = 0; + const read = reader.cached(async (id: string) => ({ id, version, fallback: ++fallbackCalls }), { + keyType: "user_id", + useCase: "LateRedisPromotion", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localAndRemote, + serializer, + }); + + const pending = reader.enable(async () => await read("123")); + await loadStarted.promise; + version = 2; + await writer.invalidateRemote("user_id", "123", 1_000); + releaseLoad.resolve(); + await expect(pending).resolves.toEqual({ id: "123", version: 1, fallback: 0 }); + + await expect(reader.enable(async () => await read("123"))).resolves.toMatchObject({ + version: 2, + fallback: 1, + }); + await expect(reader.enable(async () => await read("123"))).resolves.toMatchObject({ + fallback: 2, + }); + }); + + it("rejects old-epoch publication across detected disconnect and recovery", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + const started = deferred(); + const release = deferred(); + let version = 1; + let calls = 0; + const load = cache.cached(async (id: string) => { + calls += 1; + const observedVersion = version; + started.resolve(); + await release.promise; + return { id, observedVersion, call: calls }; + }, { + keyType: "user_id", + useCase: "HealthEpochPublication", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + const pending = cache.enable(async () => await load("123")); + await started.promise; + version = 2; + coordinator.unavailable(new Error("connection lost")); + coordinator.ready(); + release.resolve(); + await pending; + + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + expect(calls).toBe(2); + }); + + it("keeps provisional eviction and fencing when coordinated publication fails", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + let version = 1; + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, version, call: ++calls }), { + keyType: "user_id", + useCase: "FailedCoordinatedInvalidation", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + await cache.enable(async () => await load("123")); + version = 2; + redis.failSet = true; + + await expect(cache.invalidateRemote("user_id", "123", 1_000)).rejects.toThrow( + "redis set failed", + ); + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + expect(calls).toBe(3); + }); + + it("keeps request-local snapshot semantics while refreshing the next request", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + let version = 1; + const load = cache.cached(async (id: string) => ({ id, version }), { + keyType: "user_id", + useCase: "CoordinatedRequestSnapshot", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: new DialCacheKeyConfig({ + requestLocal: true, + ttlSec: { [CacheLayer.LOCAL]: 60 }, + ramp: { [CacheLayer.LOCAL]: 100 }, + }), + }); + + const sameRequest = await cache.enable(async () => { + const before = await load("123"); + version = 2; + await cache.invalidateRemote("user_id", "123"); + const after = await load("123"); + return { before, after }; + }); + const nextRequest = await cache.enable(async () => await load("123")); + + expect(sameRequest.after).toBe(sameRequest.before); + expect(nextRequest.version).toBe(2); + }); + + it("suppresses coordinated local storage until initial readiness", async () => { + const redis = new FakeRedis(); + const coordinator = new InvalidationCoordinator("urn"); + const cache = coordinatedCache(redis, coordinator); + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, call: ++calls }), { + keyType: "user_id", + useCase: "InitialCoordinationHealth", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + expect(calls).toBe(2); + + coordinator.ready(); + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + expect(calls).toBe(3); + }); + + it("permanently disables coordinated local storage when an instance is disposed", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, call: ++calls }), { + keyType: "user_id", + useCase: "DisposedCoordination", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + await cache.enable(async () => await load("123")); + cache.dispose(); + cache.dispose(); + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + + expect(calls).toBe(3); + }); + + it("establishes terminal local safety before invoking coordinator cleanup", async () => { + const redis = new FakeRedis(); + let attachedListener: DialCacheInvalidationCoordinatorListener | undefined; + const coordinator: DialCacheInvalidationCoordinator = { + namespace: "urn", + state: "ready", + addListener(listener) { + attachedListener = listener; + listener.onStateChange("ready"); + return () => { + throw new Error("listener removal failed"); + }; + }, + invalidate(invalidation) { + attachedListener?.onInvalidation(invalidation); + }, + }; + const cache = coordinatedCache(redis, coordinator); + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, call: ++calls }), { + keyType: "user_id", + useCase: "ThrowingCoordinationCleanup", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + await cache.enable(async () => await load("123")); + expect(() => cache.dispose()).toThrow("listener removal failed"); + attachedListener?.onStateChange("ready"); + attachedListener?.onInvalidation({ + namespace: "urn", + keyType: "user_id", + id: "123", + remainingMs: 1_000, + source: "event", + }); + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + + expect(calls).toBe(3); + }); + + it("records bounded local applications and isolates logger/metrics failures", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const invalidation = vi.fn(() => { + throw new Error("metrics failed"); + }); + const metrics = metricsAdapter({ invalidation }); + const logger = { + debug: vi.fn(() => { + throw new Error("logger failed"); + }), + warn: vi.fn(() => { + throw new Error("logger failed"); + }), + error: vi.fn(), + }; + const cache = coordinatedCache(redis, coordinator, { metrics, logger }); + + await expect(cache.invalidateRemote("user_id", "123", 100)).resolves.toBeUndefined(); + expect(invalidation).toHaveBeenCalledWith({ + cacheNamespace: "urn", + keyType: "user_id", + layer: CacheLayer.REMOTE, + }); + expect(invalidation).toHaveBeenCalledWith({ + cacheNamespace: "urn", + keyType: "user_id", + layer: CacheLayer.LOCAL, + }); + + expect(() => coordinator.unavailable(new Error("subscriber failed"))).not.toThrow(); + expect(() => coordinator.ready()).not.toThrow(); + }); + + it("rejects invalid caller identity without degrading coordination or evicting local values", async () => { + const redis = new FakeRedis(); + const coordinator = readyCoordinator(); + const cache = coordinatedCache(redis, coordinator); + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, call: ++calls }), { + keyType: "user_id", + useCase: "InvalidCallerIdentity", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + await cache.enable(async () => await load("safe")); + await expect(cache.invalidateRemote("user_id", "bad{id", 100)).rejects.toThrow(/braces/); + + expect(coordinator.state).toBe("ready"); + await cache.enable(async () => await load("safe")); + expect(calls).toBe(1); + }); + + it("fails the affected instance unavailable on an invalid custom-coordinator signal", async () => { + const redis = new FakeRedis(); + let attachedListener: DialCacheInvalidationCoordinatorListener | undefined; + const coordinator: DialCacheInvalidationCoordinator = { + namespace: "urn", + state: "ready", + addListener(listener) { + attachedListener = listener; + listener.onStateChange("ready"); + return () => undefined; + }, + invalidate(invalidation) { + attachedListener?.onInvalidation(invalidation); + }, + }; + const cache = coordinatedCache(redis, coordinator); + let calls = 0; + const load = cache.cached(async (id: string) => ({ id, call: ++calls }), { + keyType: "user_id", + useCase: "InvalidCustomSignal", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + await cache.enable(async () => await load("123")); + attachedListener?.onInvalidation({ + namespace: "urn", + keyType: "user_id", + id: "123", + remainingMs: Number.NaN, + source: "event", + }); + await cache.enable(async () => await load("123")); + await cache.enable(async () => await load("123")); + + expect(calls).toBe(3); + }); + + it("validates coordinator namespace and coordinated client capability at construction", () => { + const redis = new FakeRedis(); + const wrongNamespace = new InvalidationCoordinator("other"); + expect(() => new DialCache({ + namespace: "urn", + redis: { client: redis, coordinator: wrongNamespace }, + })).toThrow(/namespace must match/); + + const legacyClient = { + read: async () => null, + write: async () => true, + invalidate: async () => undefined, + }; + expect(() => new DialCache({ + redis: { + client: legacyClient, + coordinator: readyCoordinator(), + } as unknown as RedisConfig, + })).toThrow(/supports invalidateAndPublish/); + + expect(() => new DialCache({ + redis: { + client: redis, + coordinator: null, + } as unknown as RedisConfig, + })).toThrow(/coordinator must be an object/); + }); +}); + +function readyCoordinator(): InvalidationCoordinator { + const coordinator = new InvalidationCoordinator("urn"); + coordinator.ready(); + return coordinator; +} + +function coordinatedCache( + redis: FakeRedis, + coordinator: DialCacheInvalidationCoordinator, + options: { + readonly metrics?: DialCacheMetricsAdapter; + readonly logger?: Pick; + } = {}, +): DialCache { + return new DialCache({ + redis: { client: redis, coordinator, readTimeoutMs: 1_000 }, + ...options, + }); +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} + +function metricsAdapter( + overrides: Partial = {}, +): DialCacheMetricsAdapter { + return { + request: vi.fn(), + miss: vi.fn(), + disabled: vi.fn(), + error: vi.fn(), + invalidation: vi.fn(), + observeGet: vi.fn(), + observeFallback: vi.fn(), + observeSerialization: vi.fn(), + observeSize: vi.fn(), + ...overrides, + }; +} diff --git a/test/fake-redis.ts b/test/fake-redis.ts index 93f5102..48e78f9 100644 --- a/test/fake-redis.ts +++ b/test/fake-redis.ts @@ -1,6 +1,8 @@ import type { - DialCacheRedisClient, + DialCacheCoordinatedRedisClient, + DialCacheInvalidationEventV1, RedisCachePayload, + RedisCoordinatedInvalidationRequest, RedisInvalidationRequest, RedisReadRequest, RedisWriteRequest, @@ -17,7 +19,7 @@ interface StoredValue { expiresAtMs: number; } -export class FakeRedis implements DialCacheRedisClient { +export class FakeRedis implements DialCacheCoordinatedRedisClient { readonly values = new Map(); getCalls = 0; mGetCalls = 0; @@ -26,6 +28,9 @@ export class FakeRedis implements DialCacheRedisClient { failSet = false; failWatermarkGet = false; getGate: Promise | null = null; + coordinatedInvalidationGate: Promise | null = null; + readonly coordinatedInvalidations: RedisCoordinatedInvalidationRequest[] = []; + readonly publishedInvalidations: DialCacheInvalidationEventV1[] = []; async read({ valueKey, watermarkKey }: RedisReadRequest): Promise { if (watermarkKey === undefined) { @@ -81,6 +86,31 @@ export class FakeRedis implements DialCacheRedisClient { this.storeWatermark(watermarkKey, watermark, desiredTtlMs); } + async invalidateAndPublish( + request: RedisCoordinatedInvalidationRequest, + ): Promise { + this.coordinatedInvalidations.push(request); + if (this.coordinatedInvalidationGate !== null) { + await this.coordinatedInvalidationGate; + } + const redisNowMs = Date.now(); + await this.invalidate(request); + const effectiveWatermarkMs = this.readWatermarkValue(request.watermarkKey); + if (effectiveWatermarkMs === null) { + throw new Error("missing coordinated invalidation watermark"); + } + const event = { + version: 1, + namespace: request.namespace, + keyType: request.keyType, + id: request.id, + effectiveWatermarkMs: String(Math.ceil(effectiveWatermarkMs)), + redisNowMs: String(redisNowMs), + } as const; + this.publishedInvalidations.push(event); + return event; + } + raw(key: string): Buffer { const value = this.readRaw(key); if (value === null) { diff --git a/test/invalidation-event.test.ts b/test/invalidation-event.test.ts new file mode 100644 index 0000000..e23c0bf --- /dev/null +++ b/test/invalidation-event.test.ts @@ -0,0 +1,356 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { + DialCacheInvalidationCoordinatorListener, + DialCacheInvalidationEventV1, +} from "../src/invalidation.js"; +import { InvalidationCoordinator } from "../src/internal/invalidation-coordinator.js"; +import { + MAX_REDIS_INVALIDATION_EVENT_BYTES, + REDIS_INVALIDATION_EVENT_VERSION, + decodeRedisInvalidationEvent, + isValidLocalInvalidation, + localInvalidationFromEvent, + redisInvalidationChannel, + validateRedisInvalidationEvent, +} from "../src/internal/invalidation-event.js"; +import { MAX_SUPPORTED_DURATION_MS } from "../src/internal/duration.js"; +import { DialCacheRedisProtocolError } from "../src/redis-client.js"; + +const validEvent: DialCacheInvalidationEventV1 = { + version: 1, + namespace: "users:production", + keyType: "user_id", + id: "123", + effectiveWatermarkMs: "1785300010000", + redisNowMs: "1785300000000", +}; + +describe("Redis invalidation event protocol", () => { + it("decodes the versioned event from strings and bytes", () => { + const payload = JSON.stringify(validEvent); + + expect(decodeRedisInvalidationEvent(payload)).toEqual(validEvent); + expect(decodeRedisInvalidationEvent(Buffer.from(payload), { + namespace: validEvent.namespace, + keyType: validEvent.keyType, + id: validEvent.id, + })).toEqual(validEvent); + expect(localInvalidationFromEvent(validEvent)).toEqual({ + namespace: "users:production", + keyType: "user_id", + id: "123", + remainingMs: 10_000, + source: "event", + }); + }); + + it("builds one deterministic namespace channel and validates namespace braces", () => { + expect(REDIS_INVALIDATION_EVENT_VERSION).toBe(1); + expect(redisInvalidationChannel("users:production")).toBe( + "dialcache:invalidation:v1:users%3Aproduction", + ); + expect(redisInvalidationChannel("tenant*prod!(v1)")).toBe( + "dialcache:invalidation:v1:tenant%2Aprod%21%28v1%29", + ); + expect(() => redisInvalidationChannel("bad{namespace")).toThrow(TypeError); + }); + + it.each([ + null, + [], + "event", + 1, + true, + ])("rejects non-object event value %j", (value) => { + expect(() => validateRedisInvalidationEvent(value)).toThrow( + DialCacheRedisProtocolError, + ); + }); + + it("rejects missing and extra fields", () => { + const { id: _id, ...missing } = validEvent; + + expect(() => validateRedisInvalidationEvent(missing)).toThrow(/unexpected fields/); + expect(() => validateRedisInvalidationEvent({ ...validEvent, extra: true })).toThrow( + /unexpected fields/, + ); + }); + + it.each([ + { ...validEvent, version: 2 }, + { ...validEvent, version: "1" }, + { ...validEvent, namespace: 1 }, + { ...validEvent, keyType: null }, + { ...validEvent, id: {} }, + { ...validEvent, namespace: "bad}namespace" }, + { ...validEvent, keyType: "bad{type" }, + { ...validEvent, id: "bad}id" }, + ])("rejects invalid versions and identities", (event) => { + expect(() => validateRedisInvalidationEvent(event)).toThrow( + DialCacheRedisProtocolError, + ); + }); + + it.each([ + { ...validEvent, effectiveWatermarkMs: 1 }, + { ...validEvent, effectiveWatermarkMs: "01" }, + { ...validEvent, effectiveWatermarkMs: "1.5" }, + { ...validEvent, effectiveWatermarkMs: "-1" }, + { ...validEvent, effectiveWatermarkMs: String(Number.MAX_SAFE_INTEGER + 1) }, + { ...validEvent, effectiveWatermarkMs: "100", redisNowMs: "101" }, + { + ...validEvent, + effectiveWatermarkMs: String(1_000 + MAX_SUPPORTED_DURATION_MS + 1), + redisNowMs: "1000", + }, + ])("rejects invalid timing values", (event) => { + expect(() => validateRedisInvalidationEvent(event)).toThrow(/event timing/); + }); + + it("rejects a mismatched expected identity", () => { + expect(() => decodeRedisInvalidationEvent(JSON.stringify(validEvent), { + namespace: "other", + })).toThrow(/event identity/); + expect(() => decodeRedisInvalidationEvent(JSON.stringify(validEvent), { + keyType: "other", + })).toThrow(/event identity/); + expect(() => decodeRedisInvalidationEvent(JSON.stringify(validEvent), { + id: "other", + })).toThrow(/event identity/); + }); + + it("rejects malformed JSON, invalid UTF-8, and oversized payloads", () => { + expect(() => decodeRedisInvalidationEvent("{")).toThrow(/valid JSON/); + expect(() => decodeRedisInvalidationEvent(Buffer.from([0xff]))).toThrow(/valid UTF-8/); + expect(() => decodeRedisInvalidationEvent("x".repeat(MAX_REDIS_INVALIDATION_EVENT_BYTES + 1))) + .toThrow(/too large/); + expect(() => decodeRedisInvalidationEvent( + Buffer.alloc(MAX_REDIS_INVALIDATION_EVENT_BYTES + 1), + )).toThrow(/too large/); + }); + + it("validates backend-neutral local signals without trusting their runtime shape", () => { + const invalidation = { + namespace: "urn", + keyType: "user_id", + id: "123", + remainingMs: 1_000, + source: "event", + }; + + expect(isValidLocalInvalidation(invalidation, "urn")).toBe(true); + expect(isValidLocalInvalidation(null, "urn")).toBe(false); + expect(isValidLocalInvalidation([], "urn")).toBe(false); + expect(isValidLocalInvalidation({}, "urn")).toBe(false); + expect(isValidLocalInvalidation({ ...invalidation, namespace: "other" }, "urn")).toBe(false); + expect(isValidLocalInvalidation({ ...invalidation, id: 123 }, "urn")).toBe(false); + expect(isValidLocalInvalidation({ ...invalidation, id: "bad{id" }, "urn")).toBe(false); + expect(isValidLocalInvalidation({ ...invalidation, remainingMs: Number.NaN }, "urn")).toBe( + false, + ); + expect(isValidLocalInvalidation({ ...invalidation, source: "unknown" }, "urn")).toBe(false); + }); +}); + +describe("backend-neutral invalidation coordinator", () => { + it("delivers current health, synchronous fan-out, transitions, and removal", () => { + const coordinator = new InvalidationCoordinator("urn"); + const first = listener(); + const second = listener(); + + const removeFirst = coordinator.addListener(first.value); + coordinator.addListener(second.value); + expect(first.states).toEqual([["unavailable", undefined]]); + expect(second.states).toEqual([["unavailable", undefined]]); + + coordinator.ready(); + const invalidation = { + namespace: "urn", + keyType: "user_id", + id: "123", + remainingMs: 50, + source: "event" as const, + }; + expect(coordinator.invalidate(invalidation)).toBe(true); + expect(first.invalidations).toEqual([invalidation]); + expect(second.invalidations).toEqual([invalidation]); + + removeFirst(); + removeFirst(); + expect(coordinator.invalidate({ ...invalidation, id: "456" })).toBe(true); + expect(first.invalidations).toHaveLength(1); + expect(second.invalidations).toHaveLength(2); + + const error = new Error("subscriber unavailable"); + coordinator.unavailable(error); + coordinator.unavailable(new Error("duplicate state")); + coordinator.ready(); + coordinator.dispose(); + coordinator.dispose(); + expect(second.states).toEqual([ + ["unavailable", undefined], + ["ready", undefined], + ["unavailable", error], + ["ready", undefined], + ["disposed", undefined], + ]); + }); + + it("isolates listener exceptions from sibling listeners", () => { + const coordinator = new InvalidationCoordinator("urn"); + const sibling = listener(); + coordinator.addListener({ + onInvalidation: () => { + throw new Error("listener failed"); + }, + onStateChange: () => { + throw new Error("listener failed"); + }, + }); + coordinator.addListener(sibling.value); + + expect(() => coordinator.ready()).not.toThrow(); + expect(() => coordinator.invalidate({ + namespace: "urn", + keyType: "user_id", + id: "123", + remainingMs: 0, + source: "provisional", + })).not.toThrow(); + expect(sibling.invalidations).toHaveLength(1); + }); + + it("does not deliver stale fan-out after a listener reentrantly disposes", () => { + const transitionCoordinator = new InvalidationCoordinator("urn"); + let disposeOnReady = false; + transitionCoordinator.addListener({ + onInvalidation: () => undefined, + onStateChange: (state) => { + if (disposeOnReady && state === "ready") { + transitionCoordinator.dispose(); + } + }, + }); + const transitionSibling = listener(); + transitionCoordinator.addListener(transitionSibling.value); + + disposeOnReady = true; + transitionCoordinator.ready(); + + expect(transitionCoordinator.state).toBe("disposed"); + expect(transitionSibling.states).toEqual([ + ["unavailable", undefined], + ["disposed", undefined], + ]); + + const invalidationCoordinator = new InvalidationCoordinator("urn"); + invalidationCoordinator.ready(); + invalidationCoordinator.addListener({ + onInvalidation: () => invalidationCoordinator.dispose(), + onStateChange: () => undefined, + }); + const invalidationSibling = listener(); + invalidationCoordinator.addListener(invalidationSibling.value); + + expect(invalidationCoordinator.invalidate({ + namespace: "urn", + keyType: "user_id", + id: "123", + remainingMs: 0, + source: "event", + })).toBe(true); + + expect(invalidationCoordinator.state).toBe("disposed"); + expect(invalidationSibling.states).toEqual([ + ["ready", undefined], + ["disposed", undefined], + ]); + expect(invalidationSibling.invalidations).toEqual([]); + }); + + it("decodes subscriber bytes once and moves unavailable on protocol failure", () => { + const coordinator = new InvalidationCoordinator(validEvent.namespace); + const recording = listener(); + coordinator.addListener(recording.value); + coordinator.ready(); + + coordinator.receive(JSON.stringify(validEvent)); + expect(recording.invalidations).toEqual([localInvalidationFromEvent(validEvent)]); + + const protocolError = vi.fn(); + coordinator.addListener({ + onInvalidation: () => undefined, + onStateChange: (state, error) => { + if (state === "unavailable") { + protocolError(error); + } + }, + }); + expect(() => coordinator.receive(Buffer.from([0xff]))).not.toThrow(); + expect(coordinator.state).toBe("unavailable"); + expect(protocolError).toHaveBeenCalledWith(expect.any(DialCacheRedisProtocolError)); + }); + + it("rejects wrong channels and invalid local signals conservatively", () => { + const coordinator = new InvalidationCoordinator(validEvent.namespace); + coordinator.ready(); + + coordinator.receive(JSON.stringify(validEvent), "wrong-channel"); + expect(coordinator.state).toBe("unavailable"); + + coordinator.ready(); + coordinator.invalidate({ + namespace: validEvent.namespace, + keyType: validEvent.keyType, + id: validEvent.id, + remainingMs: MAX_SUPPORTED_DURATION_MS + 1, + source: "event", + }); + expect(coordinator.state).toBe("unavailable"); + + coordinator.ready(); + coordinator.invalidate({ + namespace: "wrong", + keyType: validEvent.keyType, + id: validEvent.id, + remainingMs: 0, + source: "event", + }); + expect(coordinator.state).toBe("unavailable"); + }); + + it("ignores messages and registrations safely after disposal", () => { + const coordinator = new InvalidationCoordinator(validEvent.namespace); + coordinator.dispose(); + const recording = listener(); + + const remove = coordinator.addListener(recording.value); + expect(coordinator.receive(JSON.stringify(validEvent))).toBe(false); + expect(coordinator.invalidate(localInvalidationFromEvent(validEvent))).toBe(false); + coordinator.ready(); + coordinator.unavailable(new Error("late health callback")); + remove(); + + expect(coordinator.state).toBe("disposed"); + expect(recording.states).toEqual([["disposed", undefined]]); + expect(recording.invalidations).toEqual([]); + }); +}); + +function listener(): { + readonly value: DialCacheInvalidationCoordinatorListener; + readonly states: Array; + readonly invalidations: unknown[]; +} { + const states: Array = []; + const invalidations: unknown[] = []; + return { + states, + invalidations, + value: { + onInvalidation: (invalidation) => invalidations.push(invalidation), + onStateChange: (state, error) => states.push([state, error]), + }, + }; +} diff --git a/test/local-invalidation.test.ts b/test/local-invalidation.test.ts new file mode 100644 index 0000000..d2009f2 --- /dev/null +++ b/test/local-invalidation.test.ts @@ -0,0 +1,267 @@ +import { performance } from "node:perf_hooks"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { DialCacheKey } from "../src/key.js"; +import { LocalCache } from "../src/internal/local-cache.js"; +import { LocalInvalidationState } from "../src/internal/local-invalidation.js"; + +const resolvedLocal = { ttlSec: 60, ramp: 100 }; +const tracked = ( + id: string, + useCase: string, + args: ReadonlyArray = [], + namespace = "urn", +) => new DialCacheKey({ + namespace, + keyType: "user_id", + id, + useCase, + args, + trackForInvalidation: true, +}); +const untracked = (id: string, useCase: string) => new DialCacheKey({ + keyType: "user_id", + id, + useCase, +}); + +describe("LocalCache tracked eviction", () => { + it("deletes one exact tracked identity across use cases and argument variants", async () => { + const cache = new LocalCache(() => null, 10); + const profile = tracked("123", "Profile"); + const permissions = tracked("123", "Permissions", [["scope", "admin"]]); + const adjacent = tracked("1234", "Adjacent"); + const otherType = new DialCacheKey({ + keyType: "account_id", + id: "123", + useCase: "OtherType", + trackForInvalidation: true, + }); + const plain = untracked("123", "Plain"); + + await Promise.all([ + cache.put(profile, "profile", resolvedLocal), + cache.put(permissions, "permissions", resolvedLocal), + cache.put(adjacent, "adjacent", resolvedLocal), + cache.put(otherType, "other-type", resolvedLocal), + cache.put(plain, "plain", resolvedLocal), + ]); + + expect(cache.deleteTrackedPrefix(profile.prefix)).toBe(2); + expect(cache.getWithResolvedConfig(profile, resolvedLocal).status).toBe("miss"); + expect(cache.getWithResolvedConfig(permissions, resolvedLocal).status).toBe("miss"); + expect(cache.getWithResolvedConfig(adjacent, resolvedLocal)).toMatchObject({ + status: "hit", + value: "adjacent", + }); + expect(cache.getWithResolvedConfig(otherType, resolvedLocal)).toMatchObject({ + status: "hit", + value: "other-type", + }); + expect(cache.getWithResolvedConfig(plain, resolvedLocal)).toMatchObject({ + status: "hit", + value: "plain", + }); + }); + + it("clears every tracked entry while preserving untracked entries", async () => { + const cache = new LocalCache(() => null, 10); + const first = tracked("1", "First"); + const second = tracked("2", "Second"); + const plain = untracked("1", "Plain"); + await cache.put(first, 1, resolvedLocal); + await cache.put(second, 2, resolvedLocal); + await cache.put(plain, 3, resolvedLocal); + + expect(cache.clearTracked()).toBe(2); + expect(cache.clearTracked()).toBe(0); + expect(cache.getWithResolvedConfig(first, resolvedLocal).status).toBe("miss"); + expect(cache.getWithResolvedConfig(second, resolvedLocal).status).toBe("miss"); + expect(cache.getWithResolvedConfig(plain, resolvedLocal)).toMatchObject({ + status: "hit", + value: 3, + }); + }); + + it("keeps zero-capacity operations allocation-free and no-op", async () => { + const cache = new LocalCache(() => { + throw new Error("provider must not run"); + }, 0); + const key = tracked("1", "Disabled"); + const guard = vi.fn(() => true); + + await expect(cache.put(key, "value", undefined, guard)).resolves.toBeUndefined(); + expect(guard).not.toHaveBeenCalled(); + expect(cache.deleteTrackedPrefix(key.prefix)).toBe(0); + expect(cache.clearTracked()).toBe(0); + }); + + it("checks a publication guard immediately before the synchronous set", async () => { + const cache = new LocalCache(() => null, 2); + const key = tracked("1", "Guarded"); + const denied = vi.fn(() => false); + + await cache.put(key, "denied", resolvedLocal, denied); + expect(denied).toHaveBeenCalledTimes(1); + expect(cache.getWithResolvedConfig(key, resolvedLocal).status).toBe("miss"); + + await cache.put(key, "allowed", resolvedLocal, () => true); + expect(cache.getWithResolvedConfig(key, resolvedLocal)).toMatchObject({ + status: "hit", + value: "allowed", + }); + }); +}); + +describe("bounded process-local invalidation state", () => { + let nowMs = 1_000; + + beforeEach(() => { + nowMs = 1_000; + vi.spyOn(performance, "now").mockImplementation(() => nowMs); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("installs the fence before eviction and uses only monotonic time", async () => { + const cache = new LocalCache(() => null, 10); + const state = new LocalInvalidationState(cache, 10); + const key = tracked("123", "Profile"); + const other = tracked("456", "Other"); + state.transition("ready"); + const permit = state.capturePublicationPermit(); + await cache.put(key, "old", resolvedLocal); + + expect(state.apply(signal("123", 100))).toBe(1); + expect(cache.getWithResolvedConfig(key, resolvedLocal).status).toBe("miss"); + expect(state.canPublish(key, permit)).toBe(false); + expect(state.canPublish(other, permit)).toBe(true); + + vi.spyOn(Date, "now").mockReturnValue(Date.parse("2040-01-01T00:00:00.000Z")); + nowMs = 1_100; + expect(state.canPublish(key, permit)).toBe(false); + nowMs = 1_100.001; + expect(state.canPublish(key, permit)).toBe(true); + }); + + it("never shortens an identity fence and rescans equal events", async () => { + const cache = new LocalCache(() => null, 10); + const state = new LocalInvalidationState(cache, 10); + const key = tracked("123", "Profile"); + state.transition("ready"); + const permit = state.capturePublicationPermit(); + + state.apply(signal("123", 100)); + nowMs = 1_050; + await cache.put(key, "late", resolvedLocal); + expect(state.apply(signal("123", 10))).toBe(1); + expect(cache.getWithResolvedConfig(key, resolvedLocal).status).toBe("miss"); + nowMs = 1_070; + expect(state.canPublish(key, permit)).toBe(false); + nowMs = 1_101; + expect(state.canPublish(key, permit)).toBe(true); + }); + + it("advances health epochs on outage and recovery and preserves active fences", async () => { + const cache = new LocalCache(() => null, 10); + const state = new LocalInvalidationState(cache, 10); + const key = tracked("123", "Profile"); + const plain = untracked("123", "Plain"); + state.transition("ready"); + const beforeGap = state.capturePublicationPermit(); + state.apply(signal("123", 100)); + await cache.put(key, "tracked", resolvedLocal); + await cache.put(plain, "plain", resolvedLocal); + + expect(state.transition("unavailable")).toEqual({ changed: true, evicted: 1 }); + expect(state.transition("unavailable")).toEqual({ changed: false, evicted: 0 }); + expect(cache.getWithResolvedConfig(plain, resolvedLocal)).toMatchObject({ + status: "hit", + value: "plain", + }); + expect(state.canPublish(key, beforeGap)).toBe(false); + + expect(state.transition("ready")).toEqual({ changed: true, evicted: 0 }); + const afterGap = state.capturePublicationPermit(); + expect(state.canPublish(key, beforeGap)).toBe(false); + expect(state.canPublish(key, afterGap)).toBe(false); + nowMs = 1_101; + expect(state.canPublish(key, afterGap)).toBe(true); + }); + + it("collapses active overflow into a conservative global deadline", () => { + const cache = new LocalCache(() => null, 2); + const state = new LocalInvalidationState(cache, 2); + const one = tracked("1", "One"); + const unrelated = tracked("unrelated", "Unrelated"); + state.transition("ready"); + const permit = state.capturePublicationPermit(); + + state.apply(signal("1", 100)); + state.apply(signal("2", 200)); + state.apply(signal("3", 150)); + + expect(state.canPublish(one, permit)).toBe(false); + expect(state.canPublish(unrelated, permit)).toBe(false); + nowMs = 1_200; + expect(state.canPublish(unrelated, permit)).toBe(false); + nowMs = 1_200.001; + expect(state.canPublish(unrelated, permit)).toBe(true); + }); + + it("prunes expired identities before choosing global overflow", () => { + const cache = new LocalCache(() => null, 2); + const state = new LocalInvalidationState(cache, 2); + const unrelated = tracked("unrelated", "Unrelated"); + state.transition("ready"); + const permit = state.capturePublicationPermit(); + + state.apply(signal("1", 10)); + state.apply(signal("2", 100)); + nowMs = 1_020; + state.apply(signal("3", 50)); + + expect(state.canPublish(unrelated, permit)).toBe(true); + expect(state.canPublish(tracked("2", "Two"), permit)).toBe(false); + expect(state.canPublish(tracked("3", "Three"), permit)).toBe(false); + }); + + it("suppresses publication permanently after disposal and releases fence state", () => { + const cache = new LocalCache(() => null, 1); + const state = new LocalInvalidationState(cache, 1); + const key = tracked("1", "One"); + state.transition("ready"); + const permit = state.capturePublicationPermit(); + state.apply(signal("1", 100)); + + expect(state.transition("disposed")).toEqual({ changed: true, evicted: 0 }); + expect(state.apply(signal("1", 100))).toBe(0); + expect(state.transition("ready")).toEqual({ changed: false, evicted: 0 }); + nowMs = 2_000; + expect(state.canPublish(key, permit)).toBe(false); + }); + + it("retains no values or fences when local capacity is zero", () => { + const cache = new LocalCache(() => null, 0); + const state = new LocalInvalidationState(cache, 0); + const key = tracked("1", "One"); + + state.transition("ready"); + const permit = state.capturePublicationPermit(); + expect(state.apply(signal("1", 100))).toBe(0); + expect(state.canPublish(key, permit)).toBe(true); + }); +}); + +function signal(id: string, remainingMs: number) { + return { + namespace: "urn", + keyType: "user_id", + id, + remainingMs, + source: "event" as const, + }; +} diff --git a/test/node-redis-coordination.test.ts b/test/node-redis-coordination.test.ts new file mode 100644 index 0000000..e5e45e0 --- /dev/null +++ b/test/node-redis-coordination.test.ts @@ -0,0 +1,437 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { DialCacheInvalidationCoordinatorListener } from "../src/invalidation.js"; +import { + createNodeRedisDialCacheClient, + createNodeRedisDialCacheInvalidationCoordinator, + dialcacheRedisScripts, + type DialCacheNodeRedisSubscriberClient, +} from "../src/node-redis.js"; +import { INVALIDATE_AND_PUBLISH_CACHE_SCRIPT } from "../src/redis-protocol.js"; +import { DialCacheRedisProtocolError } from "../src/redis-client.js"; + +const event = { + version: 1, + namespace: "users", + keyType: "user_id", + id: "123", + effectiveWatermarkMs: "1785300001000", + redisNowMs: "1785300000000", +} as const; +const payload = JSON.stringify(event); + +describe("node-redis coordinated invalidation script adapter", () => { + it("registers one-key write routing and the exact argument order", () => { + const script = dialcacheRedisScripts.dialcacheInvalidateAndPublish; + + expect(script.SCRIPT).toBe(INVALIDATE_AND_PUBLISH_CACHE_SCRIPT); + expect(script.NUMBER_OF_KEYS).toBe(1); + expect(script.FIRST_KEY_INDEX).toBe(0); + expect(script.IS_READ_ONLY).toBe(false); + expect(script.transformArguments( + "{users:user_id:123}#watermark", + 250, + "dialcache:invalidation:v1:users", + "users", + "user_id", + "123", + )).toEqual([ + "{users:user_id:123}#watermark", + "250", + "dialcache:invalidation:v1:users", + "users", + "user_id", + "123", + ]); + expect(script.transformReply(payload)).toBe(payload); + }); + + it.each([ + 1, + null, + undefined, + "{}", + "{", + JSON.stringify({ ...event, version: 2 }), + ])("rejects malformed coordinated script reply %j", (reply) => { + expect(() => dialcacheRedisScripts.dialcacheInvalidateAndPublish.transformReply( + reply as string, + )).toThrow(DialCacheRedisProtocolError); + }); + + it("adds the coordinated method only when the registered script is present", async () => { + const client = coordinatedScriptClient(payload); + const adapter = createNodeRedisDialCacheClient(client); + + await expect(adapter.invalidateAndPublish({ + watermarkKey: "{users:user_id:123}#watermark", + futureBufferMs: 1_000, + channel: "dialcache:invalidation:v1:users", + namespace: "users", + keyType: "user_id", + id: "123", + })).resolves.toEqual(event); + expect(client.dialcacheInvalidateAndPublish).toHaveBeenCalledWith( + "{users:user_id:123}#watermark", + 1_000, + "dialcache:invalidation:v1:users", + "users", + "user_id", + "123", + ); + + const legacy = createNodeRedisDialCacheClient(baseScriptClient()); + expect("invalidateAndPublish" in legacy).toBe(false); + }); + + it("rejects a returned event that does not match the requested identity", async () => { + const adapter = createNodeRedisDialCacheClient( + coordinatedScriptClient(JSON.stringify({ ...event, id: "other" })), + ); + + await expect(adapter.invalidateAndPublish({ + watermarkKey: "{users:user_id:123}#watermark", + futureBufferMs: 0, + channel: "dialcache:invalidation:v1:users", + namespace: "users", + keyType: "user_id", + id: "123", + })).rejects.toThrow(/event identity/); + }); +}); + +describe("node-redis invalidation coordinator", () => { + it("awaits subscription acknowledgement and delivers Buffer events", async () => { + const subscriber = new FakeSubscriber(); + const acknowledgement = deferred(); + subscriber.subscribeGate = acknowledgement.promise; + + let settled = false; + const pending = createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }).then((value) => { + settled = true; + return value; + }); + await Promise.resolve(); + + expect(settled).toBe(false); + expect(subscriber.subscribe).toHaveBeenCalledWith( + "dialcache:invalidation:v1:users", + expect.any(Function), + true, + ); + expect(subscriber.listenerCount("error")).toBe(1); + acknowledgement.resolve(); + const coordinator = await pending; + expect(coordinator.state).toBe("ready"); + + const recording = listener(); + coordinator.addListener(recording.value); + subscriber.publish(Buffer.from(payload)); + expect(recording.invalidations).toEqual([{ + namespace: "users", + keyType: "user_id", + id: "123", + remainingMs: 1_000, + source: "event", + }]); + + await coordinator.dispose(); + }); + + it("does not erase a protocol failure delivered with the initial subscribe acknowledgement", async () => { + const subscriber = new FakeSubscriber(); + subscriber.subscribeAcknowledgementHook = () => { + subscriber.publish(Buffer.from([0xff])); + }; + + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }); + + expect(coordinator.state).toBe("unavailable"); + subscriber.emit("ready"); + expect(coordinator.state).toBe("unavailable"); + + subscriber.emit("reconnecting"); + subscriber.emit("ready"); + expect(coordinator.state).toBe("ready"); + await coordinator.dispose(); + }); + + it("tracks reconnect, ready-after-resubscribe, error, and end transitions", async () => { + const subscriber = new FakeSubscriber(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }); + const recording = listener(); + coordinator.addListener(recording.value); + + subscriber.emit("reconnecting"); + subscriber.emit("ready"); + const redisError = new Error("socket failed"); + subscriber.emit("error", redisError); + subscriber.emit("ready"); + subscriber.emit("end"); + + expect(recording.states).toEqual([ + ["ready", undefined], + ["unavailable", expect.any(Error)], + ["ready", undefined], + ["unavailable", redisError], + ["ready", undefined], + ["unavailable", expect.any(Error)], + ]); + await coordinator.dispose(); + }); + + it("contains malformed callbacks and remains unavailable until acknowledged recovery", async () => { + const subscriber = new FakeSubscriber(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }); + const recording = listener(); + coordinator.addListener(recording.value); + + expect(() => subscriber.publish(Buffer.from([0xff]))).not.toThrow(); + expect(coordinator.state).toBe("unavailable"); + subscriber.publish(Buffer.from(payload)); + expect(recording.invalidations).toHaveLength(1); + expect(coordinator.state).toBe("unavailable"); + + subscriber.emit("ready"); + expect(coordinator.state).toBe("unavailable"); + subscriber.emit("reconnecting"); + subscriber.emit("ready"); + expect(coordinator.state).toBe("ready"); + subscriber.publish(Buffer.from(payload), Buffer.from("wrong")); + expect(coordinator.state).toBe("unavailable"); + await coordinator.dispose(); + }); + + it("does not erase a protocol failure received between reconnect and ready", async () => { + const subscriber = new FakeSubscriber(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }); + + subscriber.emit("reconnecting"); + subscriber.publish(Buffer.from([0xff])); + subscriber.emit("ready"); + expect(coordinator.state).toBe("unavailable"); + + subscriber.emit("reconnecting"); + subscriber.emit("ready"); + expect(coordinator.state).toBe("ready"); + await coordinator.dispose(); + }); + + it("unsubscribes only its listener, detaches health listeners, and never closes the client", async () => { + const subscriber = new FakeSubscriber(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }); + const recording = listener(); + coordinator.addListener(recording.value); + const unsubscribe = deferred(); + subscriber.unsubscribeGate = unsubscribe.promise; + let reentrantDispose: Promise | undefined; + coordinator.addListener({ + onInvalidation: () => undefined, + onStateChange: (state) => { + if (state === "disposed") { + reentrantDispose = coordinator.dispose(); + } + }, + }); + + const firstDispose = coordinator.dispose(); + const secondDispose = coordinator.dispose(); + expect(reentrantDispose).toBe(firstDispose); + expect(secondDispose).toBe(firstDispose); + expect(subscriber.unsubscribe).toHaveBeenCalledTimes(1); + expect(subscriber.listenerCount("error")).toBe(0); + expect(subscriber.listenerCount("reconnecting")).toBe(0); + expect(subscriber.listenerCount("ready")).toBe(0); + expect(subscriber.listenerCount("end")).toBe(0); + + subscriber.emit("ready"); + subscriber.emit("error", new Error("late error")); + expect(coordinator.state).toBe("disposed"); + + unsubscribe.resolve(); + await Promise.all([firstDispose, secondDispose]); + + expect(subscriber.unsubscribe).toHaveBeenCalledTimes(1); + expect(subscriber.unsubscribe).toHaveBeenCalledWith( + "dialcache:invalidation:v1:users", + expect.any(Function), + true, + ); + expect(subscriber.isOpen).toBe(true); + expect(recording.states.at(-1)).toEqual(["disposed", undefined]); + }); + + it("attempts listener-specific unsubscribe while an open client reports inactive Pub/Sub", async () => { + const subscriber = new FakeSubscriber(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace: "users", + }); + subscriber.isPubSubActive = false; + + await coordinator.dispose(); + + expect(subscriber.unsubscribe).toHaveBeenCalledTimes(1); + expect(subscriber.unsubscribe).toHaveBeenCalledWith( + "dialcache:invalidation:v1:users", + expect.any(Function), + true, + ); + }); + + it.each([ + { isOpen: false, isReady: false, isPubSubActive: false, message: /connected and ready/ }, + { isOpen: true, isReady: false, isPubSubActive: false, message: /connected and ready/ }, + { isOpen: true, isReady: true, isPubSubActive: true, message: /dedicated and unsubscribed/ }, + ])("rejects invalid caller-owned subscriber state", async (state) => { + const subscriber = new FakeSubscriber(); + subscriber.isOpen = state.isOpen; + subscriber.isReady = state.isReady; + subscriber.isPubSubActive = state.isPubSubActive; + + await expect(createNodeRedisDialCacheInvalidationCoordinator(subscriber)) + .rejects.toThrow(state.message); + expect(subscriber.subscribe).not.toHaveBeenCalled(); + }); + + it("cleans up helper listeners when initial subscribe fails", async () => { + const subscriber = new FakeSubscriber(); + const failure = new Error("subscribe failed"); + subscriber.subscribe.mockRejectedValueOnce(failure); + + await expect(createNodeRedisDialCacheInvalidationCoordinator(subscriber)) + .rejects.toBe(failure); + expect(subscriber.listenerCount("error")).toBe(0); + expect(subscriber.listenerCount("ready")).toBe(0); + expect(subscriber.isOpen).toBe(true); + expect(subscriber.unsubscribe).toHaveBeenCalledTimes(1); + }); +}); + +function baseScriptClient() { + return { + dialcacheRead: vi.fn(async () => null), + dialcacheReadTracked: vi.fn(async () => null), + dialcacheWrite: vi.fn(async () => 1), + dialcacheWriteTracked: vi.fn(async () => 1), + dialcacheInvalidate: vi.fn(async () => 1), + }; +} + +function coordinatedScriptClient(reply: string) { + return { + ...baseScriptClient(), + dialcacheInvalidateAndPublish: vi.fn(async () => reply), + }; +} + +type SubscriberEvent = "error" | "reconnecting" | "ready" | "end"; +type SubscriberHandler = (...args: never[]) => void; + +class FakeSubscriber implements DialCacheNodeRedisSubscriberClient { + isOpen = true; + isReady = true; + isPubSubActive = false; + subscribeGate: Promise | null = null; + subscribeAcknowledgementHook: (() => void) | null = null; + unsubscribeGate: Promise | null = null; + private messageListener: ((message: Buffer, channel: Buffer) => unknown) | null = null; + private subscribedChannel = ""; + private readonly handlers = new Map>(); + + readonly subscribe = vi.fn(async ( + channel: string, + listener: (message: Buffer, channel: Buffer) => unknown, + _bufferMode: true, + ) => { + this.isPubSubActive = true; + this.subscribedChannel = channel; + this.messageListener = listener; + if (this.subscribeGate !== null) { + await this.subscribeGate; + } + this.subscribeAcknowledgementHook?.(); + }); + + readonly unsubscribe = vi.fn(async ( + _channel: string, + listener: (message: Buffer, channel: Buffer) => unknown, + _bufferMode: true, + ) => { + if (this.unsubscribeGate !== null) { + await this.unsubscribeGate; + } + if (this.messageListener === listener) { + this.messageListener = null; + this.isPubSubActive = false; + } + }); + + on(event: "error", listener: (error: Error) => void): this; + on(event: "reconnecting" | "ready" | "end", listener: () => void): this; + on(event: SubscriberEvent, listener: SubscriberHandler): this { + const handlers = this.handlers.get(event) ?? new Set(); + handlers.add(listener); + this.handlers.set(event, handlers); + return this; + } + + off(event: "error", listener: (error: Error) => void): this; + off(event: "reconnecting" | "ready" | "end", listener: () => void): this; + off(event: SubscriberEvent, listener: SubscriberHandler): this { + this.handlers.get(event)?.delete(listener); + return this; + } + + emit(event: "error", error: Error): void; + emit(event: "reconnecting" | "ready" | "end"): void; + emit(event: SubscriberEvent, error?: Error): void { + for (const handler of [...(this.handlers.get(event) ?? [])]) { + handler(error as never); + } + } + + publish(message: Buffer, channel = Buffer.from(this.subscribedChannel)): void { + this.messageListener?.(message, channel); + } + + listenerCount(event: SubscriberEvent): number { + return this.handlers.get(event)?.size ?? 0; + } +} + +function listener(): { + readonly value: DialCacheInvalidationCoordinatorListener; + readonly states: Array; + readonly invalidations: unknown[]; +} { + const states: Array = []; + const invalidations: unknown[] = []; + return { + states, + invalidations, + value: { + onInvalidation: (invalidation) => invalidations.push(invalidation), + onStateChange: (state, error) => states.push([state, error]), + }, + }; +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} diff --git a/test/redis-cluster.integration.test.ts b/test/redis-cluster.integration.test.ts index 1a5f721..19546d7 100644 --- a/test/redis-cluster.integration.test.ts +++ b/test/redis-cluster.integration.test.ts @@ -1,4 +1,4 @@ -import { commandOptions, createCluster, type RedisClusterOptions } from "redis"; +import { commandOptions, createClient, createCluster, type RedisClusterOptions } from "redis"; import { GenericContainer, Network, @@ -8,8 +8,19 @@ import { } from "testcontainers"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { CacheLayer, DialCache, DialCacheKeyConfig, type DialCacheRedisClient } from "../src/index.js"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../src/node-redis.js"; +import { + CacheLayer, + DialCache, + DialCacheKeyConfig, + type DialCacheInvalidationCoordinator, + type DialCacheLocalInvalidation, + type DialCacheRedisClient, +} from "../src/index.js"; +import { + createNodeRedisDialCacheClient, + createNodeRedisDialCacheInvalidationCoordinator, + dialcacheRedisScripts, +} from "../src/node-redis.js"; const remoteOnly = new DialCacheKeyConfig({ ttlSec: { [CacheLayer.REMOTE]: 60 }, @@ -33,10 +44,50 @@ async function waitForCluster(container: StartedTestContainer): Promise { throw new Error("Redis Cluster did not become ready"); } +function nextInvalidation( + coordinator: DialCacheInvalidationCoordinator, + expectedId: string, +): { + readonly promise: Promise; + cancel(): void; + } { + let removeListener = (): void => undefined; + let timeout: ReturnType | undefined; + const promise = new Promise((resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`Timed out waiting for DialCache invalidation ${expectedId}`)), + 5_000, + ); + removeListener = coordinator.addListener({ + onInvalidation(invalidation) { + if (invalidation.id !== expectedId || invalidation.source !== "event") { + return; + } + if (timeout !== undefined) { + clearTimeout(timeout); + } + removeListener(); + resolve(invalidation); + }, + onStateChange: () => undefined, + }); + }); + return { + promise, + cancel() { + if (timeout !== undefined) { + clearTimeout(timeout); + } + removeListener(); + }, + }; +} + describe("DialCache Lua protocol on Redis Cluster", () => { let network: StartedNetwork | undefined; let containers: Array = []; let cluster: ReturnType | undefined; + let standaloneNodeUrl: string | undefined; beforeAll(async () => { const startedNetwork = await new Network().start(); @@ -73,6 +124,8 @@ describe("DialCache Lua protocol on Redis Cluster", () => { if (firstContainer === undefined) { throw new Error("Redis Cluster containers did not start"); } + standaloneNodeUrl = + `redis://${firstContainer.getHost()}:${firstContainer.getMappedPort(6379)}`; const createResult = await firstContainer.exec([ "redis-cli", "--cluster", @@ -219,4 +272,61 @@ describe("DialCache Lua protocol on Redis Cluster", () => { ).toBe(true); expect(await scriptClient.read({ valueKey: trackedValueKey, watermarkKey })).toEqual(trackedPayload); }); + + it("routes coordinated publication and delivers it to a standalone subscriber on a cluster node", async () => { + if (cluster === undefined || standaloneNodeUrl === undefined) { + throw new Error("Redis Cluster did not start"); + } + const activeCluster = cluster; + const namespace = "cluster-coordinated"; + const subscriber = createClient({ url: standaloneNodeUrl }); + subscriber.on("error", () => undefined); + await subscriber.connect(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace, + }); + const scriptClient = createNodeRedisDialCacheClient(activeCluster); + + const publishAndAssertDelivery = async (id: string) => { + const watermarkKey = `{${namespace}:user_id:${id}}#watermark`; + const delivery = nextInvalidation(coordinator, id); + try { + const event = await scriptClient.invalidateAndPublish({ + watermarkKey, + futureBufferMs: 100, + channel: coordinator.channel, + namespace, + keyType: "user_id", + id, + }); + const received = await delivery.promise; + + expect(await activeCluster.get(watermarkKey)).toBe(event.effectiveWatermarkMs); + expect(received).toEqual({ + namespace, + keyType: "user_id", + id, + remainingMs: Number(event.effectiveWatermarkMs) - Number(event.redisNowMs), + source: "event", + }); + } finally { + delivery.cancel(); + } + }; + + try { + expect(coordinator.state).toBe("ready"); + await publishAndAssertDelivery("before-flush"); + await Promise.all( + activeCluster.masters.map(async (master) => { + const client = await activeCluster.nodeClient(master); + await client.scriptFlush(); + }), + ); + await publishAndAssertDelivery("after-flush"); + } finally { + await coordinator.dispose(); + await subscriber.quit(); + } + }); }); diff --git a/test/redis-real.integration.test.ts b/test/redis-real.integration.test.ts index b78883e..6f245c4 100644 --- a/test/redis-real.integration.test.ts +++ b/test/redis-real.integration.test.ts @@ -7,6 +7,9 @@ import { CacheLayer, DialCache, DialCacheKeyConfig, + type DialCacheCoordinatedRedisClient, + type DialCacheInvalidationCoordinator, + type DialCacheLocalInvalidation, type DialCacheMetricsAdapter, type DialCacheRedisClient, type Serializer, @@ -16,7 +19,15 @@ import { WRITE_CACHE_SCRIPT, WRITE_TRACKED_CACHE_SCRIPT, } from "../src/internal/redis-scripts.js"; -import { createNodeRedisDialCacheClient, dialcacheRedisScripts } from "../src/node-redis.js"; +import { + createNodeRedisDialCacheClient, + createNodeRedisDialCacheInvalidationCoordinator, + dialcacheRedisScripts, +} from "../src/node-redis.js"; +import { + MAX_REDIS_INVALIDATION_EVENT_BYTES, + redisInvalidationChannel, +} from "../src/redis-protocol.js"; import { createValkeyGlideDialCacheClient, type ValkeyGlideDialCacheClient, @@ -40,6 +51,11 @@ const remoteOnly = new DialCacheKeyConfig({ ramp: { [CacheLayer.REMOTE]: 100 }, }); +const localOnly = new DialCacheKeyConfig({ + ttlSec: { [CacheLayer.LOCAL]: 60 }, + ramp: { [CacheLayer.LOCAL]: 100 }, +}); + const createTestClient = (url: string) => createClient({ url, scripts: dialcacheRedisScripts }); type NodeRedisTestClient = ReturnType; @@ -61,7 +77,7 @@ interface RawRedisScriptClient { } interface RedisAdapterHarness { - readonly adapter: DialCacheRedisClient; + readonly adapter: DialCacheCoordinatedRedisClient; /** Exercise Lua argument validation that the semantic adapter cannot represent. */ readonly raw: RawRedisScriptClient; dispose(): void; @@ -141,12 +157,52 @@ function encodeFrame(payload: string | Buffer, encoding: number, createdAtMs = D return Buffer.concat([Buffer.from([version]), timestamp, Buffer.from([encoding]), Buffer.from(payload)]); } +function nextInvalidation( + coordinator: DialCacheInvalidationCoordinator, + expectedId: string, +): { + readonly promise: Promise; + cancel(): void; + } { + let removeListener = (): void => undefined; + let timeout: ReturnType | undefined; + const promise = new Promise((resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`Timed out waiting for DialCache invalidation ${expectedId}`)), + 5_000, + ); + removeListener = coordinator.addListener({ + onInvalidation(invalidation) { + if (invalidation.id !== expectedId || invalidation.source !== "event") { + return; + } + if (timeout !== undefined) { + clearTimeout(timeout); + } + removeListener(); + resolve(invalidation); + }, + onStateChange: () => undefined, + }); + }); + return { + promise, + cancel() { + if (timeout !== undefined) { + clearTimeout(timeout); + } + removeListener(); + }, + }; +} + describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { let container: StartedTestContainer | undefined; // This connection controls and inspects server state; cache operations use the selected adapter harness. let admin: NodeRedisTestClient | undefined; let glide: valkeyGlide.GlideClient | undefined; let harnesses: Record | undefined; + let redisUrl: string | undefined; beforeAll(async () => { container = await new GenericContainer(image) @@ -155,7 +211,8 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { .start(); const host = container.getHost(); const port = container.getMappedPort(6379); - admin = createTestClient(`redis://${host}:${port}`); + redisUrl = `redis://${host}:${port}`; + admin = createTestClient(redisUrl); admin.on("error", () => undefined); await admin.connect(); glide = await valkeyGlide.GlideClient.createClient({ addresses: [{ host, port }] }); @@ -313,6 +370,137 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { futureBufferMs: 0, }), ).resolves.toBeUndefined(); + + const coordinatedWatermarkKey = "script-recovery:{item:coordinated}:watermark"; + const coordinatedRequest = { + watermarkKey: coordinatedWatermarkKey, + futureBufferMs: 10, + channel: redisInvalidationChannel("script-recovery"), + namespace: "script-recovery", + keyType: "item", + id: "coordinated", + } as const; + await expect(scriptClient.invalidateAndPublish(coordinatedRequest)).resolves.toEqual( + expect.objectContaining({ + version: 1, + namespace: "script-recovery", + keyType: "item", + id: "coordinated", + }), + ); + await admin.scriptFlush(); + await expect( + scriptClient.invalidateAndPublish({ + ...coordinatedRequest, + watermarkKey: "script-recovery:{item:coordinated-after-flush}:watermark", + id: "coordinated-after-flush", + }), + ).resolves.toEqual( + expect.objectContaining({ + version: 1, + namespace: "script-recovery", + keyType: "item", + id: "coordinated-after-flush", + }), + ); + }); + + it("publishes coordinated invalidations without subscribers and returns the effective watermark", async () => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const namespace = `coordinated-${kind}`; + const channel = redisInvalidationChannel(namespace); + const watermarkKey = `coordinated:{${namespace}:user_id:123}:watermark`; + const futureBufferMs = 5_000; + const first = await client.adapter.invalidateAndPublish({ + watermarkKey, + futureBufferMs, + channel, + namespace, + keyType: "user_id", + id: "123", + }); + + expect(first).toEqual({ + version: 1, + namespace, + keyType: "user_id", + id: "123", + effectiveWatermarkMs: expect.any(String), + redisNowMs: expect.any(String), + }); + expect(Number(first.effectiveWatermarkMs)).toBeGreaterThanOrEqual( + Number(first.redisNowMs) + futureBufferMs, + ); + expect(await admin.get(watermarkKey)).toBe(first.effectiveWatermarkMs); + expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(60_000); + + const redisNowMs = (await admin.time()).getTime(); + const longerWatermarkMs = redisNowMs + 5_000; + await admin.set(watermarkKey, String(longerWatermarkMs), { PX: 120_000 }); + const ttlBefore = await admin.pTTL(watermarkKey); + + const extended = await client.adapter.invalidateAndPublish({ + watermarkKey, + futureBufferMs: 0, + channel, + namespace, + keyType: "user_id", + id: "123", + }); + + expect(extended.effectiveWatermarkMs).toBe(String(longerWatermarkMs)); + expect(await admin.get(watermarkKey)).toBe(String(longerWatermarkMs)); + expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(ttlBefore - 1_000); + expect(await admin.pTTL(watermarkKey)).toBeLessThanOrEqual(ttlBefore); + }); + + it("rejects unsafe coordinated events before mutating their watermarks", async () => { + if (client === undefined || admin === undefined) { + throw new Error("Redis test clients did not start"); + } + const namespace = `coordinated-safety-${kind}`; + const channel = redisInvalidationChannel(namespace); + const oversizedWatermarkKey = `coordinated-safety:{${kind}:oversized}:watermark`; + await expect( + client.adapter.invalidateAndPublish({ + watermarkKey: oversizedWatermarkKey, + futureBufferMs: 0, + channel, + namespace, + keyType: "user_id", + id: "x".repeat(MAX_REDIS_INVALIDATION_EVENT_BYTES + 1_024), + }), + ).rejects.toThrow(/invalidation event is too large/i); + expect(await admin.exists(oversizedWatermarkKey)).toBe(0); + + const futureWatermarkKey = `coordinated-safety:{${kind}:future}:watermark`; + const redisNowMs = (await admin.time()).getTime(); + const unsupportedFutureWatermarkMs = + redisNowMs + MAX_SUPPORTED_DURATION_MS + 60_000; + await admin.set(futureWatermarkKey, String(unsupportedFutureWatermarkMs), { + PX: 120_000, + }); + const ttlBefore = await admin.pTTL(futureWatermarkKey); + + await expect( + client.adapter.invalidateAndPublish({ + watermarkKey: futureWatermarkKey, + futureBufferMs: 0, + channel, + namespace, + keyType: "user_id", + id: "future", + }), + ).rejects.toThrow(/invalid DialCache watermark/i); + expect(await admin.get(futureWatermarkKey)).toBe( + String(unsupportedFutureWatermarkMs), + ); + expect(await admin.pTTL(futureWatermarkKey)).toBeGreaterThan( + ttlBefore - 1_000, + ); + expect(await admin.pTTL(futureWatermarkKey)).toBeLessThanOrEqual(ttlBefore); }); it("treats every invalid read frame and watermark state as a miss", async () => { @@ -836,4 +1024,247 @@ describe.each(engines)("DialCache Lua protocol on $name", ({ image }) => { }), ).resolves.toBe("tracked"); }); + + it("delivers coordinated publications from both adapters to a node-redis peer", async () => { + if (admin === undefined || harnesses === undefined || redisUrl === undefined) { + throw new Error("Redis test clients did not start"); + } + await admin.flushAll(); + const namespace = "coordinated-peer"; + const subscriber = createClient({ url: redisUrl }); + subscriber.on("error", () => undefined); + await subscriber.connect(); + const coordinator = await createNodeRedisDialCacheInvalidationCoordinator(subscriber, { + namespace, + }); + const peerCache = new DialCache({ + namespace, + localMaxSize: 100, + redis: { + client: harnesses.nodeRedis.adapter, + coordinator, + readTimeoutMs: 10_000, + }, + }); + const versions = new Map(); + const calls = new Map(); + const getUser = peerCache.cached(async (id: string) => { + const version = versions.get(id); + if (version === undefined) { + throw new Error(`Missing test version for ${id}`); + } + const nextCalls = (calls.get(id) ?? 0) + 1; + calls.set(id, nextCalls); + return { id, version, calls: nextCalls }; + }, { + keyType: "user_id", + useCase: "CoordinatedPeerLocal", + cacheKey: (id) => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + try { + expect(coordinator.state).toBe("ready"); + for (const { kind } of adapterKinds) { + const id = `${kind}-publisher`; + versions.set(id, 1); + expect(await peerCache.enable(async () => await getUser(id))).toEqual({ + id, + version: 1, + calls: 1, + }); + expect(await peerCache.enable(async () => await getUser(id))).toEqual({ + id, + version: 1, + calls: 1, + }); + versions.set(id, 2); + const delivery = nextInvalidation(coordinator, id); + try { + const event = await harnesses[kind].adapter.invalidateAndPublish({ + watermarkKey: `coordinated-peer:{${namespace}:user_id:${id}}:watermark`, + futureBufferMs: 5_000, + channel: coordinator.channel, + namespace, + keyType: "user_id", + id, + }); + const received = await delivery.promise; + + expect(received).toEqual({ + namespace, + keyType: "user_id", + id, + remainingMs: Number(event.effectiveWatermarkMs) - Number(event.redisNowMs), + source: "event", + }); + expect(await peerCache.enable(async () => await getUser(id))).toEqual({ + id, + version: 2, + calls: 2, + }); + versions.set(id, 3); + expect(await peerCache.enable(async () => await getUser(id))).toEqual({ + id, + version: 3, + calls: 3, + }); + } finally { + delivery.cancel(); + } + } + } finally { + peerCache.dispose(); + await coordinator.dispose(); + await subscriber.quit(); + } + }); + + it("retains the watermark and origin local fence when ACLs reject PUBLISH", async () => { + if (admin === undefined || harnesses === undefined || redisUrl === undefined) { + throw new Error("Redis test clients did not start"); + } + await admin.flushAll(); + const username = "dialcache-acl-publisher"; + const password = "dialcache-acl-password"; + await admin.sendCommand([ + "ACL", + "SETUSER", + username, + "reset", + "on", + `>${password}`, + "~*", + "&*", + "+@all", + "-publish", + ]); + + const restrictedUrl = new URL(redisUrl); + restrictedUrl.username = username; + restrictedUrl.password = password; + const publisher = createTestClient(restrictedUrl.toString()); + publisher.on("error", () => undefined); + const originSubscriber = createClient({ url: redisUrl }); + originSubscriber.on("error", () => undefined); + const peerSubscriber = createClient({ url: redisUrl }); + peerSubscriber.on("error", () => undefined); + let originCoordinator: + | Awaited> + | undefined; + let peerCoordinator: + | Awaited> + | undefined; + let originCache: DialCache | undefined; + let removePeerListener = (): void => undefined; + + try { + await Promise.all([ + publisher.connect(), + originSubscriber.connect(), + peerSubscriber.connect(), + ]); + const namespace = "coordinated-acl"; + [originCoordinator, peerCoordinator] = await Promise.all([ + createNodeRedisDialCacheInvalidationCoordinator(originSubscriber, { namespace }), + createNodeRedisDialCacheInvalidationCoordinator(peerSubscriber, { namespace }), + ]); + const peerEvents: Array = []; + removePeerListener = peerCoordinator.addListener({ + onInvalidation(invalidation) { + if (invalidation.source === "event") { + peerEvents.push(invalidation); + } + }, + onStateChange: () => undefined, + }); + originCache = new DialCache({ + namespace, + localMaxSize: 100, + logger: { + debug: () => undefined, + warn: () => undefined, + error: () => undefined, + }, + redis: { + client: createNodeRedisDialCacheClient(publisher), + coordinator: originCoordinator, + readTimeoutMs: 10_000, + }, + }); + const id = "denied-publish"; + const watermarkKey = `{${namespace}:user_id:${id}}#watermark`; + let version = 1; + let calls = 0; + const getUser = originCache.cached(async () => ({ + id, + version, + calls: ++calls, + }), { + keyType: "user_id", + useCase: "AclDeniedPublication", + cacheKey: () => id, + trackForInvalidation: true, + defaultConfig: localOnly, + }); + + expect(originCoordinator.state).toBe("ready"); + expect(peerCoordinator.state).toBe("ready"); + expect(await originCache.enable(getUser)).toEqual({ id, version: 1, calls: 1 }); + expect(await originCache.enable(getUser)).toEqual({ id, version: 1, calls: 1 }); + + version = 2; + const redisNowBeforeMs = (await admin.time()).getTime(); + await expect( + originCache.invalidateRemote("user_id", id, 5_000), + ).rejects.toThrow(/ACL failure|can't run this command|NOPERM|permission/i); + + const effectiveWatermarkMs = Number(await admin.get(watermarkKey)); + expect(Number.isSafeInteger(effectiveWatermarkMs)).toBe(true); + expect(effectiveWatermarkMs).toBeGreaterThanOrEqual(redisNowBeforeMs + 5_000); + expect(await admin.pTTL(watermarkKey)).toBeGreaterThan(60_000); + + // A later valid publication is a deterministic Pub/Sub delivery barrier: + // if the denied script had emitted the target event, the peer would have + // processed it before this sentinel from the same Redis server. + const sentinelId = "delivery-barrier"; + const sentinel = nextInvalidation(peerCoordinator, sentinelId); + try { + await harnesses.nodeRedis.adapter.invalidateAndPublish({ + watermarkKey: `{${namespace}:user_id:${sentinelId}}#watermark`, + futureBufferMs: 0, + channel: peerCoordinator.channel, + namespace, + keyType: "user_id", + id: sentinelId, + }); + await sentinel.promise; + } finally { + sentinel.cancel(); + } + expect( + peerEvents.filter((event) => event.id === id), + ).toEqual([]); + + // The provisional local invalidation is intentionally not rolled back + // when the Redis script fails after advancing the durable watermark. + expect(await originCache.enable(getUser)).toEqual({ id, version: 2, calls: 2 }); + version = 3; + expect(await originCache.enable(getUser)).toEqual({ id, version: 3, calls: 3 }); + } finally { + originCache?.dispose(); + removePeerListener(); + await Promise.allSettled([ + originCoordinator?.dispose(), + peerCoordinator?.dispose(), + ]); + await Promise.allSettled([ + originSubscriber.isOpen ? originSubscriber.quit() : Promise.resolve(), + peerSubscriber.isOpen ? peerSubscriber.quit() : Promise.resolve(), + publisher.isOpen ? publisher.quit() : Promise.resolve(), + ]); + await admin.sendCommand(["ACL", "DELUSER", username]); + } + }); }); diff --git a/test/valkey-glide.test.ts b/test/valkey-glide.test.ts index cb7d405..aa379b5 100644 --- a/test/valkey-glide.test.ts +++ b/test/valkey-glide.test.ts @@ -5,6 +5,7 @@ import { DialCacheRedisPayloadError, DialCacheRedisProtocolError, } from "../src/redis-client.js"; +import { INVALIDATE_AND_PUBLISH_CACHE_SCRIPT } from "../src/redis-protocol.js"; import { createValkeyGlideDialCacheClient } from "../src/valkey-glide.js"; const INVALID_WRITE_REPLIES: readonly unknown[] = [ @@ -267,4 +268,128 @@ describe("Valkey GLIDE adapter", () => { expect(options?.decoder).not.toBe(otherGlide.Decoder.Bytes); adapter.dispose(); }); + + it("lazily registers and invokes coordinated invalidation", async () => { + const event = { + version: 1, + namespace: "users", + keyType: "user_id", + id: "123", + effectiveWatermarkMs: "1785300001000", + redisNowMs: "1785300000000", + } as const; + const client = fakeClient(Buffer.from(JSON.stringify(event))); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + expect(scriptInstances).toHaveLength(5); + + await expect(adapter.invalidateAndPublish({ + watermarkKey: "{users:user_id:123}#watermark", + futureBufferMs: 1_000, + channel: "dialcache:invalidation:v1:users", + namespace: "users", + keyType: "user_id", + id: "123", + })).resolves.toEqual(event); + + expect(scriptInstances).toHaveLength(6); + expect(scriptInstances[5]?.code).toBe(INVALIDATE_AND_PUBLISH_CACHE_SCRIPT); + expect(client.invokeScript).toHaveBeenCalledWith( + scriptInstances[5], + { + keys: ["{users:user_id:123}#watermark"], + args: [ + "1000", + "dialcache:invalidation:v1:users", + "users", + "user_id", + "123", + ], + decoder: decoderBytes, + }, + ); + + adapter.dispose(); + expect(scriptInstances[5]?.release).toHaveBeenCalledTimes(1); + }); + + it("rejects malformed or mismatched coordinated invalidation replies", async () => { + const wrongIdentity = Buffer.from(JSON.stringify({ + version: 1, + namespace: "users", + keyType: "user_id", + id: "other", + effectiveWatermarkMs: "1000", + redisNowMs: "1000", + })); + const client = fakeClient("not-bytes", wrongIdentity); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + const request = { + watermarkKey: "{users:user_id:123}#watermark", + futureBufferMs: 0, + channel: "dialcache:invalidation:v1:users", + namespace: "users", + keyType: "user_id", + id: "123", + } as const; + + await expect(adapter.invalidateAndPublish(request)).rejects.toBeInstanceOf( + DialCacheRedisPayloadError, + ); + await expect(adapter.invalidateAndPublish(request)).rejects.toBeInstanceOf( + DialCacheRedisProtocolError, + ); + adapter.dispose(); + }); + + it("does not release a lazy coordinated script while its invocation is active", async () => { + let resolveInvalidation: ((value: Buffer) => void) | undefined; + const client = fakeClient(); + client.invokeScript.mockImplementationOnce( + async () => await new Promise((resolve) => { + resolveInvalidation = resolve; + }), + ); + const adapter = createValkeyGlideDialCacheClient(client, mockGlide); + const operation = adapter.invalidateAndPublish({ + watermarkKey: "{users:user_id:123}#watermark", + futureBufferMs: 0, + channel: "dialcache:invalidation:v1:users", + namespace: "users", + keyType: "user_id", + id: "123", + }); + + expect(() => adapter.dispose()).toThrow( + "Cannot dispose Valkey GLIDE DialCache client while operations are in flight", + ); + expect(scriptInstances).toHaveLength(6); + expect(scriptInstances[5]?.release).not.toHaveBeenCalled(); + + resolveInvalidation?.(Buffer.from(JSON.stringify({ + version: 1, + namespace: "users", + keyType: "user_id", + id: "123", + effectiveWatermarkMs: "1000", + redisNowMs: "1000", + }))); + await operation; + adapter.dispose(); + expect(scriptInstances[5]?.release).toHaveBeenCalledTimes(1); + }); + + it("does not create a coordinated script after disposal", async () => { + const adapter = createValkeyGlideDialCacheClient(fakeClient(), mockGlide); + adapter.dispose(); + + await expect(adapter.invalidateAndPublish({ + watermarkKey: "{users:user_id:123}#watermark", + futureBufferMs: 0, + channel: "dialcache:invalidation:v1:users", + namespace: "users", + keyType: "user_id", + id: "123", + })).rejects.toThrow("Valkey GLIDE DialCache client is disposed"); + expect(scriptInstances).toHaveLength(5); + }); });