From f6b26ae8f0e85889c1f513e263cf44f912a5bab3 Mon Sep 17 00:00:00 2001 From: Ariel Weisberg Date: Wed, 20 May 2026 14:59:12 -0400 Subject: [PATCH 1/7] CEP-45: Fix swapped args in ShortMutationId(MutationId) constructor The private ShortMutationId(int hostId, int hostLogId, int offset) ctor was being called with (hostLogId, hostId), transposing the two fields. Currently latent - no caller passes a MutationId - but the constructor is public. Cherry-picked from aci-cassandra 09b9b05fd0. --- src/java/org/apache/cassandra/replication/ShortMutationId.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/replication/ShortMutationId.java b/src/java/org/apache/cassandra/replication/ShortMutationId.java index 856790a8a1df..c35ad84978b4 100644 --- a/src/java/org/apache/cassandra/replication/ShortMutationId.java +++ b/src/java/org/apache/cassandra/replication/ShortMutationId.java @@ -76,7 +76,7 @@ private ShortMutationId(int hostId, int hostLogId, int offset) public ShortMutationId(MutationId mutationId) { - this(mutationId.hostLogId(), mutationId.hostId(), mutationId.offset()); + this(mutationId.hostId(), mutationId.hostLogId(), mutationId.offset()); } public int hostId() From 83dcb6572436f7a285ea4251cb9477abd2bf9e3c Mon Sep 17 00:00:00 2001 From: Ariel Weisberg Date: Wed, 20 May 2026 16:40:11 -0400 Subject: [PATCH 2/7] CEP-45: Isolate mutation journal state between unit tests cleanup() now wipes the mutation tracking journal directory alongside the Accord journal, and MutationJournal.start() moves after cleanupAndLeaveDirs() so the journal is not started against directories that are about to be deleted. Cherry-picked from aci-cassandra 09b9b05fd0 and 6c206f15de. --- test/unit/org/apache/cassandra/ServerTestUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/org/apache/cassandra/ServerTestUtils.java b/test/unit/org/apache/cassandra/ServerTestUtils.java index 448ac0468e77..5d1d8e8fef68 100644 --- a/test/unit/org/apache/cassandra/ServerTestUtils.java +++ b/test/unit/org/apache/cassandra/ServerTestUtils.java @@ -155,9 +155,6 @@ public static void prepareServerNoRegister() { daemonInitialization(); - // Need to happen after daemonInitialization for config to be set, but before CFS initialization - MutationJournal.start(); - if (isServerPrepared) return; @@ -196,6 +193,8 @@ public void uncaughtException(Thread t, Throwable e) ThreadAwareSecurityManager.install(); CassandraRelevantProperties.GOSSIPER_SKIP_WAITING_TO_SETTLE.setInt(0); + // Need to happen after daemonInitialization for config to be set, but before CFS initialization + MutationJournal.start(); initCMS(); SystemKeyspace.persistLocalMetadata(); AuditLogManager.instance.initialize(); @@ -230,6 +229,7 @@ public static void cleanup() cleanupDirectory(cdcDir); cleanupDirectory(DatabaseDescriptor.getHintsDirectory()); cleanupDirectory(DatabaseDescriptor.getAccordJournalDirectory()); + cleanupDirectory(DatabaseDescriptor.getMutationTrackingJournalDirectory()); cleanupSavedCaches(); // clean up data directory which are stored as data directory/keyspace/data files From 9f8f35909df3bd92194663fabcc3dfb1a6be050e Mon Sep 17 00:00:00 2001 From: Ariel Weisberg Date: Thu, 21 May 2026 15:43:13 -0400 Subject: [PATCH 3/7] CEP-45: Add missing nodetool mtadmin help test fixtures NodetoolHelpCommandsOutputTest had 8 failures without these. Cherry-picked from aci-cassandra e4d1296037. --- test/resources/nodetool/help/mtadmin | 43 +++++++++++++++++++ .../resources/nodetool/help/mtadmin$getconfig | 24 +++++++++++ .../resources/nodetool/help/mtadmin$setconfig | 36 ++++++++++++++++ test/resources/nodetool/help/nodetool | 1 + 4 files changed, 104 insertions(+) create mode 100644 test/resources/nodetool/help/mtadmin create mode 100644 test/resources/nodetool/help/mtadmin$getconfig create mode 100644 test/resources/nodetool/help/mtadmin$setconfig diff --git a/test/resources/nodetool/help/mtadmin b/test/resources/nodetool/help/mtadmin new file mode 100644 index 000000000000..0d2b141ea7e6 --- /dev/null +++ b/test/resources/nodetool/help/mtadmin @@ -0,0 +1,43 @@ +NAME + nodetool mtadmin - Manage mutation tracking + +SYNOPSIS + nodetool [(-h | --host )] [(-p | --port )] + [(-pw | --password )] + [(-pwf | --password-file )] + [(-u | --username )] mtadmin [] + + nodetool [(-h | --host )] [(-p | --port )] + [(-pw | --password )] + [(-pwf | --password-file )] + [(-u | --username )] mtadmin getconfig + + nodetool [(-h | --host )] [(-p | --port )] + [(-pw | --password )] + [(-pwf | --password-file )] + [(-u | --username )] mtadmin setconfig [--] + + +OPTIONS + -h , --host + Node hostname or ip address + + -p , --port + Remote jmx agent port number + + -pw , --password + Remote jmx agent password + + -pwf , --password-file + Path to the JMX password file + + -u , --username + Remote jmx agent username + +COMMANDS + With no arguments, Display help information + + getconfig + Print mutation tracking configurations + setconfig + Sets the mutation tracking configuration diff --git a/test/resources/nodetool/help/mtadmin$getconfig b/test/resources/nodetool/help/mtadmin$getconfig new file mode 100644 index 000000000000..46d743bfba52 --- /dev/null +++ b/test/resources/nodetool/help/mtadmin$getconfig @@ -0,0 +1,24 @@ +NAME + nodetool mtadmin getconfig - Print mutation tracking configurations + +SYNOPSIS + nodetool [(-h | --host )] [(-p | --port )] + [(-pw | --password )] + [(-pwf | --password-file )] + [(-u | --username )] mtadmin getconfig + +OPTIONS + -h , --host + Node hostname or ip address + + -p , --port + Remote jmx agent port number + + -pw , --password + Remote jmx agent password + + -pwf , --password-file + Path to the JMX password file + + -u , --username + Remote jmx agent username diff --git a/test/resources/nodetool/help/mtadmin$setconfig b/test/resources/nodetool/help/mtadmin$setconfig new file mode 100644 index 000000000000..ea30d9c77941 --- /dev/null +++ b/test/resources/nodetool/help/mtadmin$setconfig @@ -0,0 +1,36 @@ +NAME + nodetool mtadmin setconfig - Sets the mutation tracking configuration + +SYNOPSIS + nodetool [(-h | --host )] [(-p | --port )] + [(-pw | --password )] + [(-pwf | --password-file )] + [(-u | --username )] mtadmin setconfig [--] + + +OPTIONS + -h , --host + Node hostname or ip address + + -p , --port + Remote jmx agent port number + + -pw , --password + Remote jmx agent password + + -pwf , --password-file + Path to the JMX password file + + -u , --username + Remote jmx agent username + + -- + This option can be used to separate command-line options from the + list of argument, (useful when arguments might be mistaken for + command-line options + + + Mutation tracking param type. + + + Mutation tracking param value diff --git a/test/resources/nodetool/help/nodetool b/test/resources/nodetool/help/nodetool index a3fe2c520e28..023cc4f82fe4 100644 --- a/test/resources/nodetool/help/nodetool +++ b/test/resources/nodetool/help/nodetool @@ -95,6 +95,7 @@ The most commonly used nodetool commands are: listpendinghints Print all pending hints that this node has listsnapshots Lists all the snapshots along with the size on disk and true size. True size is the total size of all SSTables which are not backed up to disk. Size on disk is total size of the snapshot on disk. Total TrueDiskSpaceUsed does not make any SSTable deduplication. move Move node on the token ring to a new token + mtadmin Manage mutation tracking netstats Print network information on provided host (connecting node by default) pausehandoff Pause hints delivery process profile Manage Async-Profiler on a Cassandra process From 3c37ba7b42900a1dfd8897996e05630e6e3c0aa5 Mon Sep 17 00:00:00 2001 From: Blake Eggleston Date: Tue, 2 Jun 2026 14:23:23 -0700 Subject: [PATCH 4/7] CEP-45: Fix MutationTrackingRepairTest.testRepairFailsOnTopologyChange The test wrote consistent data before installing the offset broadcast filter, which let the write path mark the writes reconciled and the sync coordinator complete before the topology change was observed. Write inconsistent data after the filter is in place instead. Cherry-picked from aci-cassandra 274dc829ea. The @Ignore(NO_TOPOLOGY_CHANGES) it added was removed again by 55af9fd957 once the underlying issue was fixed, so only the ignore reason constant is carried over here. --- .../distributed/test/repair/MutationTrackingRepairTest.java | 5 +++-- .../distributed/test/tracking/MutationTrackingUtils.java | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/repair/MutationTrackingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/repair/MutationTrackingRepairTest.java index f5ad9f7d26ff..87b1eb0457be 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/repair/MutationTrackingRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/repair/MutationTrackingRepairTest.java @@ -881,11 +881,12 @@ public void testRepairSyncTimeout() @Test public void testRepairFailsOnTopologyChange() throws Exception { - insertData("tbl", 0, 50); - // Block offset broadcasts so the sync coordinator stays alive waiting IMessageFilters.Filter offsetFilter = CLUSTER.filters().verbs(Verb.MT_BROADCAST_LOG_OFFSETS.id).drop(); + // must use inconsistent data to prevent write process from marking the writes reconciled + insertDataWithInconsistency("tbl", 0, 50); + // Use a latch to detect when the sync request has been sent, meaning // the sync coordinator is active and tracking shard references CountDownLatch syncStarted = new CountDownLatch(1); diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java index f3d6adc1215c..b9c89b7778d1 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java @@ -60,6 +60,7 @@ public class MutationTrackingUtils public static class IgnoreReasons { + public static final String NO_TOPOLOGY_CHANGES = "NO_TOPOLOGY_CHANGES"; public static final String NO_RANGE_MOVEMENTS = "NO_RANGE_MOVEMENTS"; public static final String NO_PER_PARTITION_RANGE_READ_LIMITS = "NO_PER_PARTITION_RANGE_READ_LIMITS"; } From 84e04ced5645cd2b0c25f48d17c0959872d86974 Mon Sep 17 00:00:00 2001 From: Blake Eggleston Date: Tue, 2 Jun 2026 14:37:01 -0700 Subject: [PATCH 5/7] CEP-45: Fix MutationTrackingSyncCoordinatorTest cancellation test Drop MT_SYNC_RSP so the coordinator never receives sync responses, rather than pausing offset broadcasts, which no longer keeps the coordinator waiting. Cherry-picked from aci-cassandra eff3d6de2b. --- .../MutationTrackingSyncCoordinatorTest.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/replication/MutationTrackingSyncCoordinatorTest.java b/test/distributed/org/apache/cassandra/distributed/test/replication/MutationTrackingSyncCoordinatorTest.java index 7a13f606f2b1..c1d82ef104fc 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/replication/MutationTrackingSyncCoordinatorTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/replication/MutationTrackingSyncCoordinatorTest.java @@ -29,6 +29,7 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.replication.MutationTrackingService; @@ -64,12 +65,6 @@ private String tableName(String suffix) return KS_NAME + suffix + '.' + TBL_NAME; } - private void pauseOffsetBroadcasts(Cluster cluster, boolean pause) - { - for (int i = 1; i <= cluster.size(); i++) - cluster.get(i).runOnInstance(() -> MutationTrackingService.instance().pauseOffsetBroadcast(pause)); - } - private static Range fullTokenRange() { return new Range<>( @@ -233,9 +228,6 @@ public void testSyncCoordinatorCancel() throws Throwable { createTrackedKeyspace(cluster, "4"); - // Pause offset broadcasts on all nodes to prevent sync from completing - pauseOffsetBroadcasts(cluster, true); - for (int i = 0; i < 100; i++) { cluster.coordinator(1).execute( @@ -243,7 +235,10 @@ public void testSyncCoordinatorCancel() throws Throwable ConsistencyLevel.ONE, i, i); } - // Start coordinator - it will be stuck waiting for offsets + // Drop sync responses so the coordinator never receives them + cluster.filters().verbs(Verb.MT_SYNC_RSP.id).drop(); + + // Start coordinator - it will be stuck waiting for sync responses Boolean wasCancelled = cluster.get(1).callOnInstance(() -> { Range range = fullTokenRange(); RepairJobDesc desc = new RepairJobDesc(TimeUUID.Generator.nextTimeUUID(), From 0174c86b010796a2a4b6959bfaaec1615479c325 Mon Sep 17 00:00:00 2001 From: Blake Eggleston Date: Tue, 2 Jun 2026 15:38:28 -0700 Subject: [PATCH 6/7] CEP-45: Tolerate dropped keyspaces in getOrCreateShards A keyspace can be dropped locally while offset broadcasts referencing it are still in flight from peers. getOrCreateShards now returns null in that case instead of failing on missing keyspace metadata, and updateReplicatedOffsets ignores the broadcast. Cherry-picked from aci-cassandra 55af9fd957, with the null check expressed as an if/else rather than an early return, so the sync coordinator notification that follows the lock scope on this branch still runs. --- .../replication/MutationTrackingService.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/java/org/apache/cassandra/replication/MutationTrackingService.java b/src/java/org/apache/cassandra/replication/MutationTrackingService.java index db87a88f010a..010fe31dcb22 100644 --- a/src/java/org/apache/cassandra/replication/MutationTrackingService.java +++ b/src/java/org/apache/cassandra/replication/MutationTrackingService.java @@ -462,7 +462,12 @@ public void updateReplicatedOffsets(String keyspace, Range range, List KeyspaceShards.make(ksm, csm, this::nextLogId, this::onNewLog)); } From 65d9ec813ee0777bf787da31a5b57c9ec2b75dbc Mon Sep 17 00:00:00 2001 From: Blake Eggleston Date: Wed, 3 Jun 2026 10:45:36 -0700 Subject: [PATCH 7/7] ninja - minor mutation tracking test fixes IndexStreamingTest: include isWide in the parameterized test name. TrackedImportFailureTest: disable the background reconciler in importMissedActivation. Cherry-picked from aci-cassandra bde1f1daf7 and c3789023fa. --- .../cassandra/distributed/test/sai/IndexStreamingTest.java | 2 +- .../distributed/test/tracking/TrackedImportFailureTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java index db3d841bddf2..ebec55383b06 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexStreamingTest.java @@ -69,7 +69,7 @@ private static int sstableStreamingComponentsCount() @Parameterized.Parameter(2) public boolean isWide; - @Parameterized.Parameters(name = "isLiteral={0}, isZeroCopyStreaming={1}") + @Parameterized.Parameters(name = "isLiteral={0}, isZeroCopyStreaming={1}, isWide={2}") public static List data() { List result = new ArrayList<>(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java index ee897a31886a..4c1f29ff3c27 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java @@ -79,7 +79,7 @@ public void importMissedActivationCommit() throws Throwable public void importMissedActivation(ActivationRequest.Phase phase) throws Throwable { int MISSED_ACTIVATION = 2; - try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION))) + try (Cluster cluster = disableBackgroundReconciler(cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))) { TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase); createSchema(cluster);