diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 42630eeae2e3..30391ee7a029 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -1062,6 +1062,52 @@ sstable_preemptive_open_interval: 50MiB # and eventually get removed from the configuration. uuid_sstable_identifiers_enabled: false +# Anticompact an sstable by splitting it - copying its compression chunks verbatim - instead of rewriting every +# row, when its full/transient/unrepaired partitions form contiguous token runs. Anything else, notably the +# interleaved ranges vnodes produce, falls back to the normal rewrite. The copy is bounded by +# compaction_throughput, appears in nodetool compactionstats as an ANTICOMPACTION, and is stopped by nodetool +# stop ANTICOMPACTION, TRUNCATE and DROP. +# +# NOTE: a verbatim copy cannot purge tombstones, so anticompaction stops dropping droppable tombstones and +# shadowed data for the sstables it handles. That is retention only, never data loss, but disk usage after +# anticompaction can be higher until the children are compacted normally. +# +# NOTE: the children's sstable statistics are inherited rather than recomputed, since recomputing them means +# deserializing every row - the cost this path avoids. Each child inherits the parent's whole-sstable cell +# count, row count and tombstone-drop histogram, while its partition count and partition size histogram are +# exact. Expect per-table aggregates to over-report by roughly the number of children, and single-sstable +# tombstone compaction to fire less readily than tombstone_threshold suggests. All conservative in direction. +# +# NOTE: children written from the middle or end of the parent carry a dead prefix at the head of their Data.db. +# Every read path tolerates it, but a node running an older build will fail nodetool verify on such an sstable. +# +# Tables with a secondary index are refused and anticompact the old way, so no configuration is needed. Not +# supported on JBOD: children are always written into the parent sstable's own directory with no free-space +# check, so they cannot move to a disk that has room and splitting a parent larger than the free space on its +# own disk will fill that disk. +# zero_copy_anticompaction_enabled: false + +# Let the zero-copy splitter share each child's Data.db extents with the parent instead of copying them, using +# the Linux FICLONERANGE ioctl ("reflink"). Where it works a split writes no data blocks and needs no additional +# disk space, since the parent's extents become the children's when the parent is unlinked. Requires xfs +# formatted with `-m reflink=1` or btrfs; elsewhere the first attempt per data directory fails, is logged once, +# and every split from then on copies exactly as before. Both paths produce identical children. +# +# NOTE: sharing costs up to 64 KiB of alignment padding at the head of each child's Data.db, so children smaller +# than 1 MiB are copied regardless. Shared extents are counted once per file by `du` but once in total by `df`, +# so per-table disk usage over-reports until the parent is unlinked. +# zero_copy_split_reflink_enabled: true + +# Write Digest.crc32 for the children of a zero-copy split. Producing it is one full sequential read of every +# child, which once the extents above are shared is the entire remaining cost of a split. Nothing requires the +# component, and a compressed sstable is self-checking without it because every chunk carries an inline CRC32 +# that this path preserves and every read verifies. +# +# NOTE: what it costs is verification speed, not strength. `nodetool verify` and `nodetool import +# --verify-sstables` treat a missing digest as a reason to run a full extended verification instead of a +# whole-file CRC. `nodetool verify -q` never looks at it. +# zero_copy_split_digest_enabled: true + # When enabled, permits Cassandra to zero-copy stream entire eligible # SSTables between nodes, including every component. # This speeds up the network transfer significantly subject to diff --git a/doc/modules/cassandra/pages/operating/metrics.adoc b/doc/modules/cassandra/pages/operating/metrics.adoc index 11a204f740d9..de73ccf765bd 100644 --- a/doc/modules/cassandra/pages/operating/metrics.adoc +++ b/doc/modules/cassandra/pages/operating/metrics.adoc @@ -258,6 +258,11 @@ read during validation. anticompacting because the sstable was fully contained in the repaired range. +|BytesZeroCopyAnticompaction |Counter |How many Data.db bytes we copied +verbatim during anticompaction instead of rewriting them, because the +sstable's full/transient/unrepaired partitions formed contiguous token +runs. A subset of BytesAnticompacted. + |MutatedAnticompactionGauge |Gauge |Ratio of bytes mutated vs total bytes repaired. |=== diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 21ca1b595cd1..b05eced878c2 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -438,6 +438,34 @@ public MemtableOptions() @Replaces(oldName = "sstable_preemptive_open_interval_in_mb", converter = Converters.NEGATIVE_MEBIBYTES_DATA_STORAGE_INT, deprecated = true) public volatile DataStorageSpec.IntMebibytesBound sstable_preemptive_open_interval = new DataStorageSpec.IntMebibytesBound("50MiB"); + /** + * Anticompact by splitting an sstable with {@code ZeroCopySSTableSplitter} -- copying compression chunks + * verbatim -- when its full / transient / unrepaired partitions form contiguous token runs. Interleaved + * ranges, which is what vnodes produce, fall back to the normal rewrite. + *

+ * A verbatim copy cannot purge tombstones, so anticompaction stops doing so for the sstables it handles + * (retention only, never data loss), and the children's per-sstable statistics are inherited rather than + * recomputed. Refused outright for tables with a secondary index; unsupported but NOT refused on JBOD, since + * children are always written into the parent's directory with no free-space check. + */ + public volatile boolean zero_copy_anticompaction_enabled = false; + + /** + * Let the zero-copy splitter share a child's Data.db extents with its parent via {@code FICLONERANGE} rather + * than copying them, so a split writes no data blocks and uses no extra disk space. Needs xfs with + * {@code -m reflink=1} or btrfs, discovered by trying: elsewhere the first attempt per directory fails, is + * logged once, and every split from then on copies as before. Both paths produce identical children. + */ + public volatile boolean zero_copy_split_reflink_enabled = true; + + /** + * Write Digest.crc32 for the children of a zero-copy split. Producing it is one full read of every child, + * which with the extents shared is the whole remaining cost of a split. Nothing requires the component and a + * compressed sstable is self-checking without it, but {@code Verifier} answers its absence with a full + * extended verification, so {@code nodetool verify} gets slower for those children. + */ + public volatile boolean zero_copy_split_digest_enabled = true; + public volatile boolean key_cache_migrate_during_compaction = true; public volatile int key_cache_keys_to_save = Integer.MAX_VALUE; @Replaces(oldName = "key_cache_size_in_mb", converter = Converters.MEBIBYTES_DATA_STORAGE_LONG, deprecated = true) diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 82ef40f6ee09..ee70f78d2062 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -3405,6 +3405,48 @@ public static void setSSTablePreemptiveOpenIntervalInMiB(int mib) conf.sstable_preemptive_open_interval = new DataStorageSpec.IntMebibytesBound(mib); } + /** @see Config#zero_copy_anticompaction_enabled */ + public static boolean getZeroCopyAnticompactionEnabled() + { + return conf.zero_copy_anticompaction_enabled; + } + + public static void setZeroCopyAnticompactionEnabled(boolean enabled) + { + if (conf.zero_copy_anticompaction_enabled != enabled) + logger.info("Changing zero_copy_anticompaction_enabled to {}", enabled); + conf.zero_copy_anticompaction_enabled = enabled; + } + + /** + * @see Config#zero_copy_split_reflink_enabled -- filesystem support is discovered by trying, so true here does + * not mean any extent will actually be shared. + */ + public static boolean getZeroCopySplitReflinkEnabled() + { + return conf.zero_copy_split_reflink_enabled; + } + + public static void setZeroCopySplitReflinkEnabled(boolean enabled) + { + if (conf.zero_copy_split_reflink_enabled != enabled) + logger.info("Changing zero_copy_split_reflink_enabled to {}", enabled); + conf.zero_copy_split_reflink_enabled = enabled; + } + + /** @see Config#zero_copy_split_digest_enabled */ + public static boolean getZeroCopySplitDigestEnabled() + { + return conf.zero_copy_split_digest_enabled; + } + + public static void setZeroCopySplitDigestEnabled(boolean enabled) + { + if (conf.zero_copy_split_digest_enabled != enabled) + logger.info("Changing zero_copy_split_digest_enabled to {}", enabled); + conf.zero_copy_split_digest_enabled = enabled; + } + public static boolean getTrickleFsync() { return conf.trickle_fsync; diff --git a/src/java/org/apache/cassandra/db/compaction/AntiCompactionRunPlanner.java b/src/java/org/apache/cassandra/db/compaction/AntiCompactionRunPlanner.java new file mode 100644 index 000000000000..b0ab2872a23a --- /dev/null +++ b/src/java/org/apache/cassandra/db/compaction/AntiCompactionRunPlanner.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.compaction; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Predicate; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.CorruptSSTableException; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.TimeUUID; + +/** + * Decides whether one sstable can be anticompacted by {@link ZeroCopySSTableSplitter} instead of by the + * three-writer rewrite in {@code CompactionManager.antiCompactGroup}, and if so produces the split boundaries + * and the per-child repair state. + * + *

Every partition is labelled {@link Label#FULL}, {@link Label#TRANSIENT} or {@link Label#UNREPAIRED} by token, + * using the same {@code Range.OrderedRangeContainmentChecker} pair and full-wins-over-transient precedence as + * {@code antiCompactGroup}, in a single sequential pass over Index.db -- Data.db is never touched. The labels are + * run-length encoded, and the sstable is eligible iff FULL and TRANSIENT each occupy at most one run, since every + * output child has to be a single contiguous key range. UNREPAIRED may appear as both a leading and a trailing run, + * so {@code UNREPAIRED, FULL, UNREPAIRED} is the common eligible shape; interleaved ranges, which is what vnodes + * produce, fall back to the rewrite. + * + *

Calling an sstable eligible means accepting that its children RETAIN the droppable tombstones and shadowed data + * a rewrite would have purged, since a verbatim chunk copy has no {@code CompactionController}. Retention, never + * loss. Deliberately not conditioned on the droppable-tombstone ratio. + * + *

The gate is per partition, so a FULL child's {@code [first, last]} may span a hole in the full ranges where no + * partition happens to live. Nothing rejects that, and it is what {@code fullWriter} already produces when it routes + * non-adjacent partitions into one output. + */ +public final class AntiCompactionRunPlanner +{ + /** + * The most runs worth remembering in detail. FULL and TRANSIENT are capped at one run each, so the largest + * eligible shape is {@code UNREPAIRED, FULL, UNREPAIRED, TRANSIENT, UNREPAIRED} = 5 runs. Past this cap the + * sstable is certainly ineligible, so the walk stops retaining boundary keys and just counts -- an alternating + * vnode layout would otherwise retain one key per partition. + */ + private static final int MAX_RETAINED_RUNS = 8; + + /** How one partition is classified, by token, for this repair session. */ + public enum Label + { + /** Inside a full replica range: becomes pending-repair, non-transient. */ + FULL, + /** Inside a transient replica range (and not a full one): becomes pending-repair, transient. */ + TRANSIENT, + /** Owned by neither: stays plain unrepaired. */ + UNREPAIRED + } + + /** The verdict, plus everything the split needs when the verdict is "eligible". */ + public static final class Plan + { + /** True iff the zero-copy split can produce exactly this anticompaction's output. */ + public final boolean eligible; + /** Human-readable, for logging; null when {@link #eligible}. */ + public final String ineligibleReason; + /** {@code runCount - 1} interior split points, in the form {@code ZeroCopySSTableSplitter} wants. */ + public final List boundaries; + /** One repair state per run, in order; {@code boundaries.size() + 1} entries. */ + public final List perChild; + /** Number of contiguous label runs found. Meaningful (and exact) even when ineligible. */ + public final int runCount; + + private Plan(boolean eligible, + String ineligibleReason, + List boundaries, + List perChild, + int runCount) + { + this.eligible = eligible; + this.ineligibleReason = ineligibleReason; + this.boundaries = boundaries; + this.perChild = perChild; + this.runCount = runCount; + } + + static Plan ineligible(String reason, int runCount) + { + return new Plan(false, reason, ImmutableList.of(), ImmutableList.of(), runCount); + } + + static Plan eligible(List boundaries, List perChild) + { + Preconditions.checkArgument(perChild.size() == boundaries.size() + 1, + "perChild must have one entry per range, got %s for %s boundaries", + perChild.size(), boundaries.size()); + return new Plan(true, null, ImmutableList.copyOf(boundaries), ImmutableList.copyOf(perChild), + perChild.size()); + } + + @Override + public String toString() + { + return eligible ? String.format("Plan[eligible runs=%d]", runCount) + : String.format("Plan[ineligible runs=%d: %s]", runCount, ineligibleReason); + } + } + + private AntiCompactionRunPlanner() + { + } + + /** + * Plan the zero-copy anticompaction of one sstable. Reads only the sstable's Index.db; never throws for an + * ineligible sstable, only for an unreadable one. + * + * @param sstable the parent, still live and marked compacting + * @param ranges the full and transient ranges of this repair session + * @param sessionID the repair session id stamped onto FULL and TRANSIENT children + * @throws CorruptSSTableException if the Index.db walk fails + */ + public static Plan plan(SSTableReader sstable, RangesAtEndpoint ranges, TimeUUID sessionID) + { + Preconditions.checkNotNull(sstable, "sstable"); + Preconditions.checkNotNull(ranges, "ranges"); + // a null session id would silently stamp FULL/TRANSIENT children as plain unrepaired + Preconditions.checkNotNull(sessionID, "sessionID"); + + if (!ZeroCopySSTableSplitter.isSupported(sstable)) + return Plan.ineligible("not a compressed BIG-format sstable (format=" + sstable.descriptor.formatType + + ", compressed=" + sstable.compression + ')', 0); + + return walk(sstable, ranges, sessionID); + } + + /** + * The pure form of {@link #plan(SSTableReader, RangesAtEndpoint, TimeUUID)}: everything after the Index.db walk, + * over an already-labelled partition sequence, so the run logic can be unit tested with no sstable at all. + * + * @param labels one label per partition, in on-disk (token) order + * @param keys the matching partition keys, same size as {@code labels}. Only the first key of each run is used, + * so a test that only cares about the verdict may pass any strictly increasing sequence + */ + @VisibleForTesting + static Plan planFromLabels(List

+ * {@code OrderedRangeContainmentChecker} is stateful and forward-only, so the two checkers must be distinct + * instances, fresh per sstable, and fed tokens in non-decreasing order -- which an Index.db walk satisfies by + * construction, on-disk order being token-major DecoratedKey order. The {@code isEmpty()} guards are mandatory: + * the constructor asserts the normalized range list is non-empty. Calling {@code transChecker} only when + * {@code fullChecker} said no is safe, since the cursor position for a token is a monotone function of that token + * alone. + */ + private static Plan walk(SSTableReader sstable, RangesAtEndpoint ranges, TimeUUID sessionID) + { + Predicate fullChecker = !ranges.onlyFull().isEmpty() + ? new Range.OrderedRangeContainmentChecker(ranges.onlyFull().ranges()) + : t -> false; + Predicate transChecker = !ranges.onlyTransient().isEmpty() + ? new Range.OrderedRangeContainmentChecker(ranges.onlyTransient().ranges()) + : t -> false; + + IPartitioner partitioner = sstable.getPartitioner(); + RunEncoding runs = new RunEncoding(); + Label previous = null; + + // Buffered rather than mmap'd, and opened straight off the descriptor so it starts at 0 with no index-summary + // lookup, as the splitter's own walk does. + try (RandomAccessReader in = RandomAccessReader.open(sstable.descriptor.fileFor(Component.PRIMARY_INDEX))) + { + long indexSize = in.length(); + while (in.getFilePointer() != indexSize) + { + ByteBuffer key = ByteBufferUtil.readWithShortLength(in); + RowIndexEntry.Serializer.skip(in, sstable.descriptor.version); // position + promoted index + + DecoratedKey dk = partitioner.decorateKey(key); + Token token = dk.getToken(); + // full wins over transient, exactly as antiCompactGroup routes partitions + Label label = fullChecker.test(token) ? Label.FULL + : transChecker.test(token) ? Label.TRANSIENT + : Label.UNREPAIRED; + if (label != previous) + { + runs.add(label, SSTable.getMinimalKey(dk)); + previous = label; + } + } + } + catch (IOException e) + { + throw new CorruptSSTableException(e, sstable.descriptor.filenameFor(Component.PRIMARY_INDEX)); + } + + return finish(runs, sessionID); + } + + /** + * Labels for an explicit token sequence, for tests and callers that already have the keys in hand. Same + * precedence and same checker semantics as the Index.db walk, so the keys must be in ascending order. + */ + @VisibleForTesting + static List

+ * Unlike the rewrite this does not purge tombstones -- a verbatim chunk copy retains everything the parent held. + * Retention, never loss; see {@link Config#zero_copy_anticompaction_enabled}. + *

+ * Planning happens before anything is carved out, so an ineligible or unreadable sstable simply stays in + * {@code groupTxn}. Once carved into its own transaction a failed split falls back to {@link #antiCompactGroup} + * on that same unused transaction, which is the race-free way to keep the sstable anticompacted -- aborting + * would unmark it as compacting and let a normal compaction take it. + * + * @param handledByZeroCopy out-param: every parent this removed from {@code groupTxn}, all fully anticompacted + * @return the number of output sstables produced + */ + @VisibleForTesting + int zeroCopyAntiCompact(ColumnFamilyStore cfs, + RangesAtEndpoint ranges, + LifecycleTransaction groupTxn, + TimeUUID pendingRepair, + BooleanSupplier isCancelled, + Set handledByZeroCopy) + { + int produced = 0; + // groupTxn.originals() is a live view that split() mutates, so iterate over a copy + for (SSTableReader parent : new ArrayList<>(groupTxn.originals())) + { + // The repair session's own cancellation, checked between sstables: whatever is left stays in groupTxn and + // antiCompactGroup raises the interruption. The other channel is the CompactionInfo.Holder registered in + // zeroCopySplitOne, used by nodetool stop / truncate / drop, which aborts mid-copy. + if (isCancelled.getAsBoolean()) + { + logger.info("Zero-copy anticompaction cancelled for {}, leaving the rest to the rewrite path", + pendingRepair); + break; + } + + AntiCompactionRunPlanner.Plan plan; + try + { + plan = AntiCompactionRunPlanner.plan(parent, ranges, pendingRepair); + } + catch (Throwable t) + { + JVMStabilityInspector.inspectThrowable(t); + logger.warn("Could not plan a zero-copy anticompaction of {}, falling back to the rewrite path", + parent.descriptor, t); + continue; // untouched, still in groupTxn + } + + if (!plan.eligible) + { + logger.debug("Not zero-copy anticompacting {}: {}", parent.descriptor, plan.ineligibleReason); + continue; // untouched, still in groupTxn + } + + produced += zeroCopySplitOne(cfs, ranges, parent, plan, groupTxn, pendingRepair, isCancelled); + handledByZeroCopy.add(parent); + } + return produced; + } + + /** + * Carve one eligible sstable out of {@code groupTxn} and replace it with its zero-copy split children. + *

+ * The transaction sequence is {@code SSTableRewriter.doPrepare}'s for a 1-to-N replacement, so the parent leaves + * the live set and the children enter it in one {@code tracker.apply} -- no window where the key range is served + * by neither. The parent is obsoleted rather than cancelled: a plain {@code finish()} would remove it from its + * compaction strategy while leaving it in the live view. + *

+ * The parent's reference in {@code validatedForRepair} is deliberately not released here; commit only drops the + * Tracker's, and the repair session's is what defers the unlink to the end of {@code performAnticompaction}. + */ + private int zeroCopySplitOne(ColumnFamilyStore cfs, + RangesAtEndpoint ranges, + SSTableReader parent, + AntiCompactionRunPlanner.Plan plan, + LifecycleTransaction groupTxn, + TimeUUID pendingRepair, + BooleanSupplier isCancelled) + { + // split() asserts the transaction has never been used, so this must happen before groupTxn is updated or + // obsoleted. The new transaction owns its own txn log file, which is what makes the children crash-safe: the + // splitter trackNew's them on it, so an abort or a crash deletes them. + LifecycleTransaction zcTxn = groupTxn.split(singleton(parent)); + List children = Collections.emptyList(); + int published = 0; + boolean settled = false; + try + { + ZeroCopySSTableSplitter.Result result; + try + { + // Registering with `active` is what makes the copy an ordinary compaction-family operation: visible in + // nodetool compactionstats, bounded by getRateLimiter(), and stoppable by nodetool stop / truncate / + // drop / runWithCompactionsDisabled, all of which walk active.getCompactions() and call Holder.stop(). + ZeroCopySSTableSplitter.Progress progress = + ZeroCopySSTableSplitter.progressFor(parent, getRateLimiter()); + active.beginCompaction(progress); + try + { + result = ZeroCopySSTableSplitter.split(parent, plan.boundaries, plan.perChild, zcTxn, progress); + } + finally + { + active.finishCompaction(progress); + } + children = result.children; + // Every planned run holds at least one partition, so no child can have been dropped. If one somehow + // were, the per-child repair state would be mis-paired and children stamped with the wrong session. + if (children.size() != plan.perChild.size()) + throw new IllegalStateException("zero-copy split of " + parent.descriptor + " produced " + + children.size() + " children for " + plan.perChild.size() + + " planned runs; repair state would be mis-paired"); + } + catch (CompactionInterruptedException e) + { + // Somebody asked for this to stop. Falling back to the rewrite would answer that with strictly more + // work than the copy just cancelled, so propagate and let the finally below delete the children. + logger.info("Zero-copy anticompaction of {} was stopped", parent.descriptor); + throw e; + } + catch (Throwable t) + { + // Nothing has been update()d yet, so zcTxn is unused and the ordinary rewrite can run on it verbatim. + // Skipping the sstable would leave data unrepaired that the repair session believes is pending. + JVMStabilityInspector.inspectThrowable(t); + logger.warn("Zero-copy anticompaction of {} failed, falling back to the rewrite path", + parent.descriptor, t); + discardUnpublishedChildren(zcTxn, children, 0); + children = Collections.emptyList(); + int rewritten = antiCompactGroup(cfs, ranges, zcTxn, pendingRepair, isCancelled); + settled = true; // antiCompactGroup committed zcTxn + return rewritten; + } + + for (ZeroCopySSTableSplitter.Child child : children) + { + zcTxn.update(child.reader, false); // ownership of the child's selfRef transfers here + published++; + } + zcTxn.obsoleteOriginals(); + zcTxn.prepareToCommit(); + zcTxn.commit(); + settled = true; + + // The metric is the data volume that went through this path, not the I/O it cost: it is a subset of + // bytesAnticompacted, and sharing extents does not make less data get anticompacted. + cfs.metric.bytesZeroCopyAnticompaction.inc(result.totalPhysicalBytesCopied); + logger.info("Zero-copy anticompacted {} in {}.{} into {} children for {}: {} bytes handled, {} shared " + + "with the parent as copy-on-write extents and {} actually written, {} bytes dead prefix, " + + "{} bytes head pad, {} bytes duplicated, {} ms. NOTE: this path copies compression chunks " + + "verbatim and therefore RETAINS droppable tombstones and shadowed data that a rewriting " + + "anticompaction would have purged (retention only, never data loss).", + parent.descriptor, cfs.keyspace.getName(), cfs.getTableName(), children.size(), + pendingRepair, result.totalPhysicalBytesCopied, result.totalBytesCloned, + result.totalBytesWritten(), result.totalDeadPrefixBytes, result.totalHeadPadBytes, + result.duplicatedChunkBytes, TimeUnit.NANOSECONDS.toMillis(result.nanos)); + return children.size(); + } + finally + { + if (!settled) + { + // Children that never reached update() are still ours; those that did belong to the transaction and + // are released by abort(). Releasing one twice throws "BAD RELEASE". + discardUnpublishedChildren(zcTxn, children, published); + try + { + zcTxn.abort(); + } + catch (Throwable t) + { + logger.error("Failed aborting the zero-copy anticompaction of {}", parent.descriptor, t); + } + } + try + { + zcTxn.close(); // no-op once committed or aborted + } + catch (Throwable t) + { + logger.error("Failed closing the zero-copy anticompaction transaction of {}", parent.descriptor, t); + } + } + } + + /** + * Release and delete the children from {@code from} onwards: those written and trackNew'd but never handed to + * {@code update()}. Never call this for a child that reached {@code update()} -- the transaction owns that + * reference and releases it itself. + */ + private static void discardUnpublishedChildren(LifecycleTransaction zcTxn, + List children, + int from) + { + for (int i = from; i < children.size(); i++) + { + ZeroCopySSTableSplitter.Child child = children.get(i); + try + { + child.reader.selfRef().release(); + } + catch (Throwable t) + { + logger.warn("Failed releasing unpublished zero-copy child {}", child.descriptor, t); + } + try + { + zcTxn.untrackNew(child.reader); // drops the ADD record and deletes any surviving files + } + catch (Throwable t) + { + logger.warn("Failed untracking unpublished zero-copy child {}", child.descriptor, t); + } + } + } + @VisibleForTesting int antiCompactGroup(ColumnFamilyStore cfs, RangesAtEndpoint ranges, diff --git a/src/java/org/apache/cassandra/db/compaction/Scrubber.java b/src/java/org/apache/cassandra/db/compaction/Scrubber.java index 56825d09ace6..b78518db9138 100644 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@ -183,9 +183,12 @@ public void scrub() nextIndexKey = indexAvailable() ? ByteBufferUtil.readWithShortLength(indexFile) : null; if (indexAvailable()) { - // throw away variable so we don't have a side effect in the assert long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); - assert firstRowPositionFromIndex == 0 : firstRowPositionFromIndex; + // Usually 0, making both statements no-ops. A ZeroCopySSTableSplitter child instead begins with a + // dead prefix: chunk boundaries are pinned to multiples of chunkLength, so a child not starting on + // one carries leading bytes belonging to no partition. Walk from where the index points. + nextPartitionPositionFromIndex = firstRowPositionFromIndex; + dataFile.seek(firstRowPositionFromIndex); } StatsMetadata metadata = sstable.getSSTableMetadata(); diff --git a/src/java/org/apache/cassandra/db/compaction/Verifier.java b/src/java/org/apache/cassandra/db/compaction/Verifier.java index 29eb29951b90..870f6313181c 100644 --- a/src/java/org/apache/cassandra/db/compaction/Verifier.java +++ b/src/java/org/apache/cassandra/db/compaction/Verifier.java @@ -245,8 +245,15 @@ public void verify() ByteBuffer nextIndexKey = ByteBufferUtil.readWithShortLength(indexFile); { long firstRowPositionFromIndex = rowIndexEntrySerializer.deserializePositionAndSkip(indexFile); - if (firstRowPositionFromIndex != 0) - markAndThrow(new RuntimeException("firstRowPositionFromIndex != 0: "+firstRowPositionFromIndex)); + // Usually 0, making the seek a no-op. A ZeroCopySSTableSplitter child instead begins with a dead + // prefix: chunk boundaries are pinned to multiples of chunkLength, so a child not starting on one + // carries leading bytes belonging to no partition. A position OUTSIDE the file is still a corrupt + // index, and goes through markAndThrow so it resets the sstable to UNREPAIRED and honours the disk + // failure policy, rather than escaping as the bare IllegalArgumentException seek() would raise. + if (firstRowPositionFromIndex < 0 || firstRowPositionFromIndex >= dataFile.length()) + markAndThrow(new RuntimeException("firstRowPositionFromIndex is outside the data file: " + + firstRowPositionFromIndex + " not in [0, " + dataFile.length() + ')')); + dataFile.seek(firstRowPositionFromIndex); } List> ownedRanges = isOffline ? Collections.emptyList() : Range.normalize(tokenLookup.apply(cfs.metadata().keyspace)); diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java index 367c304b0805..2deb69ed0b15 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java @@ -196,9 +196,15 @@ public boolean contained(List sections, S if (sections == null || sections.isEmpty()) return false; - // if transfer sections contain entire sstable + // Entire-sstable streaming copies component files verbatim, so it is eligible whenever the sections cover all + // of the sstable's LIVE data, not only when their span equals the physical data length. A + // ZeroCopySSTableSplitter child can carry a dead prefix -- bytes before its first indexed partition that no + // read path enters -- and getPositionsForRanges() starts the first section at the first partition, so the + // eligible span runs from there to the end of the file. For an ordinary sstable firstPosition == 0 and this + // reduces to the original transferLength == uncompressedLength check. + long firstPosition = sstable.getPosition(sstable.first.getToken().minKeyBound(), SSTableReader.Operator.GT).position; long transferLength = sections.stream().mapToLong(p -> p.upperPosition - p.lowerPosition).sum(); - return transferLength == sstable.uncompressedLength(); + return transferLength == sstable.uncompressedLength() - firstPosition; } @Override diff --git a/src/java/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitter.java b/src/java/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitter.java new file mode 100644 index 000000000000..6e0df4785afb --- /dev/null +++ b/src/java/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitter.java @@ -0,0 +1,2126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.zip.CRC32; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.RateLimiter; + +import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; +import com.clearspring.analytics.stream.cardinality.ICardinality; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionInterruptedException; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.metadata.CompactionMetadata; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileOutputStreamPlus; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.io.util.Reflink; +import org.apache.cassandra.io.util.SequentialWriter; +import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.BloomFilter; +import org.apache.cassandra.utils.BloomFilterSerializer; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.FilterFactory; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.MurmurHash; +import org.apache.cassandra.utils.SyncUtil; +import org.apache.cassandra.utils.TimeUUID; + +/** + * Splits one BIG-format SSTable into K children by copying verbatim compression-chunk runs of Data.db and + * rebuilding every other component from an Index.db-only pass. No decompression, no row deserialization. + * + *

Chunk boundaries are pinned to multiples of {@code chunkLength} -- {@link CompressionMetadata#chunkFor(long)} + * indexes the offsets array by {@code position / chunkLength}, and no per-chunk uncompressed length is stored -- so + * a child can only be a verbatim run of whole chunks {@code [i, j]}. A child whose first partition does not sit on + * a chunk boundary therefore carries a dead prefix of {@code lo mod chunkLength} bytes, since index + * positions are rebased by {@code shift = i * chunkLength} rather than by {@code lo}. Read paths enter Data.db only + * at positions taken from Index.db and tolerate it; {@code Scrubber} and {@code Verifier}, which walked linearly + * from 0, were changed to seek to the first index position. + * + *

Where the filesystem can share extents, the child's Data.db is reflinked from the parent's rather than copied + * (see {@link Reflink}), so the split writes no data blocks. That costs a head pad: {@code FICLONERANGE} + * needs block-aligned offsets, so the range is extended back to the previous 64 KiB boundary and the child's + * offsets rebased by it, leaving up to 64 KiB of the parent's previous chunk at the head with + * {@code offsets[0] == pad}. A second, physical dead prefix, independent of the uncompressed one, and covered by + * Digest.crc32 since {@code Verifier} checksums the whole file. {@link CopyPlan} is the arithmetic. A padded range + * that ends up copied instead produces a byte-for-byte identical child, so the two paths cannot diverge. + * + *

Trailing slack is forbidden. The last chunk's length is derived as + * {@code compressedFileLength - offsets[C-1] - 4}, so one extra byte inflates it and can flip the reader's + * {@code length < maxCompressedLength} test, handing compressed bytes back as raw data. The child's Data.db is + * truncated to exactly {@code headPad + O(j+1) - O(i)} and asserted. + * + *

Uncompressed sstables and tables with a secondary index are refused; see {@link #isSupported(SSTableReader)}. + * JBOD is unsupported but NOT refused, being a deployment constraint: {@link #descriptorAllocator} allocates every + * child in the PARENT's directory and never asks {@code Directories} for a location with room, so children cannot + * spill to a sibling disk and a parent larger than the free space on its own disk will fill it. + * + *

Statistics.db is deliberately imprecise. Four {@code StatsMetadata} fields are per-sstable totals that cannot + * be recomputed without deserialising rows, so every child inherits the parent-wide + * {@code estimatedCellPerPartitionCount}, {@code totalRows}, {@code totalColumnsSet} and + * {@code estimatedTombstoneDropTime} while {@code estimatedPartitionSize} is re-derived exactly. Per-table + * aggregates then over-report by roughly K and {@code worthDroppingTombstones} under-fires by roughly K. Every + * inherited value is at least as wide as the truth, so nothing can lose or resurrect data. + * + *

Every component is fsynced before {@code SSTableReader.open}, plus one {@link SyncUtil#trySyncDir} per child: + * committing unlinks the parent and the COMMIT record is itself fsynced, so a component still in page cache at that + * moment could be lost while the parent's removal survives. That is why Statistics.db is not written through + * {@code MetadataSerializer.rewriteSSTableMetadata} and the filter and summary not through + * {@code SSTableReader.saveBloomFilter}/{@code saveSummary} -- none of those fsync, and the latter two swallow the + * IOException and delete the half-written file. + * + *

With a {@link Progress} the copy joins the compaction framework: visible in {@code nodetool compactionstats}, + * bounded by {@code compaction_throughput}, and stoppable. Without one -- offline tools and tests -- it runs + * unthrottled. + */ +public final class ZeroCopySSTableSplitter +{ + private static final Logger logger = LoggerFactory.getLogger(ZeroCopySSTableSplitter.class); + + /** Prefix of the refusal for an uncompressed parent, so tests need not match the whole sentence. */ + public static final String UNCOMPRESSED_UNSUPPORTED_MESSAGE = + "ZeroCopySSTableSplitter requires a compressed sstable"; + + /** + * One {@code transferTo} slice. Deliberately small: it is the granularity at which {@link Progress} + * throttles against {@code compaction_throughput} and notices a stop request, so a multi-GiB slice would + * make the copy effectively unthrottled and uninterruptible. + */ + private static final int TRANSFER_SLICE = 4 << 20; + + /** Same buffer size the digest/checksum writers use. */ + private static final int COPY_BUFFER_SIZE = 64 * 1024; + + /** Alignment the head pad is computed against; see {@link Reflink#RANGE_ALIGNMENT}. */ + private static final long CLONE_ALIGNMENT = Reflink.RANGE_ALIGNMENT; + + /** + * A child smaller than this is copied rather than shared: the head pad costs disk space and a longer digest + * pass, so sharing only pays when the range dwarfs it. 1 MiB is 16 times the pad, a 6% overhead ceiling at + * the very bottom of the range. + */ + private static final long MIN_CLONE_BYTES = 1L << 20; + + /** + * Test hook: lay every child out as if extent sharing were available -- head pad and all -- so the aligned + * layout is covered on filesystems that cannot share extents, i.e. every laptop and CI box. Also lifts + * {@link #MIN_CLONE_BYTES}, since test sstables are smaller than that. The copy mechanism is unaffected. + */ + @VisibleForTesting + static volatile boolean forceAlignedLayoutForTesting = false; + + /** {@code MetadataCollector.defaultPartitionSizeHistogram()} is package-private; this is bit-identical. */ + static final int PARTITION_SIZE_HISTOGRAM_BUCKETS = 150; + + /** {@code MetadataCollector.cardinality} is {@code new HyperLogLogPlus(13, 25)} (CASSANDRA-5906). */ + static final int HLL_P = 13; + static final int HLL_SP = 25; + + /** Every component this class can write, i.e. everything {@link #cleanUp} has to remove. */ + private static final List WRITTEN_COMPONENTS = ImmutableList.of(Component.DATA, + Component.PRIMARY_INDEX, + Component.COMPRESSION_INFO, + Component.STATS, + Component.SUMMARY, + Component.FILTER, + Component.DIGEST, + Component.TOC); + + private ZeroCopySSTableSplitter() + { + } + + // ------------------------------------------------------------------------------------------------ + // Arithmetic. Deliberately static and free of any sstable dependency so it can be unit tested alone. + // ------------------------------------------------------------------------------------------------ + + /** Index of the compression chunk containing {@code uncompressedPosition}, as {@code chunkFor} computes it. */ + public static long chunkIndexFor(long uncompressedPosition, int chunkLength) + { + checkChunkLength(chunkLength); + if (uncompressedPosition < 0) + throw new IllegalArgumentException("negative uncompressed position: " + uncompressedPosition); + return uncompressedPosition / chunkLength; + } + + /** First (inclusive) chunk of a child whose first live byte is at parent uncompressed offset {@code lo}. */ + public static long firstChunk(long lo, int chunkLength) + { + return chunkIndexFor(lo, chunkLength); + } + + /** + * Last (inclusive) chunk of a child whose live bytes end at exclusive parent uncompressed offset {@code hi}. + * {@code (hi - 1) / L}, not {@code hi / L}: when {@code hi} lands on a chunk boundary the final chunk is the + * one before it, and {@code hi / L} would read a chunk too far. + */ + public static long lastChunk(long hi, int chunkLength) + { + checkChunkLength(chunkLength); + if (hi <= 0) + throw new IllegalArgumentException("child must contain at least one byte, hi=" + hi); + return (hi - 1) / chunkLength; + } + + /** + * The child's {@code CompressionInfo.dataLength}: from the start of its first chunk to the end of its last + * live partition. No trailing slack -- {@code getPositionsForRanges} bounds on {@code uncompressedLength()}. + */ + public static long childDataLength(long hi, long firstChunk, int chunkLength) + { + checkChunkLength(chunkLength); + long dataLength = hi - firstChunk * chunkLength; + if (dataLength <= 0) + throw new IllegalArgumentException("non-positive child dataLength " + dataLength + + " (hi=" + hi + ", firstChunk=" + firstChunk + ", L=" + chunkLength + ')'); + return dataLength; + } + + /** Bytes at the head of the child Data.db that belong to no partition: {@code lo mod chunkLength}. */ + public static long deadPrefixBytes(long lo, int chunkLength) + { + checkChunkLength(chunkLength); + if (lo < 0) + throw new IllegalArgumentException("negative uncompressed position: " + lo); + return lo % chunkLength; + } + + /** + * The whole chunk-range computation for one child, as an immutable value. + * + * @param lo first live byte, inclusive, in PARENT uncompressed space (a partition start) + * @param hi last live byte + 1, exclusive, in PARENT uncompressed space (a partition end) + * @param chunkLength the parent's compression chunk length + */ + public static ChunkRange chunkRange(long lo, long hi, int chunkLength) + { + checkChunkLength(chunkLength); + if (lo < 0) + throw new IllegalArgumentException("negative lo: " + lo); + if (hi <= lo) + throw new IllegalArgumentException("empty child range [" + lo + ", " + hi + ')'); + + long i = firstChunk(lo, chunkLength); + long j = lastChunk(hi, chunkLength); + if (i > j) + throw new IllegalStateException("firstChunk " + i + " > lastChunk " + j + + " for [" + lo + ", " + hi + ") L=" + chunkLength); + + long chunkCount = j - i + 1; + long dataLength = childDataLength(hi, i, chunkLength); + + // The reason a verbatim run works at all: the last chunk holds at least one live byte (so it is + // mapped and decompressed) and at most a full chunk of them (so dataLength never overruns the run). + if (!((chunkCount - 1) * (long) chunkLength < dataLength && dataLength <= chunkCount * (long) chunkLength)) + throw new IllegalStateException(String.format("invariant (C-1)*L < Dp <= C*L violated: " + + "C=%d L=%d Dp=%d lo=%d hi=%d", + chunkCount, chunkLength, dataLength, lo, hi)); + + return new ChunkRange(lo, hi, chunkLength, i, j, chunkCount, dataLength, + i * (long) chunkLength, deadPrefixBytes(lo, chunkLength)); + } + + private static void checkChunkLength(int chunkLength) + { + if (chunkLength <= 0) + throw new IllegalArgumentException("chunkLength must be positive: " + chunkLength); + } + + /** + * Where the child's Data.db comes from and how it gets there: the physical half of the arithmetic, and the only + * part that knows about extent sharing. + *

+ * {@code FICLONERANGE} needs source offset, destination offset and length all aligned, and a chunk boundary is + * aligned to nothing. We control the destination offset and the length but not the source, so the copy is + * extended BACKWARDS to the previous alignment boundary and the child's offsets are rebased by it, putting + * {@code pad = O(i) mod A} bytes of the parent's previous chunk at the head of the child. That is a physical + * dead prefix, distinct from {@link ChunkRange#deadPrefixBytes}, which lives in uncompressed space. + *

+ * {@code cloneLength} is the aligned part of the child's length; the remaining {@code tailLength < A} bytes are + * copied conventionally. Rounding the clone up and truncating instead would work on xfs, but would depend on + * truncate unsharing a partially shared final block for the sake of a sub-64-KiB copy. + * + * @param copyFrom {@code O(i)}, the parent offset of the child's first chunk + * @param physicalBytes {@code O(j+1) - O(i)}, the child's live chunk bytes + * @param align whether to pad the head so that sharing is possible at all + * @param share whether to actually attempt the clone; {@code align} without {@code share} is what a + * test uses to produce the padded layout on a filesystem that cannot share + */ + public static CopyPlan copyPlan(long copyFrom, long physicalBytes, boolean align, boolean share) + { + if (copyFrom < 0) + throw new IllegalArgumentException("negative copyFrom: " + copyFrom); + if (physicalBytes <= 0) + throw new IllegalArgumentException("non-positive physicalBytes: " + physicalBytes); + + long pad = align ? copyFrom & (CLONE_ALIGNMENT - 1) : 0; + long childLength = pad + physicalBytes; + // Aligned down, so the clone can never reach past the child's last live byte into the parent's trailing + // slack -- which chunkEnd() exists to keep out of the child. + long cloneLength = share ? childLength - (childLength & (CLONE_ALIGNMENT - 1)) : 0; + return new CopyPlan(copyFrom - pad, pad, childLength, cloneLength); + } + + /** Immutable result of {@link #copyPlan(long, long, boolean, boolean)}. */ + public static final class CopyPlan + { + /** Parent offset the child's byte 0 is taken from: {@code O(i) - headPadBytes}, alignment-aligned. */ + public final long srcStart; + /** Bytes of the parent's previous chunk at the head of the child, and the child's {@code offsets[0]}. */ + public final long headPadBytes; + /** Exact length of the child's Data.db: {@code headPadBytes + physicalBytes}. */ + public final long childLength; + /** Leading part of the child that {@code FICLONERANGE} is asked for; 0 means "copy all of it". */ + public final long cloneLength; + + CopyPlan(long srcStart, long headPadBytes, long childLength, long cloneLength) + { + this.srcStart = srcStart; + this.headPadBytes = headPadBytes; + this.childLength = childLength; + this.cloneLength = cloneLength; + } + + /** Bytes that must be transferred even if the clone succeeds: {@code childLength mod A}. */ + public long tailLength() + { + return childLength - cloneLength; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + if (!(o instanceof CopyPlan)) + return false; + CopyPlan that = (CopyPlan) o; + return srcStart == that.srcStart && headPadBytes == that.headPadBytes + && childLength == that.childLength && cloneLength == that.cloneLength; + } + + @Override + public int hashCode() + { + return Objects.hash(srcStart, headPadBytes, childLength, cloneLength); + } + + @Override + public String toString() + { + return String.format("CopyPlan[src=%d pad=%d length=%d clone=%d tail=%d]", + srcStart, headPadBytes, childLength, cloneLength, tailLength()); + } + } + + /** + * Immutable result of {@link #chunkRange(long, long, int)}. All chunk indices are into the PARENT's + * offsets array; all byte counts are in the child's own space. + */ + public static final class ChunkRange + { + /** First live byte of the child, inclusive, in parent uncompressed space. */ + public final long lo; + /** Last live byte of the child + 1, exclusive, in parent uncompressed space. */ + public final long hi; + public final int chunkLength; + /** {@code i}: first parent chunk copied, inclusive. */ + public final long firstChunk; + /** {@code j}: last parent chunk copied, inclusive. */ + public final long lastChunk; + /** {@code C = j - i + 1}: the child's chunkCount. */ + public final long chunkCount; + /** {@code Dp = hi - i*L}: the child's CompressionInfo dataLength. */ + public final long dataLength; + /** {@code shift = i*L}: subtracted from every Index.db position. */ + public final long shift; + /** {@code lo mod L}: bytes at the head of the child Data.db owned by no partition. */ + public final long deadPrefixBytes; + + ChunkRange(long lo, long hi, int chunkLength, long firstChunk, long lastChunk, + long chunkCount, long dataLength, long shift, long deadPrefixBytes) + { + this.lo = lo; + this.hi = hi; + this.chunkLength = chunkLength; + this.firstChunk = firstChunk; + this.lastChunk = lastChunk; + this.chunkCount = chunkCount; + this.dataLength = dataLength; + this.shift = shift; + this.deadPrefixBytes = deadPrefixBytes; + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + if (!(o instanceof ChunkRange)) + return false; + ChunkRange that = (ChunkRange) o; + return lo == that.lo && hi == that.hi && chunkLength == that.chunkLength + && firstChunk == that.firstChunk && lastChunk == that.lastChunk + && chunkCount == that.chunkCount && dataLength == that.dataLength + && shift == that.shift && deadPrefixBytes == that.deadPrefixBytes; + } + + @Override + public int hashCode() + { + return Objects.hash(lo, hi, chunkLength, firstChunk, lastChunk, chunkCount, dataLength, shift, deadPrefixBytes); + } + + @Override + public String toString() + { + return String.format("ChunkRange[lo=%d hi=%d L=%d chunks=[%d,%d] C=%d Dp=%d shift=%d dead=%d]", + lo, hi, chunkLength, firstChunk, lastChunk, chunkCount, dataLength, shift, deadPrefixBytes); + } + } + + // ------------------------------------------------------------------------------------------------ + // Results + // ------------------------------------------------------------------------------------------------ + + /** + * Repair state to stamp into one child's Statistics.db instead of inheriting the parent's. Written by + * {@link #writeStatistics} before the child reader is opened, so the reader is born with the right + * state and no {@code mutateRepairedAndReload} is needed. + *

+ * The invariants checked here are the ones {@code CompactionStrategyHolder.managesRepairedGroup} and + * {@code PendingRepairHolder.managesRepairedGroup} assert when the Tracker routes a newly visible sstable to + * a strategy holder -- failing there means an {@code IllegalArgumentException} from inside a Tracker + * notification, a far worse place to find out. + */ + public static final class RepairState + { + /** {@code ActiveRepairService.UNREPAIRED_SSTABLE} (0) unless the data is already repaired. */ + public final long repairedAt; + /** The incremental repair session id, or {@code ActiveRepairService.NO_PENDING_REPAIR} (null). */ + public final TimeUUID pendingRepair; + /** Only ever true when {@code pendingRepair != null}. */ + public final boolean isTransient; + + public RepairState(long repairedAt, TimeUUID pendingRepair, boolean isTransient) + { + this(repairedAt, pendingRepair, isTransient, true); + } + + private RepairState(long repairedAt, TimeUUID pendingRepair, boolean isTransient, boolean validate) + { + if (validate) + { + Preconditions.checkArgument(pendingRepair == ActiveRepairService.NO_PENDING_REPAIR + || repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE, + "SSTables cannot be both repaired and pending repair"); + Preconditions.checkArgument(!isTransient || pendingRepair != ActiveRepairService.NO_PENDING_REPAIR, + "isTransient can only be true for sstables pending repairs"); + } + this.repairedAt = repairedAt; + this.pendingRepair = pendingRepair; + this.isTransient = isTransient; + } + + /** + * The state the overloads without an explicit one give every child: the parent's, verbatim and + * deliberately unvalidated, so they behave as they did before per-child repair state existed. + */ + public static RepairState inherit(StatsMetadata parentStats) + { + return new RepairState(parentStats.repairedAt, parentStats.pendingRepair, parentStats.isTransient, false); + } + + @Override + public boolean equals(Object o) + { + if (this == o) + return true; + if (!(o instanceof RepairState)) + return false; + RepairState that = (RepairState) o; + return repairedAt == that.repairedAt + && isTransient == that.isTransient + && Objects.equals(pendingRepair, that.pendingRepair); + } + + @Override + public int hashCode() + { + return Objects.hash(repairedAt, pendingRepair, isTransient); + } + + @Override + public String toString() + { + return String.format("RepairState[repairedAt=%d pendingRepair=%s transient=%s]", + repairedAt, pendingRepair, isTransient); + } + } + + /** One produced child sstable. */ + public static final class Child + { + /** Descriptor of the child, in the parent's directory, version and format. */ + public final Descriptor descriptor; + /** The child's first partition key (minimal copy). */ + public final DecoratedKey first; + /** The child's last partition key (minimal copy). */ + public final DecoratedKey last; + public final long firstChunk; + public final long lastChunk; + /** Exact physical byte length of the child Data.db, {@code O(j+1) - O(i)}. */ + public final long physicalBytes; + /** The child's CompressionInfo dataLength, {@code hi - i*L}. */ + public final long dataLength; + /** Value subtracted from every Index.db position, {@code i*L}. */ + public final long shift; + /** Bytes at the head of the child Data.db owned by no partition, {@code lo mod L}. */ + public final long deadPrefixBytes; + /** + * Bytes of the parent's PREVIOUS chunk at the head of this child's Data.db, there so its first chunk + * lands on an alignment boundary; also the child's {@code offsets[0]}. Zero unless the child's extents + * were (or were meant to be) shared with the parent. See {@link CopyPlan}. + */ + public final long headPadBytes; + /** Bytes of {@link #physicalBytes} that were shared with the parent instead of copied. */ + public final long clonedBytes; + public final long partitionCount; + /** Components written for the child; the exact set passed to {@code SSTableReader.open}. */ + public final Set components; + /** + * The repair state actually stamped into this child's Statistics.db: the state of the boundary range it + * came from, carried through rather than positionally re-derived, so an empty range that produced no + * child cannot shift the pairing. + */ + public final RepairState repairState; + /** The opened, validated child reader. The caller owns this reference and must release it. */ + public final SSTableReader reader; + + Child(Descriptor descriptor, DecoratedKey first, DecoratedKey last, ChunkRange range, + long physicalBytes, long headPadBytes, long clonedBytes, long partitionCount, + Set components, RepairState repairState, SSTableReader reader) + { + this.descriptor = descriptor; + this.first = first; + this.last = last; + this.firstChunk = range.firstChunk; + this.lastChunk = range.lastChunk; + this.physicalBytes = physicalBytes; + this.dataLength = range.dataLength; + this.shift = range.shift; + this.deadPrefixBytes = range.deadPrefixBytes; + this.headPadBytes = headPadBytes; + this.clonedBytes = clonedBytes; + this.partitionCount = partitionCount; + this.components = components; + this.repairState = repairState; + this.reader = reader; + } + + /** Exact length of the child's Data.db, and its {@code compressedFileLength}: pad included. */ + public long onDiskLength() + { + return headPadBytes + physicalBytes; + } + + @Override + public String toString() + { + return String.format("Child[%s chunks=[%d,%d] physical=%d pad=%d cloned=%d dataLength=%d shift=%d" + + " dead=%d partitions=%d %s]", + descriptor, firstChunk, lastChunk, physicalBytes, headPadBytes, clonedBytes, + dataLength, shift, deadPrefixBytes, partitionCount, repairState); + } + } + + /** Outcome of a whole split. */ + public static final class Result + { + /** The children, in token order. */ + public final List children; + /** + * Sum of every child's live chunk bytes, {@code O(j+1) - O(i)}: the data the split had to account for, + * NOT what it moved -- subtract {@link #totalBytesCloned} for that. + */ + public final long totalPhysicalBytesCopied; + public final long totalDeadPrefixBytes; + /** Sum of every child's head pad, i.e. the disk space alignment cost. */ + public final long totalHeadPadBytes; + /** + * Bytes shared with the parent as copy-on-write extents rather than copied, costing neither I/O nor disk + * space. Zero where the filesystem cannot share extents; otherwise within one alignment unit per child + * of {@code totalPhysicalBytesCopied + totalHeadPadBytes}. + */ + public final long totalBytesCloned; + /** + * Compressed bytes physically present in two children because a split boundary fell inside a chunk. + * Bounded by one chunk per interior boundary, and free rather than merely bounded when the extents are + * shared: both children point at the same physical chunk. + */ + public final long duplicatedChunkBytes; + /** Wall clock of the whole split. */ + public final long nanos; + + Result(List children, long totalPhysicalBytesCopied, long totalDeadPrefixBytes, + long totalHeadPadBytes, long totalBytesCloned, long duplicatedChunkBytes, long nanos) + { + this.children = children; + this.totalPhysicalBytesCopied = totalPhysicalBytesCopied; + this.totalDeadPrefixBytes = totalDeadPrefixBytes; + this.totalHeadPadBytes = totalHeadPadBytes; + this.totalBytesCloned = totalBytesCloned; + this.duplicatedChunkBytes = duplicatedChunkBytes; + this.nanos = nanos; + } + + /** Bytes actually written to produce every child's Data.db. */ + public long totalBytesWritten() + { + return totalPhysicalBytesCopied + totalHeadPadBytes - totalBytesCloned; + } + + @Override + public String toString() + { + return String.format("Result[children=%d physical=%d dead=%d pad=%d cloned=%d written=%d" + + " duplicated=%d %.1fms]", + children.size(), totalPhysicalBytesCopied, totalDeadPrefixBytes, + totalHeadPadBytes, totalBytesCloned, totalBytesWritten(), + duplicatedChunkBytes, nanos / 1_000_000.0); + } + } + + // ------------------------------------------------------------------------------------------------ + // Compaction-framework participation + // ------------------------------------------------------------------------------------------------ + + /** + * Makes one split a first-class member of the compaction framework rather than an invisible, unbounded burst of + * I/O: visible in {@code nodetool compactionstats}, bounded by {@code compaction_throughput} via the compaction + * {@link RateLimiter}, and stoppable by everything that walks {@code active.getCompactions()} and calls + * {@link CompactionInfo.Holder#stop()}. The {@link CompactionInfo} carries the parent sstable so + * {@code shouldStop} can match it. + *

+ * A verbatim chunk copy has no partition boundary to stop cleanly at, so the stop check lives inside the + * transfer loop and aborts the split outright. A caller must NOT answer that by falling back to the rewrite -- + * the operator asked for the work to stop, not to be done a more expensive way. + *

+ * {@code total} is deliberately an estimate: a boundary chunk lands in two children and an aligned child carries + * a head pad, so a split with many interior boundaries can report marginally over 100%. The digest pass is only + * counted when it will actually happen, otherwise a split would peg at 50% and finish. Shared bytes count + * towards {@code total} for the same reason but are not rate limited -- see {@link #cloned}. + */ + public static final class Progress extends CompactionInfo.Holder + { + private final TableMetadata metadata; + private final Set parent; + private final long total; + private final TimeUUID id; + private final AtomicLong completed = new AtomicLong(); + private final RateLimiter limiter; + + private Progress(SSTableReader parent, RateLimiter limiter) + { + this.metadata = parent.metadata(); + this.parent = ImmutableSet.of(parent); + int passes = DatabaseDescriptor.getZeroCopySplitDigestEnabled() ? 2 : 1; + this.total = passes * parent.onDiskLength(); + this.id = TimeUUID.Generator.nextTimeUUID(); + this.limiter = limiter; + } + + @Override + public CompactionInfo getCompactionInfo() + { + return new CompactionInfo(metadata, OperationType.ANTICOMPACTION, completed.get(), total, id, parent); + } + + /** One sstable of one table, so a paused global compaction must not silently stop it. */ + @Override + public boolean isGlobal() + { + return false; + } + + /** + * Called immediately BEFORE {@code bytes} move: throws if a stop was requested, then blocks until the + * rate limiter lets the slice through. Permits cover the whole slice even though {@code transferTo} may + * move fewer, which over-throttles by at most one slice per short count. + */ + void beforeSlice(int bytes) + { + checkStopped(); + if (bytes > 0) + limiter.acquire(bytes); + } + + void afterSlice(long bytes) + { + completed.addAndGet(bytes); + } + + /** The stop half of {@link #beforeSlice}, for work that moves no bytes and so must not be throttled. */ + void checkStopped() + { + if (isStopRequested()) + throw new CompactionInterruptedException(getCompactionInfo()); + } + + /** + * Bytes accounted for by sharing extents rather than moving them. Deliberately NOT rate limited: + * {@code compaction_throughput} bounds disk traffic and a clone generates none, so charging it would make + * a reflink split as slow as the copy it replaced. Still counted towards {@code total} so + * {@code compactionstats} reaches 100%. + */ + void cloned(long bytes) + { + completed.addAndGet(bytes); + } + } + + /** + * A {@link Progress} for splitting {@code parent}. The caller owns it: register it with + * {@code CompactionManager.active.beginCompaction} before {@link #split} and finish it afterwards. + */ + public static Progress progressFor(SSTableReader parent, RateLimiter limiter) + { + Preconditions.checkNotNull(parent, "parent"); + Preconditions.checkNotNull(limiter, "limiter"); + return new Progress(parent, limiter); + } + + // ------------------------------------------------------------------------------------------------ + // Entry points + // ------------------------------------------------------------------------------------------------ + + /** + * @return true iff {@link #split} can handle this parent: a compressed BIG-format sstable, at a version whose + * components this class can write ({@link #writesReadableComponents}), on a table with no secondary + * index ({@link #hasNoPerSSTableIndex}). Anything else is refused by {@link #requireSupported} with + * {@link UnsupportedOperationException}. + */ + public static boolean isSupported(SSTableReader parent) + { + return parent.descriptor.formatType == SSTableFormat.Type.BIG + && parent.compression + && writesReadableComponents(parent.descriptor.version) + && hasNoPerSSTableIndex(parent); + } + + /** + * Whether the components this class writes can be read back at {@code version}. + *

+ * A child keeps the PARENT's version, since its Data.db is the parent's bytes verbatim. Two of the component + * writers are version-blind though: {@link CompressionMetadata.Writer} always emits the + * {@code maxCompressedLength} field only {@code na}+ reads back, and {@link BloomFilterSerializer} always + * writes the 4.0 bit order only {@code na}+ expects (CASSANDRA-9067). Stamped with a 3.x version those are + * read back wrong rather than rejected -- CompressionInfo.db is parsed four bytes out of phase, making + * {@code chunkCount} the low half of {@code dataLength} and turning {@code open} into a multi-gigabyte + * {@code Memory.allocate} then {@code CorruptSSTableException}. + *

+ * Rather than teach those writers to downgrade, refuse: a 3.x sstable is one {@code upgradesstables} away and + * every caller's fallback is a rewrite, which produces a current-version sstable anyway. Statistics.db is not + * at issue -- {@link #writeStatistics} passes {@code child.version} to the metadata serializer. + */ + static boolean writesReadableComponents(Version version) + { + return version.hasMaxCompressedLength() && !version.hasOldBfFormat(); + } + + /** + * Whether the parent's table is free of secondary indexes, which a split cannot carry across. + *

+ * The rewrite this replaces hands {@code cfs.indexManager.listIndexes()} to {@code SSTableWriter.create}, so an + * index with per-sstable state gets an {@code SSTableFlushObserver} and its component is written alongside each + * output. SASI is the one such index in this tree, and rebuilding its {@code SI_*.db} means reading the rows -- + * the entire cost this class exists to avoid. Emitting children without it fails silently rather than loudly: + * {@code ColumnIndex.update} drops the un-indexed set {@code DataTracker.update} returns and + * {@code getBuiltIndexes} skips any sstable whose index file is absent, so queries just stop matching those + * partitions until a restart or {@code rebuild_index}. + *

+ * This refuses on ANY index, not only those with per-sstable components. A plain {@code CassandraIndex} keeps + * its data in a separate table and would survive a split untouched, so that is stricter than necessary -- but + * it is the cheap, obviously-correct test, it needs no {@code ColumnFamilyStore} so it holds offline too, and + * being wrong this way only costs such a table the rewrite it did before this existed. + */ + static boolean hasNoPerSSTableIndex(SSTableReader parent) + { + return parent.metadata().indexes.isEmpty(); + } + + /** + * The least {@link SSTable} that can carry a child's identity into the transaction log before any of its files + * exist, so a crash mid-split is cleaned up rather than half-adopted. See the call site in {@link #buildChild}. + *

+ * {@code LogRecord.make(ADD, table)} reads only {@code baseFilename()} and {@code getAllFilePaths().size()}, + * and the record's file list is rebuilt by listing the directory at replay, so the component set here only has + * to be non-empty -- it is not a claim about what the child will have. + */ + private static final class PendingChild extends SSTable + { + PendingChild(Descriptor descriptor, TableMetadataRef metadata) + { + super(descriptor, ImmutableSet.of(Component.DATA), metadata, + DatabaseDescriptor.getDiskOptimizationStrategy()); + } + } + + /** + * Split at the partition boundaries nearest to {@code numChildren} approximately-equal byte shares of the + * parent's uncompressed length. + * + * @param numChildren number of children to produce; must be >= 1 and <= the parent's partition count + * @param txn optional; if non-null every child is {@code trackNew}'d on it once fully written + * @throws UnsupportedOperationException if the parent is not a compressed BIG-format sstable + */ + public static Result split(SSTableReader parent, int numChildren, LifecycleTransaction txn) + { + return split(parent, numChildren, txn, null); + } + + /** + * As {@link #split(SSTableReader, int, LifecycleTransaction)}, but throttled by and interruptible through + * {@code progress}. + * + * @param progress optional; when non-null the copy is rate limited and a stop request raises + * {@link CompactionInterruptedException}. See {@link Progress}. + */ + public static Result split(SSTableReader parent, int numChildren, LifecycleTransaction txn, Progress progress) + { + Preconditions.checkArgument(numChildren >= 1, "numChildren must be >= 1, got %s", numChildren); + requireSupported(parent); + + long start = Clock.Global.nanoTime(); + // Three sequential Index.db passes, none retaining anything per partition: count, select, build. The count + // comes first because split-point selection needs the exact partition count up front for its tail-room + // clamp. Index.db is a couple of percent of Data.db, so the extra pass is cheap next to copying the chunk + // runs -- and it is what keeps heap at O(numChildren) instead of O(partitions). See RunSelector. + int partitionCount = countPartitions(parent); + if (numChildren > partitionCount) + throw new IllegalArgumentException("cannot split " + partitionCount + " partitions into " + + numChildren + " children"); + Runs runs = selectByByteShare(parent, numChildren, partitionCount); + return build(parent, runs, null, txn, progress, start); + } + + /** + * Split at explicit boundaries. Child {@code b} covers keys {@code [boundaries[b-1], boundaries[b])}, with + * the first child unbounded below and the last unbounded above -- so this produces up to + * {@code boundaries.size() + 1} children. Boundaries must be strictly increasing. + *

+ * A boundary range containing no partition produces no child (an empty sstable is not representable: + * {@code IndexSummaryBuilder.build} asserts a non-zero key count and {@code getPositionsForRanges} asserts + * {@code first < last}). So the returned list may be shorter than {@code boundaries.size() + 1}. + * + * @param txn optional; if non-null every child is {@code trackNew}'d on it once fully written + * @throws UnsupportedOperationException if the parent is not a compressed BIG-format sstable + */ + public static Result split(SSTableReader parent, List boundaries, LifecycleTransaction txn) + { + return split(parent, boundaries, null, txn); + } + + /** + * Split at explicit boundaries, stamping a caller-supplied repair state into each child instead of inheriting + * the parent's. Boundary semantics are those of {@link #split(SSTableReader, List, LifecycleTransaction)}: + * child {@code b} covers keys {@code [boundaries[b-1], boundaries[b])} and {@code perChild.get(b)} is the + * state for that range. + *

+ * Pairing. An empty boundary range still produces no child, so {@code result.children.size()} may be + * smaller than {@code perChild.size()}. The state is therefore carried with the range rather than re-derived + * afterwards, and what was written is exposed on {@link Child#repairState}. Pairing {@code children} against + * {@code perChild} positionally is only valid when every range is known to be non-empty. + * + * @param perChild one state per boundary range, so exactly {@code boundaries.size() + 1} entries, in the + * same order as the ranges; may be null to inherit the parent's state for every child + * @param txn optional; if non-null every child is {@code trackNew}'d on it once fully written + * @throws IllegalArgumentException if {@code perChild.size() != boundaries.size() + 1}, if any entry is + * null, or if the boundaries are not strictly increasing + * @throws UnsupportedOperationException if the parent is not a compressed BIG-format sstable + */ + public static Result split(SSTableReader parent, + List boundaries, + List perChild, + LifecycleTransaction txn) + { + return split(parent, boundaries, perChild, txn, null); + } + + /** + * As {@link #split(SSTableReader, List, List, LifecycleTransaction)}, but throttled by and interruptible + * through {@code progress}. This is the overload the anticompaction path uses. + * + * @param progress optional; when non-null the copy is rate limited against {@code compaction_throughput} and + * a stop request raises {@link CompactionInterruptedException}. See {@link Progress}. + */ + public static Result split(SSTableReader parent, + List boundaries, + List perChild, + LifecycleTransaction txn, + Progress progress) + { + Preconditions.checkNotNull(boundaries, "boundaries"); + requireSupported(parent); + for (int b = 1; b < boundaries.size(); b++) + { + if (boundaries.get(b - 1).compareTo(boundaries.get(b)) >= 0) + throw new IllegalArgumentException("boundaries must be strictly increasing: " + + boundaries.get(b - 1) + " >= " + boundaries.get(b)); + } + if (perChild != null) + { + if (perChild.size() != boundaries.size() + 1) + throw new IllegalArgumentException("perChild must have one entry per boundary range, i.e. " + + (boundaries.size() + 1) + " entries for " + boundaries.size() + + " interior boundaries, got " + perChild.size()); + for (int b = 0; b < perChild.size(); b++) + { + if (perChild.get(b) == null) + throw new IllegalArgumentException("perChild[" + b + "] is null"); + } + } + + long start = Clock.Global.nanoTime(); + // Two passes: the run starts fall out of the same walk that resolves the boundaries, so this form needs no + // counting pass. + Runs runs = selectByBoundaries(parent, boundaries); + return build(parent, runs, perChild, txn, progress, start); + } + + private static void requireSupported(SSTableReader parent) + { + Preconditions.checkNotNull(parent, "parent"); + if (parent.descriptor.formatType != SSTableFormat.Type.BIG) + throw new UnsupportedOperationException("ZeroCopySSTableSplitter only supports the BIG sstable " + + "format, got " + parent.descriptor.formatType); + if (!parent.compression) + throw new UnsupportedOperationException(UNCOMPRESSED_UNSUPPORTED_MESSAGE + ": " + parent.descriptor + + " has no CompressionInfo.db. An uncompressed split is a " + + "different algorithm -- the cut is exact rather than " + + "chunk-aligned, and CRC.db (whose 64KiB grid is addressed " + + "from origin 0) has to be regenerated wholesale rather " + + "than sliced. Refusing rather than emitting a child with " + + "a misaligned CRC.db."); + if (!writesReadableComponents(parent.descriptor.version)) + throw new UnsupportedOperationException("ZeroCopySSTableSplitter cannot write components for sstable " + + "version " + parent.descriptor.version + " (" + + parent.descriptor + "): a child keeps its parent's version, " + + "but CompressionInfo.db and Filter.db are written in the 'na'+ " + + "formats only. Run nodetool upgradesstables first."); + if (!hasNoPerSSTableIndex(parent)) + throw new UnsupportedOperationException("ZeroCopySSTableSplitter cannot split " + parent.descriptor + + ": table " + parent.metadata().keyspace + '.' + + parent.metadata().name + " has secondary indexes " + + parent.metadata().indexes.stream() + .map(i -> i.name).collect(Collectors.joining(", ")) + + ", whose per-sstable components a split cannot rebuild " + + "without reading the rows."); + if (!parent.descriptor.fileFor(Component.STATS).exists()) + throw new IllegalStateException("parent has no Statistics.db: " + parent.descriptor + + "; MetadataSerializer would silently fabricate defaults"); + } + + // ------------------------------------------------------------------------------------------------ + // Walking the parent Index.db + // ------------------------------------------------------------------------------------------------ + + /** Receives every Index.db record in on-disk order. */ + private interface IndexRecordConsumer + { + void accept(int index, ByteBuffer key, long position); + } + + /** + * One sequential walk of the parent Index.db, retaining nothing. + * + *

Deliberately does not hand back the positions. Collecting every partition's offset into a {@code long[]} + * is 8 bytes per partition (16-24 at the peak of a doubling), which is invisible on a 512 MiB parent and tens + * of gigabytes of heap on a terabyte of 1 KiB partitions -- for an array whose every access is sequential + * anyway. Downstream takes what it needs from the stream: {@link RunSelector} keeps O(numChildren) and + * {@link #buildChild} keeps one record of lookback. + * + * @return the exact number of records + */ + private static int walkIndex(SSTableReader parent, IndexRecordConsumer consumer) + { + long count = 0; + // A buffered reader rather than an mmap, so no record can straddle a mapping boundary. + try (RandomAccessReader in = RandomAccessReader.open(parent.descriptor.fileFor(Component.PRIMARY_INDEX))) + { + long indexSize = in.length(); + while (in.getFilePointer() != indexSize) + { + ByteBuffer key = ByteBufferUtil.readWithShortLength(in); + long position = RowIndexEntry.Serializer.readPosition(in); + int promotedSize = (int) in.readUnsignedVInt(); + if (promotedSize > 0) + in.skipBytesFully(promotedSize); + + if (count >= Integer.MAX_VALUE) + throw new IllegalStateException("parent has more than Integer.MAX_VALUE partitions, which " + + "run starts cannot address: " + parent.descriptor); + consumer.accept((int) count++, key, position); + } + } + catch (IOException e) + { + throw new CorruptSSTableException(e, parent.descriptor.filenameFor(Component.PRIMARY_INDEX)); + } + + if (count == 0) + throw new IllegalStateException("parent Index.db is empty: " + parent.descriptor); + + return (int) count; + } + + /** Just the record count, for the byte-share form, whose selection needs it before it can start. */ + private static int countPartitions(SSTableReader parent) + { + return walkIndex(parent, (index, key, position) -> {}); + } + + // ------------------------------------------------------------------------------------------------ + // Split-point selection: the START index of each run; run b is + // [runStarts[b], runStarts[b+1]) with an implicit terminator of partitionCount. + // ------------------------------------------------------------------------------------------------ + + /** + * Where each child's run of index records begins, and the parent Data.db offset of that first record. + * {@link #build} needs a run's {@code lo} before it can copy that child's chunks, so these cannot be recovered + * during the build pass -- but there are only ever {@code numChildren} of them. + */ + @VisibleForTesting + static final class Runs + { + final int[] runStarts; + /** + * {@code runPositions[b]} is the Data.db offset of record {@code runStarts[b]}. Meaningless for an + * empty trailing run, whose {@code runStarts[b] == partitionCount}; {@link #build} skips those before + * reading it. + */ + final long[] runPositions; + final int partitionCount; + + Runs(int[] runStarts, long[] runPositions, int partitionCount) + { + this.runStarts = runStarts; + this.runPositions = runPositions; + this.partitionCount = partitionCount; + } + } + + /** No record's offset can be this, so it doubles as "not filled in yet". */ + private static final long UNRESOLVED = -1; + + /** + * The explicit-boundary form: the run starts fall out of the same walk that compares keys against the + * boundaries, so this costs one pass, no extra reads, and no retained keys. + */ + private static Runs selectByBoundaries(SSTableReader parent, List boundaries) + { + IPartitioner partitioner = parent.getPartitioner(); + int[] runStarts = new int[boundaries.size() + 1]; + long[] runPositions = new long[boundaries.size() + 1]; + Arrays.fill(runPositions, UNRESOLVED); + int[] nextBoundary = { 0 }; + + int count = walkIndex(parent, (index, key, position) -> { + if (index == 0) + runPositions[0] = position; // runStarts[0] is 0 + + if (nextBoundary[0] < boundaries.size()) + { + DecoratedKey dk = partitioner.decorateKey(key); + // run b + 1 starts at the first record whose key is >= boundaries[b]; several boundaries can + // land on the same record, and each of those runs then shares its offset + while (nextBoundary[0] < boundaries.size() && dk.compareTo(boundaries.get(nextBoundary[0])) >= 0) + { + runStarts[++nextBoundary[0]] = index; + runPositions[nextBoundary[0]] = position; + } + } + }); + + // Boundaries past the parent's last key produce trailing empty runs. Their offsets stay UNRESOLVED and + // are never read: build() skips a run with from >= to, and the last non-empty run takes its hi from + // dataLength precisely because the run after it starts at partitionCount. + while (nextBoundary[0] < boundaries.size()) + runStarts[++nextBoundary[0]] = count; + + return new Runs(runStarts, runPositions, count); + } + + /** The byte-share form: one pass, driving {@link RunSelector}. */ + private static Runs selectByByteShare(SSTableReader parent, int numChildren, int partitionCount) + { + RunSelector selector = new RunSelector(parent.uncompressedLength(), numChildren, partitionCount); + int count = walkIndex(parent, (index, key, position) -> selector.offer(index, position)); + if (count != partitionCount) + throw new IllegalStateException("parent Index.db grew or shrank between passes: counted " + + partitionCount + ", then " + count + ": " + parent.descriptor); + return selector.finish(); + } + + /** + * Streaming form of {@link #chooseByByteShare}: fed every partition's Data.db offset in order, it produces the + * same run starts plus each run's first offset in O(numChildren) heap rather than O(partitions). + * + *

The selection is a forward scan with one record of lookback; only its two clamps needed random access, + * and both reach a bounded distance: + *

+ * {@link #chooseByByteShare} is kept as the reference implementation this is differentially tested against. + */ + @VisibleForTesting + static final class RunSelector + { + private final long uncompressedLength; + private final int numChildren; + private final int partitionCount; + + private final int[] runStarts; + private final long[] runPositions; + + /** Offsets of the last {@code min(numChildren, partitionCount)} records: all the tail clamp can name. */ + private final long[] tail; + private final int tailFrom; + + private long base = UNRESOLVED; + private long total; + /** The run being placed; runs {@code [1, nextRun)} have their start index decided. */ + private int nextRun = 1; + /** Runs {@code [1, firstUnresolved)} have their offset filled in. */ + private int firstUnresolved = 1; + private int previousIndex = -1; + private long previousPosition = UNRESOLVED; + + RunSelector(long uncompressedLength, int numChildren, int partitionCount) + { + Preconditions.checkArgument(numChildren >= 1 && numChildren <= partitionCount, + "numChildren %s out of range for %s partitions", numChildren, partitionCount); + this.uncompressedLength = uncompressedLength; + this.numChildren = numChildren; + this.partitionCount = partitionCount; + this.runStarts = new int[numChildren]; + this.runPositions = new long[numChildren]; + Arrays.fill(this.runPositions, UNRESOLVED); + this.tailFrom = Math.max(0, partitionCount - numChildren); + this.tail = new long[partitionCount - tailFrom]; + } + + void offer(int index, long position) + { + if (index >= tailFrom) + tail[index - tailFrom] = position; + + if (index == 0) + { + base = position; + total = uncompressedLength - base; + runStarts[0] = 0; + runPositions[0] = position; + } + + // A placement forced onto the record after the cursor could not read its offset at the time. + if (firstUnresolved < nextRun && runStarts[firstUnresolved] == index) + runPositions[firstUnresolved++] = position; + + // Several targets can fall inside one partition, so keep placing until this record is short of the next. + while (nextRun < numChildren) + { + long target = base + (total * nextRun) / numChildren; + if (position < target) + break; + place(index, position, target); + } + + previousIndex = index; + previousPosition = position; + } + + Runs finish() + { + // Targets the scan never reached: the cursor is at partitionCount, which the tail clamp pulls back to a + // real record. position and target go unread -- the snap-back is guarded on candidate < partitionCount. + while (nextRun < numChildren) + place(partitionCount, UNRESOLVED, UNRESOLVED); + + if (firstUnresolved != numChildren) + throw new IllegalStateException("run " + firstUnresolved + " of " + numChildren + + " never had its Data.db offset resolved"); + for (int m = 1; m < numChildren; m++) + { + if (runStarts[m] <= runStarts[m - 1]) + throw new IllegalStateException("run starts are not strictly increasing: " + + Arrays.toString(runStarts)); + } + return new Runs(runStarts, runPositions, partitionCount); + } + + private void place(int index, long position, long target) + { + int m = nextRun; + int candidate = index; + long candidatePosition = position; + + // snap to whichever partition boundary is nearer the target + if (candidate > 0 && candidate < partitionCount + && (position - target) > (target - previousPosition)) + { + candidate = previousIndex; + candidatePosition = previousPosition; + } + + // never emit an empty child ... + if (candidate <= runStarts[m - 1]) + { + candidate = runStarts[m - 1] + 1; + // one past the cursor: its offset arrives with the next record + candidatePosition = candidate == index ? position : UNRESOLVED; + } + // ... and always leave room for the runs still to be placed. This can only pull the candidate back into + // the tail window, never below the clamp above, since runStarts[m - 1] is itself bounded by + // partitionCount - (numChildren - (m - 1)). + int room = partitionCount - (numChildren - m); + if (candidate > room) + { + candidate = room; + candidatePosition = tail[candidate - tailFrom]; + } + + runStarts[m] = candidate; + runPositions[m] = candidatePosition; + if (candidatePosition != UNRESOLVED) + { + if (firstUnresolved != m) + throw new IllegalStateException("run " + m + " resolved out of order, expected " + firstUnresolved); + firstUnresolved = m + 1; + } + nextRun++; + } + } + + /** + * Reference implementation of split-point selection, kept because it reads far more easily than + * {@link RunSelector}, which is tested by asserting it agrees with this on randomised inputs. Not used in + * production: it needs every partition's offset at once, the allocation this class exists to avoid. + */ + @VisibleForTesting + static int[] chooseByByteShare(long[] positions, long uncompressedLength, int numChildren) + { + int n = positions.length; + int[] runStarts = new int[numChildren]; + runStarts[0] = 0; + + long base = positions[0]; + long total = uncompressedLength - base; + int cursor = 0; + for (int m = 1; m < numChildren; m++) + { + long target = base + (total * m) / numChildren; + while (cursor < n && positions[cursor] < target) + cursor++; + + int candidate = cursor; + // snap to whichever partition boundary is nearer the target + if (candidate > 0 && candidate < n + && (positions[candidate] - target) > (target - positions[candidate - 1])) + candidate--; + + // never emit an empty child, and always leave room for the runs still to be placed + candidate = Math.max(candidate, runStarts[m - 1] + 1); + candidate = Math.min(candidate, n - (numChildren - m)); + runStarts[m] = candidate; + cursor = Math.max(cursor, candidate); + } + return runStarts; + } + + // ------------------------------------------------------------------------------------------------ + // Pass 2: build every child from a single sequential walk of the parent Index.db + // ------------------------------------------------------------------------------------------------ + + private static Result build(SSTableReader parent, Runs runs, + List perRun, LifecycleTransaction txn, Progress progress, + long startNanos) + { + int[] runStarts = runs.runStarts; + int partitionCount = runs.partitionCount; + + CompressionMetadata meta = parent.getCompressionMetadata(); // owned by parent's dfile; never close it + final int chunkLength = meta.chunkLength(); + final long parentDataLength = meta.dataLength; + final long parentCompressedLength = meta.compressedFileLength; + + if (parent.uncompressedLength() != parentDataLength) + throw new IllegalStateException("uncompressedLength " + parent.uncompressedLength() + + " != CompressionMetadata.dataLength " + parentDataLength); + + // The offsets table must address every chunk the data needs, and is allowed to hold MORE: a + // compaction-produced sstable carries one extra zero-uncompressed-length chunk, because + // SSTableRewriter.doPrepare syncs the data file twice and CompressedSequentialWriter.flushData appends a + // chunk unconditionally, even on an empty buffer. Keeping those bytes out of the children is chunkEnd()'s + // job. Fewer entries than the data needs means the parent's CompressionInfo.db disagrees with its own + // dataLength and nothing here is safe. + long addressableChunks = meta.offHeapSize() / 8; + long neededChunks = (parentDataLength + chunkLength - 1) / chunkLength; + if (neededChunks > addressableChunks) + throw new IllegalStateException("parent CompressionInfo.db addresses only " + addressableChunks + + " chunks but dataLength " + parentDataLength + " needs " + + neededChunks + " at chunkLength " + chunkLength + ": " + + parent.descriptor); + + // The four parent metadata components, read once. allOf() is mandatory: unselected types are skipped + // on read and would be silently dropped from the child's Statistics.db. + Map parentMetadata = readParentMetadata(parent.descriptor); + StatsMetadata parentStats = (StatsMetadata) parentMetadata.get(MetadataType.STATS); + + if (perRun != null && perRun.size() != runStarts.length) + throw new IllegalStateException("perRun has " + perRun.size() + " entries for " + runStarts.length + + " runs; the caller-visible check in split() should have caught this"); + RepairState inherited = perRun == null ? RepairState.inherit(parentStats) : null; + + Supplier descriptors = descriptorAllocator(parent); + + List children = new ArrayList<>(runStarts.length); + List created = new ArrayList<>(runStarts.length); + long physicalTotal = 0; + long deadTotal = 0; + long padTotal = 0; + long clonedTotal = 0; + long duplicated = 0; + + boolean success = false; + try (RandomAccessReader index = RandomAccessReader.open(parent.descriptor.fileFor(Component.PRIMARY_INDEX))) + { + ChunkRange previous = null; + for (int b = 0; b < runStarts.length; b++) + { + int from = runStarts[b]; + int to = (b + 1 < runStarts.length) ? runStarts[b + 1] : partitionCount; + if (from >= to) + continue; // empty boundary range -> no child + + long lo = runs.runPositions[b]; + // The next run starts where this one's data ends; for the last run that is the end of the parent's + // data. An empty trailing run has runStarts == partitionCount, which is exactly the case that takes + // dataLength, so its UNRESOLVED offset is never read. + long hi = (to < partitionCount) ? runs.runPositions[b + 1] : parentDataLength; + if (lo == UNRESOLVED || hi == UNRESOLVED) + throw new IllegalStateException("run " + b + " has an unresolved Data.db offset"); + ChunkRange range = chunkRange(lo, hi, chunkLength); + + long copyFrom = chunkStart(meta, range.firstChunk, chunkLength); + long copyTo = chunkEnd(meta, range.lastChunk, chunkLength); + long physicalBytes = copyTo - copyFrom; + if (physicalBytes <= 0) + throw new IllegalStateException("non-positive physical length " + physicalBytes + " for " + range); + if (copyTo > parentCompressedLength) + throw new IllegalStateException("child would read past the end of the parent's " + + parentCompressedLength + "-byte Data.db (copyTo=" + copyTo + + ") for " + range); + + // Carried with the range, never re-derived positionally: an empty range above produced no child and + // must not shift the state of the ranges after it. + RepairState repairState = perRun == null ? inherited : perRun.get(b); + + Descriptor child = descriptors.get(); + created.add(child); + Child built = buildChild(parent, child, index, from, to, range, meta, copyFrom, + physicalBytes, parentMetadata, parentStats, repairState, txn, progress); + children.add(built); + + physicalTotal += physicalBytes; + deadTotal += range.deadPrefixBytes; + padTotal += built.headPadBytes; + clonedTotal += built.clonedBytes; + if (previous != null && previous.lastChunk == range.firstChunk) + { + duplicated += chunkEnd(meta, range.firstChunk, chunkLength) + - chunkStart(meta, range.firstChunk, chunkLength); + } + previous = range; + } + success = true; + } + catch (IOException e) + { + throw new UncheckedIOException("failed splitting " + parent.descriptor, e); + } + finally + { + if (!success) + cleanUp(children, created); + } + + Result result = new Result(ImmutableList.copyOf(children), physicalTotal, deadTotal, padTotal, + clonedTotal, duplicated, Clock.Global.nanoTime() - startNanos); + logger.info("Split {} into {} children: {}", parent.descriptor, children.size(), result); + return result; + } + + /** The absolute Data.db offset at which chunk {@code k} begins. */ + static long chunkStart(CompressionMetadata meta, long k, int chunkLength) + { + return chunkFor(meta, k, chunkLength).offset; + } + + /** + * The absolute Data.db offset one past the end of chunk {@code k}, INCLUDING its 4-byte inline CRC32. + *

+ * Derived from the chunk itself and deliberately never from the physical file length, which for a + * compaction-produced parent sits ~9 bytes past the last real chunk (see {@link #build}). Taking the file length + * as the end of the final chunk made the last child copy that slack, inflating its own last chunk's derived + * length and failing CRC32 on every read of it -- or, once the length crossed {@code maxCompressedLength}, + * returning compressed bytes as row data. Silent, since Digest.crc32 covers whatever was written. + */ + static long chunkEnd(CompressionMetadata meta, long k, int chunkLength) + { + CompressionMetadata.Chunk chunk = chunkFor(meta, k, chunkLength); + return chunk.offset + chunk.length + 4; // "4": the inline CRC32 the reader expects to follow the chunk + } + + private static CompressionMetadata.Chunk chunkFor(CompressionMetadata meta, long k, int chunkLength) + { + if (k < 0) + throw new IllegalArgumentException("negative chunk index " + k); + return meta.chunkFor(k * (long) chunkLength); + } + + @SuppressWarnings("resource") + private static Child buildChild(SSTableReader parent, + Descriptor child, + RandomAccessReader index, + int from, + int to, + ChunkRange range, + CompressionMetadata meta, + long copyFrom, + long physicalBytes, + Map parentMetadata, + StatsMetadata parentStats, + RepairState repairState, + LifecycleTransaction txn, + Progress progress) throws IOException + { + TableMetadata metadata = parent.metadata(); + int chunkLength = range.chunkLength; + int partitionCount = to - from; + + // DIGEST and FILTER are added below only if they are actually written; the set handed to + // SSTableReader.open and to appendTOC has to name the files that exist and no others. + Set components = Sets.newHashSet(Component.DATA, + Component.PRIMARY_INDEX, + Component.COMPRESSION_INFO, + Component.STATS, + Component.SUMMARY); + + // ---------- The transaction's ADD record, BEFORE the first byte of the child exists ---------- + // Same as BigTableWriter registering in its constructor ("must track before any files are created"), and + // for the same reason: the ADD record is the ONLY thing that makes the child's files visible to + // LogTransaction.removeUnfinishedLeftovers after a crash. Registering once the components are written + // leaves a multi-minute window in which a kill -9 strands files no boot path reclaims -- + // removeUnfinishedLeftovers skips them for want of a record and scrubDataDirectories' orphan sweep keeps + // any descriptor with a non-empty Data.db. A complete stranded child is then opened as a live sstable + // ALONGSIDE the parent it was meant to replace, so the same partitions exist twice in two repair states; + // one interrupted inside writeStatistics leaves a durable zero-length Statistics.db, which open() turns + // into a CorruptSSTableException the startup failure policy escalates on every boot. And with extents + // shared, a stranded Data.db pins the parent's blocks. + // + // The record needs nothing but the descriptor: LogRecord.make reads the base filename and component count, + // the files it deletes are found by listing the directory at replay, and LogFile's numFiles strictness is + // REMOVE-only. + if (txn != null) + txn.trackNew(new PendingChild(child, parent.metadata)); + + // ---------- Data.db: verbatim compressed chunk run, shared with the parent where possible ---------- + // Sharing needs the head of the run aligned, which costs a pad, so it is only planned for when the + // filesystem has not already said no. An unpadded run cannot be shared at all (O(i) is aligned to nothing), + // so the decision has to be made before the copy rather than after it fails. + boolean canShare = DatabaseDescriptor.getZeroCopySplitReflinkEnabled() + && Reflink.isPossibleIn(child.directory); + boolean align = forceAlignedLayoutForTesting || (canShare && physicalBytes >= MIN_CLONE_BYTES); + CopyPlan plan = copyPlan(copyFrom, physicalBytes, align, align && canShare); + long cloned = copyData(parent.descriptor.fileFor(Component.DATA), child.fileFor(Component.DATA), + child.directory, plan, progress); + long actual = child.fileFor(Component.DATA).length(); + if (actual != plan.childLength) + throw new IllegalStateException("child Data.db is " + actual + " bytes, expected exactly " + + plan.childLength + " (trailing slack corrupts the last chunk's" + + " length)"); + + // ---------- CompressionInfo.db: same params, rebased offsets, offsets[0] == headPadBytes ---------- + writeCompressionInfo(child, meta, range, plan); + + // ---------- Index.db + FILTER + SUMMARY + HLL + partition-size histogram, one pass ---------- + EstimatedHistogram partitionSizes = new EstimatedHistogram(PARTITION_SIZE_HISTOGRAM_BUCKETS); + ICardinality cardinality = new HyperLogLogPlus(HLL_P, HLL_SP); + double fpChance = metadata.params.bloomFilterFpChance; + // fpChance == 1.0 yields an AlwaysPresentFilter, which writeFilter's BloomFilter cast would fail on. + // The read path already treats a missing Filter.db as always-present, so just omit the component. + IFilter bf = fpChance < 1.0 ? FilterFactory.getFilter(partitionCount, fpChance) : null; + DecoratedKey first = null; + DecoratedKey last = null; + + try + { + try (SequentialWriter out = new SequentialWriter(child.fileFor(Component.PRIMARY_INDEX), writerOption()); + IndexSummaryBuilder summary = new IndexSummaryBuilder(partitionCount, + metadata.params.minIndexInterval, + Downsampling.BASE_SAMPLING_LEVEL)) + { + long previousPosition = UNRESOLVED; + for (int r = from; r < to; r++) + { + ByteBuffer key = ByteBufferUtil.readWithShortLength(index); + long position = RowIndexEntry.Serializer.readPosition(index); + int promotedSize = (int) index.readUnsignedVInt(); + byte[] promoted = null; + if (promotedSize > 0) + { + promoted = new byte[promotedSize]; + index.readFully(promoted); + } + + // Selection and this pass have to land on the same records. Checking the run's first offset + // against what selection recorded, plus strict monotonicity from there on, catches a + // desynchronised walk without keeping an offset per partition, and rules out a non-increasing + // parent index as well. + if (r == from) + { + if (position != range.lo) + throw new IllegalStateException("index walk desynchronised at record " + r + + ": run starts at " + position + ", selection said " + + range.lo); + } + else + { + if (position <= previousPosition) + throw new IllegalStateException("parent Index.db offsets are not strictly increasing" + + " at record " + r + ": " + previousPosition + " -> " + + position); + // exact estimatedPartitionSize: rowSize_i == position_{i+1} - position_i identically, so + // each partition is sized one record late, from the next record's offset + partitionSizes.add(position - previousPosition); + } + previousPosition = position; + + DecoratedKey dk = parent.getPartitioner().decorateKey(key); + // MetadataCollector.addKey hashes the raw key bytes, position/remaining passed explicitly + long hashed = MurmurHash.hash2_64(key, key.position(), key.remaining(), 0); + + long childIndexStart = out.position(); + ByteBufferUtil.writeWithShortLength(key, out); + // The ONLY rewritten field. Canonical minimal vint, never padded, so the child's records are + // shorter than the parent's and its index offsets are NOT the parent's minus a constant. + out.writeUnsignedVInt(position - range.shift); + out.writeUnsignedVInt(promotedSize); + if (promoted != null) + out.write(promoted, 0, promotedSize); + + if (first == null) + first = dk; + last = dk; + if (bf != null) + bf.add(dk); + summary.maybeAddEntry(dk, childIndexStart); + cardinality.offerHashed(hashed); + } + + // The run's last partition ends where the next run's first record starts, which for the last run is + // the end of the parent's data -- exactly what chunkRange() was handed as hi. + if (range.hi <= previousPosition) + throw new IllegalStateException("run ends at " + range.hi + " but its last record is at " + + previousPosition); + partitionSizes.add(range.hi - previousPosition); + out.finish(); + + first = SSTable.getMinimalKey(first); + last = SSTable.getMinimalKey(last); + try (IndexSummary built = summary.build(parent.getPartitioner())) + { + writeSummary(child, first, last, built); + } + } + requireNonEmpty(child, Component.SUMMARY); + + // ---------- Filter.db ---------- + if (bf != null) + { + writeFilter(child, bf); + requireNonEmpty(child, Component.FILTER); + components.add(Component.FILTER); + } + } + finally + { + if (bf != null) + bf.close(); + } + + // ---------- Statistics.db ---------- + // compressionRatio is compressed-over-uncompressed for the FILE, so the pad counts: it is on disk and + // every consumer of the ratio is estimating disk footprint from a partition count. + writeStatistics(child, parentMetadata, parentStats, partitionSizes, cardinality, + plan.childLength, range.dataLength, repairState); + + // ---------- Digest.crc32: CRC32 over EVERY physical byte of the child Data.db ---------- + // Optional, and the one component whose cost scales with the DATA rather than the index: with extents + // shared this read is the whole remaining cost of the split. Skipping it is a supported configuration -- + // see writeDigest and Config.zero_copy_split_digest_enabled. + if (DatabaseDescriptor.getZeroCopySplitDigestEnabled()) + { + writeDigest(child, progress); + requireNonEmpty(child, Component.DIGEST); + components.add(Component.DIGEST); + } + + // ---------- TOC.txt, last: appendTOC opens in APPEND mode so it must run exactly once ---------- + components.add(Component.TOC); + SSTable.appendTOC(child, components); + + // Every component's CONTENTS are fsynced individually above; this makes their DIRECTORY ENTRIES durable + // too, since a directory that does not list a file whose data is on disk loses it just the same. Only the + // components written through SequentialWriter (Index.db, Statistics.db) sync the directory themselves, on + // create; Data.db, Filter.db, Summary.db, Digest.crc32 and TOC.txt do not. This has to happen before the + // child is published: the transaction's COMMIT record is itself fsynced and unlinks the parent. + SyncUtil.trySyncDir(child.directory); + + SSTableReader reader = SSTableReader.open(child, components, parent.metadata); + try + { + validateChild(reader, range, plan, physicalBytes, partitionCount, chunkLength); + } + catch (Throwable t) + { + reader.selfRef().release(); + throw t; + } + + // Deliberately no trackNew(reader): the ADD record for this descriptor went in before the copy started, and + // trackNew does nothing but write that record, keyed on the base filename. + + return new Child(child, first, last, range, physicalBytes, plan.headPadBytes, cloned, partitionCount, + ImmutableSet.copyOf(components), repairState, reader); + } + + // ------------------------------------------------------------------------------------------------ + // Component writers + // ------------------------------------------------------------------------------------------------ + + /** + * Materialise the child's Data.db as the verbatim parent byte range + * {@code [plan.srcStart, plan.srcStart + plan.childLength)}, sharing as much as the filesystem allows and + * copying the rest. + *

+ * The clone is all-or-nothing: {@code FICLONERANGE} either shares every byte asked for or writes nothing, so a + * refusal costs one syscall and falls through to the transfer loop. + *

+ * transferTo returns short counts and caps near 0x7ffff000, so it MUST be looped; {@code n <= 0} means EOF, not + * "retry". The loop is also where the operation is throttled and cancelled, one {@link #TRANSFER_SLICE} at a + * time. A clone moves no bytes, so it is checked for cancellation but not throttled. + * + * @return how many bytes were shared rather than copied; 0 means the whole range was transferred + */ + private static long copyData(File src, File dst, File directory, CopyPlan plan, Progress progress) + throws IOException + { + try (FileChannel in = src.newReadChannel(); + FileChannel outChannel = dst.newWriteChannel(File.WriteMode.OVERWRITE)) + { + long cloned = 0; + if (plan.cloneLength > 0) + { + if (progress != null) + progress.checkStopped(); + if (Reflink.tryCloneRange(in, plan.srcStart, outChannel, 0, plan.cloneLength, directory)) + { + cloned = plan.cloneLength; + if (progress != null) + progress.cloned(cloned); + } + } + + // The ioctl does not move the destination's file position and transferTo writes at wherever that is, so + // the tail has to be positioned explicitly. Without this it would overwrite the head of the range just + // shared -- which, being copy-on-write, would silently succeed. + outChannel.position(cloned); + + long position = plan.srcStart + cloned; + long remaining = plan.childLength - cloned; + while (remaining > 0) + { + int slice = (int) Math.min(remaining, TRANSFER_SLICE); + if (progress != null) + progress.beforeSlice(slice); + long n = in.transferTo(position, slice, outChannel); + if (n <= 0) + throw new IOException(String.format("short transferTo of %s at %d with %d left", + src, position, remaining)); + position += n; + remaining -= n; + if (progress != null) + progress.afterSlice(n); + } + outChannel.truncate(plan.childLength); // never leave a trailing byte + outChannel.force(true); + return cloned; + } + } + + /** + * Child CompressionInfo.db via the same {@code Writer} every real sstable uses, so it cannot drift from the + * format. Only dataLength, chunkCount and the offsets differ from the parent. + *

+ * Offsets are rebased by {@link CopyPlan#srcStart} rather than {@code O(i)}, so the child's {@code offsets[0]} + * is its head pad instead of 0 whenever the run was aligned for sharing. They remain absolute positions in the + * child's own Data.db, and the last chunk's derived length is unaffected because the pad shifts both terms. + */ + private static void writeCompressionInfo(Descriptor child, CompressionMetadata meta, ChunkRange range, + CopyPlan plan) + { + CompressionMetadata.Writer writer = + CompressionMetadata.Writer.open(meta.parameters, child.filenameFor(Component.COMPRESSION_INFO)); + boolean prepared = false; + try + { + for (long k = range.firstChunk; k <= range.lastChunk; k++) + { + long offset = meta.chunkFor(k * (long) range.chunkLength).offset - plan.srcStart; + if (k == range.firstChunk && offset != plan.headPadBytes) + throw new IllegalStateException("child offsets[0] must be " + plan.headPadBytes + + ", got " + offset); + writer.addOffset(offset); + } + writer.finalizeLength(range.dataLength, Math.toIntExact(range.chunkCount)); + writer.prepareToCommit(); // doPrepare() is what writes and fsyncs the file + prepared = true; + writer.commit(); + } + catch (Throwable t) + { + // doAbort() only frees memory, it does not delete an already-written file + if (!prepared) + writer.abort(); + child.fileFor(Component.COMPRESSION_INFO).deleteIfExists(); + throw t; + } + finally + { + writer.close(); + } + } + + /** + * The child's Statistics.db: the parent's four components with two derived replacements + * (estimatedPartitionSize and the COMPACTION cardinality) plus a recomputed compressionRatio. + *

+ * HEADER passes through by reference and MUST be inherited byte-for-byte: rows in the copied Data.db encode + * timestamps/localDeletionTime/TTL as unsigned vint deltas off + * {@code stats.minTimestamp/minLocalDeletionTime/minTTL}, and their columns as a bitmap subset of + * {@code header.columns()}. Tightening any of those silently corrupts every relocated row, with all CRCs still + * passing. + *

+ * {@code commitLogIntervals} and {@code originatingHostId} are inherited as an ATOMIC PAIR. Stamping the child + * with the LOCAL host id, as every MetadataCollector constructor does, while inheriting a foreign parent's + * intervals would have CommitLogReplayer -- which gates on {@code originatingHostId.equals(localhostId)} -- + * interpret foreign segment ids against the local commitlog and discard acked-but-unflushed mutations. + *

+ * The repair state is written here rather than mutated afterwards so the reader opened a few lines later is + * already correct: the Tracker routes a newly visible sstable to a strategy holder by exactly that triple. + * {@code sstableLevel} is inherited, as {@code createWriterForAntiCompaction} does for a single-input + * anticompaction -- safe because the children are disjoint contiguous sub-ranges of the parent's range. + */ + static void writeStatistics(Descriptor child, + Map parentMetadata, + StatsMetadata parentStats, + EstimatedHistogram partitionSizes, + ICardinality cardinality, + long onDiskLength, + long dataLength, + RepairState repairState) throws IOException + { + // The four absolute TOTALS below (estimatedCellPerPartitionCount, estimatedTombstoneDropTime, + // totalColumnsSet, totalRows) are parent-wide in every child, so per-table aggregates over-report by ~K and + // worthDroppingTombstones under-fires by ~K. Accepted and conservative; see "Accepted imprecision in the + // children's Statistics.db" on the class javadoc. + StatsMetadata childStats = new StatsMetadata(partitionSizes, // DERIVED, exact + parentStats.estimatedCellPerPartitionCount, // ACCEPTED: parent-wide + parentStats.commitLogIntervals, // atomic pair, see javadoc + parentStats.minTimestamp, + parentStats.maxTimestamp, + parentStats.minLocalDeletionTime, + parentStats.maxLocalDeletionTime, + parentStats.minTTL, + parentStats.maxTTL, + (double) onDiskLength / dataLength, // DERIVED, exact + parentStats.estimatedTombstoneDropTime, // ACCEPTED: parent-wide + parentStats.sstableLevel, + parentStats.minClusteringValues, + parentStats.maxClusteringValues, + parentStats.hasLegacyCounterShards, + repairState.repairedAt, // CALLER SUPPLIED + parentStats.totalColumnsSet, // ACCEPTED: parent-wide + parentStats.totalRows, // ACCEPTED: parent-wide + parentStats.originatingHostId, // atomic pair, see javadoc + repairState.pendingRepair, // CALLER SUPPLIED + repairState.isTransient); // CALLER SUPPLIED + + Map components = new EnumMap<>(parentMetadata); + components.put(MetadataType.STATS, childStats); + components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality)); + // VALIDATION (partitioner + fp chance) and HEADER pass through by reference: no schema lookup, + // nothing that can throw, byte-identical to the parent. + + // Written the way BigTableWriter.writeMetadata does -- SequentialWriter plus finish() -- and NOT through + // MetadataSerializer.rewriteSSTableMetadata, which only flushes and renames, fsyncing neither the file nor + // the directory. That is fine for its existing callers, which mutate the repair status of an sstable whose + // Statistics.db is ALREADY durable, and not fine here: this is the only copy of the child's + // SerializationHeader and repair state. finish() ends in syncInternal() and SequentialWriter fsyncs the + // directory on create, so both are durable before the COMMIT record unlinks the parent. + File file = child.fileFor(Component.STATS); + try (SequentialWriter out = new SequentialWriter(file, writerOption())) + { + child.getMetadataSerializer().serialize(components, out, child.version); + out.finish(); + } + requireNonEmpty(child, Component.STATS); + } + + /** + * Filter.db, fsynced. {@code BigTableWriter.IndexWriter.flushBf} rather than + * {@code SSTableReader.saveBloomFilter}, which neither fsyncs nor reports failure: it logs at TRACE, deletes + * the half-written file and returns normally, so {@code open()} would quietly rebuild the filter and hide the + * error, and a crash could leave a torn one behind. + */ + static void writeFilter(Descriptor child, IFilter filter) throws IOException + { + try (FileOutputStreamPlus out = new FileOutputStreamPlus(child.fileFor(Component.FILTER))) + { + BloomFilterSerializer.serialize((BloomFilter) filter, out); + out.flush(); + out.sync(); + } + } + + /** + * Summary.db, fsynced. {@code SSTableReader.saveSummary} writes the same three things but never fsyncs and + * swallows the failure. A torn Summary.db is the most survivable of the three -- {@code SSTableReaderBuilder} + * rebuilds it from Index.db -- but that means a full Index.db pass per child at startup. + */ + static void writeSummary(Descriptor child, DecoratedKey first, DecoratedKey last, IndexSummary summary) + throws IOException + { + try (FileOutputStreamPlus out = new FileOutputStreamPlus(child.fileFor(Component.SUMMARY))) + { + IndexSummary.serializer.serialize(summary, out); + ByteBufferUtil.writeWithLength(first.getKey(), out); + ByteBufferUtil.writeWithLength(last.getKey(), out); + out.flush(); + out.sync(); + } + } + + /** + * Digest.crc32 is the plain decimal ASCII of a java.util.zip.CRC32 over EVERY physical byte of Data.db, with no + * newline and no prefix. Correct for a compressed sstable too: the writer folds the inline per-chunk CRCs into + * the full checksum ({@code appendDirect(bb, checksumIncrementalResult=true)}). + *

+ * "Every physical byte" must include the head pad, since {@code Verifier} validates this digest by CRC-ing the + * whole Data.db file with no reference to CompressionInfo.db -- and a mismatch trips {@code markAndThrow}, + * which stamps the sstable unrepaired and throws into the disk failure policy. + *

+ * This pass dominates the cost of a split whose extents were shared: the copy stops reading the parent, but + * this still reads every byte of every child. Two ways out, one implemented: + *

    + *
  • SKIP IT, with {@code zero_copy_split_digest_enabled: false}. Nothing needs the component and a + * compressed sstable is self-checking without it (every chunk carries an inline CRC32 that this path + * preserves and the read path verifies); the cost is {@code Verifier} upgrading to a full extended + * verification. See {@link org.apache.cassandra.config.Config#zero_copy_split_digest_enabled} for the + * consumer audit.
    • + *
  • DERIVE IT, not implemented. The digest covers a byte range that is verbatim parent, and each of the + * parent's per-chunk CRC32s is stored inline after its chunk with no offset or chunk index mixed in, so + * the value could be assembled with {@code crc32_combine} from 4 bytes per chunk plus the pad -- keeping + * the component for a quarter of the read at {@code chunk_length_in_kb: 16}. Separate change with its own + * correctness burden, and a wrong digest is silent until somebody runs {@code nodetool verify}.
    • + *
+ */ + private static void writeDigest(Descriptor child, Progress progress) throws IOException + { + CRC32 crc = new CRC32(); + byte[] buffer = new byte[COPY_BUFFER_SIZE]; + try (InputStream in = child.fileFor(Component.DATA).newInputStream()) + { + int n; + while ((n = in.read(buffer)) > 0) + { + // A second full pass over every byte just written, so throttled and cancellable on the same terms as + // the copy -- otherwise stopping would leave the node grinding through a read of every child. + if (progress != null) + progress.beforeSlice(n); + crc.update(buffer, 0, n); + if (progress != null) + progress.afterSlice(n); + } + } + try (FileOutputStreamPlus out = new FileOutputStreamPlus(child.fileFor(Component.DIGEST))) + { + out.write(String.valueOf(crc.getValue()).getBytes(StandardCharsets.UTF_8)); + out.flush(); + out.sync(); + } + } + + // ------------------------------------------------------------------------------------------------ + // Validation and plumbing + // ------------------------------------------------------------------------------------------------ + + /** Cheap post-write checks; every one of them catches a distinct off-by-one. */ + private static void validateChild(SSTableReader child, ChunkRange range, CopyPlan plan, long physicalBytes, + int partitionCount, int chunkLength) + { + long onDisk = child.descriptor.fileFor(Component.DATA).length(); + if (onDisk != plan.childLength) + throw new IllegalStateException("child Data.db length " + onDisk + " != " + plan.childLength); + if (onDisk - plan.headPadBytes != physicalBytes) + throw new IllegalStateException("child Data.db holds " + (onDisk - plan.headPadBytes) + + " chunk bytes after a " + plan.headPadBytes + "-byte pad, != " + + physicalBytes); + if (child.uncompressedLength() != range.dataLength) + throw new IllegalStateException("child uncompressedLength " + child.uncompressedLength() + + " != " + range.dataLength); + + CompressionMetadata childMeta = child.getCompressionMetadata(); + // The head pad is the ONE place a child's physical layout differs from a writer's, so it is asserted both + // ways: the file cannot be short of it and the offsets table cannot disagree about it. + if (childMeta.chunkFor(0).offset != plan.headPadBytes) + throw new IllegalStateException("child offsets[0] " + childMeta.chunkFor(0).offset + " != head pad " + + plan.headPadBytes); + if (childMeta.chunkLength() != chunkLength) + throw new IllegalStateException("child chunkLength " + childMeta.chunkLength() + " != " + chunkLength); + + RowIndexEntry entry = child.getPosition(child.first, SSTableReader.Operator.EQ, false); + if (entry == null) + throw new IllegalStateException("child cannot find its own first key " + child.first); + long expectedFirst = range.lo - range.shift; + if (entry.position != expectedFirst) + throw new IllegalStateException("child first position " + entry.position + " != " + expectedFirst); + if (entry.position != range.deadPrefixBytes) + throw new IllegalStateException("child first position " + entry.position + + " != dead prefix " + range.deadPrefixBytes); + if (entry.position >= chunkLength) + throw new IllegalStateException("child first position " + entry.position + + " must be inside the first chunk (L=" + chunkLength + ')'); + if (child.first.compareTo(child.last) > 0) + throw new IllegalStateException("child first > last: " + child.first + " > " + child.last); + + RowIndexEntry lastEntry = child.getPosition(child.last, SSTableReader.Operator.EQ, false); + if (lastEntry == null) + throw new IllegalStateException("child cannot find its own last key " + child.last); + + // Decompress the child's FINAL chunk -- the one construct every other check here is blind to. The last + // chunk is physically whole while the child's dataLength says only part of it is live, so its length is + // derived rather than stored and a single byte of trailing slack changes it. Digest.crc32 cannot catch that + // (it covers whatever was written, so it stays self-consistent) and the checks above only touch chunkFor(0) + // and child.first. Reading the last live byte forces CompressedChunkReader's normal path, where a wrong + // derived length fails the inline CRC32 (or LZ4's "Compressed lengths mismatch"). + try (RandomAccessReader in = child.openDataReader()) + { + in.seek(child.uncompressedLength() - 1); + in.readByte(); + } + catch (IOException e) + { + throw new CorruptSSTableException(e, child.descriptor.filenameFor(Component.DATA)); + } + + logger.trace("Child {} ok: {} partitions, {} physical bytes, dead prefix {}, last partition at {}", + child.descriptor, partitionCount, physicalBytes, range.deadPrefixBytes, lastEntry.position); + } + + static Map readParentMetadata(Descriptor parent) + { + Map components; + try + { + components = parent.getMetadataSerializer().deserialize(parent, EnumSet.allOf(MetadataType.class)); + } + catch (IOException e) + { + throw new CorruptSSTableException(e, parent.filenameFor(Component.STATS)); + } + for (MetadataType type : MetadataType.values()) + { + if (components.get(type) == null) + throw new IllegalStateException("parent Statistics.db is missing " + type + ": " + parent); + } + return components; + } + + /** + * Fresh descriptors in the parent's directory, version and format. Prefers the live ColumnFamilyStore's id + * generator so we cannot collide with a concurrent flush or compaction; falls back to a directory-derived + * generator plus an existence loop for offline use. + */ + static Supplier descriptorAllocator(SSTableReader parent) + { + Descriptor template = parent.descriptor; + ColumnFamilyStore cfs = null; + try + { + cfs = Schema.instance.getColumnFamilyStoreInstance(parent.metadata().id); + } + catch (Throwable t) + { + logger.debug("No live ColumnFamilyStore for {}, falling back to a directory-derived id generator", + template, t); + } + + if (cfs != null) + { + ColumnFamilyStore store = cfs; + return () -> store.newSSTableDescriptor(template.directory, template.version, template.formatType); + } + + Supplier ids = new Directories(parent.metadata()).getUIDGenerator(SSTableIdFactory.instance.defaultBuilder()); + return () -> { + for (int attempt = 0; attempt < 1000; attempt++) + { + Descriptor candidate = new Descriptor(template.version, template.directory, template.ksname, + template.cfname, ids.get(), template.formatType); + if (!candidate.fileFor(Component.DATA).exists()) + return candidate; + } + throw new IllegalStateException("could not allocate an unused sstable id in " + template.directory); + }; + } + + static SequentialWriterOption writerOption() + { + return SequentialWriterOption.newBuilder() + .trickleFsync(DatabaseDescriptor.getTrickleFsync()) + .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024) + .build(); + } + + /** + * A cheap post-condition. Every component here is written through a path that fsyncs and propagates + * IOException, unlike the {@code SSTableReader.save*} helpers, which log at TRACE, delete the half-written file + * and return normally. + */ + static void requireNonEmpty(Descriptor descriptor, Component component) + { + File file = descriptor.fileFor(component); + if (!file.exists() || file.length() == 0) + throw new IllegalStateException("failed to write " + component + " for " + descriptor); + } + + /** Best-effort removal of every partially written child, so a failed split leaves no orphans behind. */ + private static void cleanUp(List children, List created) + { + for (Child child : children) + { + try + { + child.reader.selfRef().release(); + } + catch (Throwable t) + { + logger.warn("Failed releasing child {} during cleanup", child.descriptor, t); + } + } + for (Descriptor descriptor : created) + { + for (Component component : WRITTEN_COMPONENTS) + { + deleteQuietly(descriptor.fileFor(component), descriptor); + } + // Statistics.db is written in place, not via rewriteSSTableMetadata's tmp file + rename, so this should + // never exist. Belt and braces: a leftover tmp would be picked up as an orphan. + deleteQuietly(new File(descriptor.tmpFilenameFor(Component.STATS)), descriptor); + } + children.clear(); + created.clear(); + } + + private static void deleteQuietly(File file, Descriptor descriptor) + { + try + { + file.deleteIfExists(); + } + catch (Throwable t) + { + logger.warn("Failed deleting {} while cleaning up {}", file, descriptor, t); + } + } +} diff --git a/src/java/org/apache/cassandra/io/util/MmappedRegions.java b/src/java/org/apache/cassandra/io/util/MmappedRegions.java index 0b7dd39353e3..b63ff227b0fa 100644 --- a/src/java/org/apache/cassandra/io/util/MmappedRegions.java +++ b/src/java/org/apache/cassandra/io/util/MmappedRegions.java @@ -155,7 +155,11 @@ private void updateState(long length) private void updateState(CompressionMetadata metadata) { long offset = 0; - long lastSegmentOffset = 0; + // Where the first chunk physically starts: 0 for every sstable a writer produces, but not for one whose + // Data.db carries leading bytes belonging to no chunk (a ZeroCopySSTableSplitter child aligned so its + // extents can be shared with its parent). Segments are placed at a cumulative sum of chunk lengths, so + // seeding that sum at 0 for such a file maps every segment too early and leaves the file's tail unmapped. + long lastSegmentOffset = metadata.dataLength > 0 ? metadata.chunkFor(0).offset : 0; long segmentSize = 0; while (offset < metadata.dataLength) diff --git a/src/java/org/apache/cassandra/io/util/Reflink.java b/src/java/org/apache/cassandra/io/util/Reflink.java new file mode 100644 index 000000000000..514aeb8b0454 --- /dev/null +++ b/src/java/org/apache/cassandra/io/util/Reflink.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.nio.channels.FileChannel; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.sun.jna.LastErrorException; +import com.sun.jna.Memory; +import com.sun.jna.Native; +import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.utils.NativeLibrary; + +/** + * Byte-range extent sharing between two files on the same filesystem -- a "reflink" -- via the Linux + * {@code FICLONERANGE} ioctl. + * + *

The ioctl points a range of the destination at the same physical extents as a range of the source and + * bumps their reference count: nothing is read or written, and the cost is a refcount-btree update proportional to + * the extent count rather than the byte count. The files stay independent afterwards, the filesystem copying an + * affected block on the first write to it, and blocks are freed only when the last reference goes away -- so cloning + * out of a file about to be unlinked hands over its extents rather than copying and then freeing them. + * + *

This goes through JNA rather than leaning on {@code FileChannel.transferTo}, whose {@code copy_file_range} + * backend will reflink when all three arguments happen to be block aligned but otherwise falls back to a + * full copy silently, with no way to find out which happened. A caller that pads its destination specifically to + * make sharing possible needs to know whether the padding bought anything. + * + *

The kernel requires {@code srcOffset}, {@code dstOffset} and {@code length} to be multiples of the filesystem + * block size, failing unaligned arguments with {@code EINVAL} rather than rounding them. + * {@link #RANGE_ALIGNMENT} is a constant 64 KiB rather than a {@code statvfs} lookup, since a block can never exceed + * the page size and 64 KiB is the largest Linux runs with -- so a multiple of it is a multiple of the block size + * everywhere, at a cost of at most 64 KiB of slack per range. + * + *

Extent sharing needs a refcount btree: xfs formatted with {@code -m reflink=1} (the mkfs default since xfsprogs + * 5.1), btrfs, bcachefs, OCFS2. There is no cheap way to ask in advance -- {@code statfs} names the filesystem but + * not whether reflink was enabled at mkfs time -- so support is discovered by trying, and "this filesystem cannot do + * it" is remembered per directory in {@link #unsupported}. Any other errno is logged at WARN, being a bug in the + * caller's arithmetic. + * + *

Note that shared extents are cheap on disk but not in RAM: the page cache is per inode, so bytes read through + * both files are cached twice for as long as both stay live and hot. + */ +public final class Reflink +{ + private static final Logger logger = LoggerFactory.getLogger(Reflink.class); + + /** + * {@code FICLONERANGE == _IOW(0x94, 13, struct file_clone_range)}, i.e. + * {@code (1 << 30) | (32 << 16) | (0x94 << 8) | 13}. The 32 is {@code sizeof(struct file_clone_range)}; + * the request number is part of the kernel ABI and is identical on every Linux architecture. + */ + private static final long FICLONERANGE = 0x4020940DL; + + /** {@code sizeof(struct file_clone_range)}: four 8-byte fields, naturally aligned, no padding. */ + private static final int FILE_CLONE_RANGE_SIZE = 32; + + /** + * Every offset and length handed to {@link #tryCloneRange} must be a multiple of this: the largest possible + * filesystem block size rather than the actual one, so no native call is needed to discover it. + */ + public static final long RANGE_ALIGNMENT = 64 << 10; + + // errno values that mean "this filesystem/kernel/mount pairing can never share extents", as opposed to + // "these particular arguments were wrong". Linux asm-generic values, identical on every architecture + // Cassandra runs on. + private static final int EPERM = 1; + private static final int EBADF = 9; + private static final int EXDEV = 18; + private static final int EINVAL = 22; + private static final int ENOTTY = 25; + private static final int ENOSYS = 38; + private static final int EOPNOTSUPP = 95; + + /** + * Directories whose filesystem has already answered "no". Keyed by directory rather than mount because that is + * what a caller has in hand, and it is strictly finer grained: one ext4 data directory cannot disable sharing on + * an xfs one. + */ + private static final Map unsupported = new ConcurrentHashMap<>(); + + private static final boolean LINKED; + + static + { + boolean linked = false; + try + { + // Registered on this class rather than added to NativeLibraryLinux so a link failure here cannot take + // mlockall/fadvise/fcntl down with it: Native.register links every native method of a class at once. + // ioctl(2) is in every libc, so this is belt and braces -- unlike copy_file_range, which glibc only + // exposes from 2.27 and would fail to link on an older base image. + if (NativeLibrary.osType == NativeLibrary.OSType.LINUX) + { + Native.register(com.sun.jna.NativeLibrary.getInstance("c", Collections.emptyMap())); + linked = true; + } + } + catch (Throwable t) + { + logger.debug("Could not link ioctl(2); byte-range extent sharing is unavailable", t); + } + LINKED = linked; + } + + private Reflink() + { + } + + private static native int ioctl(int fd, NativeLong request, Pointer argp) throws LastErrorException; + + /** + * Whether {@link #tryCloneRange} is worth attempting for a file in {@code directory}: the ioctl is linked, this + * is Linux, and nothing in {@code directory} has failed with a filesystem-level error yet. Callers use it to + * decide up front whether to lay their destination out for sharing at all, since {@link #RANGE_ALIGNMENT} costs + * something to arrange and there is no point paying for it on ext4. + *

+ * Optimistic: an untried directory answers true, so the first caller pays one failing ioctl plus its own + * alignment cost. + */ + public static boolean isPossibleIn(File directory) + { + return LINKED && !unsupported.containsKey(directory.path()); + } + + /** + * Share {@code length} bytes of {@code src} starting at {@code srcOffset} into {@code dst} at + * {@code dstOffset}, moving no data. + *

+ * All three of {@code srcOffset}, {@code dstOffset} and {@code length} must be multiples of + * {@link #RANGE_ALIGNMENT}, and {@code srcOffset + length} must not exceed the source's length. The destination's + * length grows to at least {@code dstOffset + length}; its file position is NOT moved, so a caller writing the + * tail conventionally must position the channel itself. + *

+ * Neither channel is closed, flushed or synced. A successful clone is a metadata change like any write and still + * needs {@code force()} before anything may depend on it surviving a crash. + * + * @param directory the destination's directory, used only as the key of the negative cache + * @return true if the range is now shared; false if this filesystem cannot share extents, in which case nothing + * has been written and the caller must copy the bytes itself + * @throws IllegalArgumentException if the arguments are not aligned -- a caller bug rather than a filesystem + * limitation, and not to be answered by silently copying + */ + public static boolean tryCloneRange(FileChannel src, long srcOffset, + FileChannel dst, long dstOffset, + long length, File directory) + { + if (length <= 0) + throw new IllegalArgumentException("length must be positive, got " + length); + requireAligned("srcOffset", srcOffset); + requireAligned("dstOffset", dstOffset); + requireAligned("length", length); + + if (!isPossibleIn(directory)) + return false; + + int srcFd = NativeLibrary.getfd(src); + int dstFd = NativeLibrary.getfd(dst); + if (srcFd < 0 || dstFd < 0) + { + // Only when sun.nio.ch.FileChannelImpl.fd is not reflectively reachable, i.e. the --add-opens the startup + // script passes is missing. Nothing to do but copy the bytes. + noteUnsupported(directory, EBADF, "file descriptors are not reachable from FileChannel"); + return false; + } + + // struct file_clone_range, filled by hand rather than through a JNA Structure: four fixed-width fields with + // no padding on any ABI, and Memory is already in native byte order. Freed by the GC; JNA 5.9 has no close(). + Memory arg = new Memory(FILE_CLONE_RANGE_SIZE); + arg.setLong(0, srcFd); // __s64 src_fd + arg.setLong(8, srcOffset); // __u64 src_offset + arg.setLong(16, length); // __u64 src_length + arg.setLong(24, dstOffset); // __u64 dest_offset + + try + { + // All-or-nothing: 0 having shared every byte, or -1 having shared none. No short-clone case to loop over, + // and length is a u64, so even a terabyte range is one call. The return value is checked as well as the + // exception because JNA raises LastErrorException off errno rather than off the return value. + int rc = ioctl(dstFd, new NativeLong(FICLONERANGE), arg); + if (rc != 0) + { + logger.warn("FICLONERANGE of {} bytes at {} returned {} without setting errno; copying instead", + length, srcOffset, rc); + return false; + } + logger.trace("Shared {} bytes at {} into {} at {}", length, srcOffset, directory, dstOffset); + return true; + } + catch (LastErrorException e) + { + int errno = e.getErrorCode(); + if (isFilesystemLimitation(errno)) + noteUnsupported(directory, errno, strerror(errno)); + else + logger.warn("FICLONERANGE of {} bytes at {} failed with errno {} ({}); copying instead", + length, srcOffset, errno, strerror(errno)); + return false; + } + catch (UnsatisfiedLinkError e) + { + noteUnsupported(directory, ENOSYS, "ioctl(2) is not linked"); + return false; + } + } + + /** + * Forget every remembered negative answer. For tests, which mount and unmount filesystems under the same paths + * far more often than a running node does. + */ + public static void resetSupportCache() + { + unsupported.clear(); + } + + /** + * The errnos that mean the filesystem itself cannot do this, so no future call for the same directory can succeed + * either. {@code EINVAL} is included reluctantly: it also covers bad arguments, but alignment is validated up + * front, so the remaining source of it is a filesystem whose block size exceeds {@link #RANGE_ALIGNMENT}. + */ + private static boolean isFilesystemLimitation(int errno) + { + return errno == EOPNOTSUPP || errno == ENOTTY || errno == ENOSYS + || errno == EXDEV || errno == EINVAL || errno == EPERM; + } + + private static void noteUnsupported(File directory, int errno, String reason) + { + Integer previous = unsupported.putIfAbsent(directory.path(), errno); + if (previous == null) + logger.info("Byte-range extent sharing (reflink) is unavailable in {}: {} ({}). Ranges will be" + + " copied instead.", directory, reason, errno); + } + + private static void requireAligned(String what, long value) + { + if ((value & (RANGE_ALIGNMENT - 1)) != 0) + throw new IllegalArgumentException(what + " must be a multiple of " + RANGE_ALIGNMENT + + ", got " + value); + } + + private static String strerror(int errno) + { + switch (errno) + { + case EPERM: return "EPERM"; + case EBADF: return "EBADF"; + case EXDEV: return "EXDEV: source and destination are on different filesystems"; + case EINVAL: return "EINVAL"; + case ENOTTY: return "ENOTTY: the filesystem does not implement FICLONERANGE"; + case ENOSYS: return "ENOSYS"; + case EOPNOTSUPP: return "EOPNOTSUPP: the filesystem cannot share extents"; + default: return "errno " + errno; + } + } +} diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 24c4b16153e6..0661c763defc 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -204,6 +204,12 @@ public class TableMetrics public final Counter bytesAnticompacted; /** number of bytes where the whole sstable was contained in a repairing range so that we only mutated the repair status */ public final Counter bytesMutatedAnticompaction; + /** + * number of Data.db bytes copied verbatim by the zero-copy anticompaction split. A subset of + * {@link #bytesAnticompacted}, which is charged before the split path can claim any sstable, so + * {@link #mutatedAnticompactionGauge} keeps its existing meaning. + */ + public final Counter bytesZeroCopyAnticompaction; /** ratio of how much we anticompact vs how much we could mutate the repair status*/ public final Gauge mutatedAnticompactionGauge; @@ -900,6 +906,7 @@ protected double getDenominator() partitionsValidated = createTableHistogram("PartitionsValidated", cfs.keyspace.metric.partitionsValidated, false); bytesAnticompacted = createTableCounter("BytesAnticompacted"); bytesMutatedAnticompaction = createTableCounter("BytesMutatedAnticompaction"); + bytesZeroCopyAnticompaction = createTableCounter("BytesZeroCopyAnticompaction"); mutatedAnticompactionGauge = createTableGauge("MutatedAnticompactionGauge", () -> { double bytesMutated = bytesMutatedAnticompaction.getCount(); diff --git a/test/unit/org/apache/cassandra/db/compaction/AntiCompactionRunPlannerTest.java b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionRunPlannerTest.java new file mode 100644 index 000000000000..9ee47af28b42 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/compaction/AntiCompactionRunPlannerTest.java @@ -0,0 +1,634 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.compaction; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import org.junit.Test; + +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.KeyIterator; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter.RepairState; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.TimeUUID; + +import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; +import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; +import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * The run-planning half of zero-copy anticompaction: {@link AntiCompactionRunPlanner} decides whether one + * sstable's FULL / TRANSIENT / UNREPAIRED partitions form few enough contiguous runs that the splitter can + * reproduce the anticompaction, and if so where to cut and what repair state each piece gets. + * + *

Two layers. The pure run-length encoding is driven through + * {@link AntiCompactionRunPlanner#planFromLabels} with no sstable at all, so every shape -- including the + * pathological vnode interleavings -- is cheap to express. Then a handful of real compressed sstables go through + * {@link AntiCompactionRunPlanner#plan} with ranges derived from their own index keys, the only way to prove the + * Index.db walk labels and cuts in the same places. + * + *

The load-bearing assertion throughout is the exact identity of the boundary keys. A boundary is the FIRST key + * of the NEW run (the splitter starts a run at the first record whose key is {@code >=} the boundary), so an + * off-by-one silently hands one partition to the wrong repair state -- data that should stay unrepaired marked + * pending-repair for a session that never validated it, or vice versa. That is the worst bug this feature can + * have and it is invisible to a "the children add up to the parent" test, so every eligible case pins the + * boundary keys down exactly. + */ +public class AntiCompactionRunPlannerTest extends CQLTester +{ + // Shorthand so a label sequence reads like the on-disk shape it describes. + private static final AntiCompactionRunPlanner.Label F = AntiCompactionRunPlanner.Label.FULL; + private static final AntiCompactionRunPlanner.Label T = AntiCompactionRunPlanner.Label.TRANSIENT; + private static final AntiCompactionRunPlanner.Label U = AntiCompactionRunPlanner.Label.UNREPAIRED; + + // ---------------------------------------------------------------------------------------------------- + // Pure run encoding: no sstable, no files + // ---------------------------------------------------------------------------------------------------- + + @Test + public void noPartitionsIsIneligible() + { + List noLabels = Collections.emptyList(); + List noKeys = Collections.emptyList(); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.planFromLabels(noLabels, noKeys, nextTimeUUID()); + + assertFalse(plan.eligible); + assertEquals(0, plan.runCount); + assertEquals("sstable has no partitions", plan.ineligibleReason); + assertTrue(plan.boundaries.isEmpty()); + assertTrue(plan.perChild.isEmpty()); + } + + /** + * A single run means there is nothing to cut. All three flavours are ineligible, and each says which label + * it was -- "entire sstable is FULL" and "entire sstable is UNREPAIRED" are operationally very different + * situations (one is already covered by the fully-contained mutate path, the other is a no-op), so they must + * not collapse into one generic message. + */ + @Test + public void singleRunIsIneligibleWithItsOwnReason() + { + TimeUUID session = nextTimeUUID(); + AntiCompactionRunPlanner.Plan allUnrepaired = planOf(session, U, U, U, U); + AntiCompactionRunPlanner.Plan allFull = planOf(session, F, F, F, F); + AntiCompactionRunPlanner.Plan allTransient = planOf(session, T, T, T, T); + + for (AntiCompactionRunPlanner.Plan plan : Arrays.asList(allUnrepaired, allFull, allTransient)) + { + assertFalse(plan.toString(), plan.eligible); + assertEquals(1, plan.runCount); + assertTrue(plan.boundaries.isEmpty()); + assertTrue(plan.perChild.isEmpty()); + } + + assertTrue(allUnrepaired.ineligibleReason, + allUnrepaired.ineligibleReason.contains("entire sstable is UNREPAIRED")); + assertTrue(allFull.ineligibleReason, allFull.ineligibleReason.contains("entire sstable is FULL")); + assertTrue(allTransient.ineligibleReason, + allTransient.ineligibleReason.contains("entire sstable is TRANSIENT")); + + HashSet distinct = new HashSet<>(Arrays.asList(allUnrepaired.ineligibleReason, + allFull.ineligibleReason, + allTransient.ineligibleReason)); + assertEquals("the three single-run reasons must be distinguishable", 3, distinct.size()); + } + + /** A one-partition sstable is one run, whatever that partition is labelled. */ + @Test + public void singlePartitionIsIneligible() + { + TimeUUID session = nextTimeUUID(); + for (AntiCompactionRunPlanner.Label label : Arrays.asList(U, F, T)) + { + AntiCompactionRunPlanner.Plan plan = planOf(session, label); + assertFalse(plan.eligible); + assertEquals(1, plan.runCount); + assertTrue(plan.ineligibleReason.contains("entire sstable is " + label)); + } + } + + @Test + public void unrepairedThenFullIsEligible() + { + TimeUUID session = nextTimeUUID(); + List keys = ascendingKeys(6); + + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(U, U, U, F, F, F), keys, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertNull(plan.ineligibleReason); + assertEquals(2, plan.runCount); + // the cut is the first FULL key, not the last UNREPAIRED one + assertEquals(Arrays.asList(keys.get(3)), plan.boundaries); + assertNotEquals(keys.get(2), plan.boundaries.get(0)); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session)), plan.perChild); + } + + @Test + public void fullThenUnrepairedIsEligible() + { + TimeUUID session = nextTimeUUID(); + List keys = ascendingKeys(5); + + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(F, F, U, U, U), keys, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(2, plan.runCount); + assertEquals(Arrays.asList(keys.get(2)), plan.boundaries); + assertEquals(Arrays.asList(pendingFull(session), unrepaired()), plan.perChild); + } + + /** The common straddle: an sstable whose middle is owned and whose ends are not. */ + @Test + public void unrepairedFullUnrepairedIsEligible() + { + TimeUUID session = nextTimeUUID(); + List keys = ascendingKeys(9); + + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(U, U, F, F, F, F, U, U, U), keys, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(3, plan.runCount); + assertEquals(Arrays.asList(keys.get(2), keys.get(6)), plan.boundaries); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session), unrepaired()), plan.perChild); + } + + /** + * Two partitions with a transition between them: the tightest possible off-by-one. The boundary must be the + * SECOND key. If it were the first, the splitter would put partition 0 in the FULL child and the whole + * sstable would be marked pending-repair. + */ + @Test + public void twoPartitionTransitionCutsAtTheSecondKey() + { + TimeUUID session = nextTimeUUID(); + List keys = ascendingKeys(2); + + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(U, F), keys, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(2, plan.runCount); + assertEquals(1, plan.boundaries.size()); + assertEquals("the boundary must be the first key of the new run", keys.get(1), plan.boundaries.get(0)); + assertNotEquals("the boundary must not be the last key of the old run", keys.get(0), plan.boundaries.get(0)); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session)), plan.perChild); + + // and the mirror image, so the assertion above cannot pass by accident on a constant + AntiCompactionRunPlanner.Plan mirrored = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(F, U), keys, session); + assertTrue(mirrored.eligible); + assertEquals(Arrays.asList(keys.get(1)), mirrored.boundaries); + assertEquals(Arrays.asList(pendingFull(session), unrepaired()), mirrored.perChild); + } + + @Test + public void fullInTwoRunsIsIneligible() + { + TimeUUID session = nextTimeUUID(); + + AntiCompactionRunPlanner.Plan plan = planOf(session, F, F, U, U, F, F); + + assertFalse(plan.eligible); + assertEquals(3, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("FULL appears in 2 runs")); + assertTrue(plan.boundaries.isEmpty()); + assertTrue(plan.perChild.isEmpty()); + } + + /** What vnodes actually produce, and the whole reason for the gate. */ + @Test + public void alternatingRunsAreIneligible() + { + TimeUUID session = nextTimeUUID(); + + AntiCompactionRunPlanner.Plan plan = planOf(session, F, U, F, U, F); + + assertFalse(plan.eligible); + assertEquals(5, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("FULL appears in 3 runs")); + } + + /** + * Past {@code MAX_RETAINED_RUNS} the planner stops retaining boundary keys so an alternating layout cannot + * pin one key per partition on the heap -- but the run counters must stay exact, since they are what the + * INFO/DEBUG log reports. + */ + @Test + public void longAlternatingRunsStillCountExactlyWithoutRetainingKeys() + { + TimeUUID session = nextTimeUUID(); + List labels = new ArrayList<>(); + for (int i = 0; i < 40; i++) + labels.add(i % 2 == 0 ? F : U); + List keys = ascendingKeys(labels.size()); + + AntiCompactionRunPlanner.RunEncoding runs = AntiCompactionRunPlanner.encodeRuns(labels, keys); + assertEquals(40, runs.runCount); + assertEquals(20, runs.fullRuns); + assertEquals(20, runs.unrepairedRuns); + assertEquals(0, runs.transientRuns); + assertTrue("run detail must be dropped past the retention cap", runs.runLabels.isEmpty()); + assertTrue("run detail must be dropped past the retention cap", runs.runFirstKeys.isEmpty()); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.planFromLabels(labels, keys, session); + assertFalse(plan.eligible); + assertEquals(40, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("FULL appears in 20 runs")); + } + + /** Run detail is retained for shapes that can still turn out eligible. */ + @Test + public void runEncodingRetainsDetailForShortShapes() + { + List keys = ascendingKeys(7); + + AntiCompactionRunPlanner.RunEncoding runs = + AntiCompactionRunPlanner.encodeRuns(Arrays.asList(U, U, F, F, T, U, U), keys); + + assertEquals(4, runs.runCount); + assertEquals(1, runs.fullRuns); + assertEquals(1, runs.transientRuns); + assertEquals(2, runs.unrepairedRuns); + assertEquals(Arrays.asList(U, F, T, U), runs.runLabels); + assertEquals(Arrays.asList(keys.get(0), keys.get(2), keys.get(4), keys.get(5)), runs.runFirstKeys); + } + + @Test + public void fullThenTransientThenUnrepairedIsEligible() + { + TimeUUID session = nextTimeUUID(); + List keys = ascendingKeys(7); + + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(F, F, T, T, U, U, U), keys, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(3, plan.runCount); + assertEquals(Arrays.asList(keys.get(2), keys.get(4)), plan.boundaries); + assertEquals(Arrays.asList(pendingFull(session), pendingTransient(session), unrepaired()), plan.perChild); + } + + /** The widest eligible shape: FULL once, TRANSIENT once, UNREPAIRED leading, trailing and in between. */ + @Test + public void fiveRunShapeWithOneFullAndOneTransientIsEligible() + { + TimeUUID session = nextTimeUUID(); + List keys = ascendingKeys(10); + + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(Arrays.asList(U, U, F, F, U, U, T, T, U, U), keys, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(5, plan.runCount); + assertEquals(Arrays.asList(keys.get(2), keys.get(4), keys.get(6), keys.get(8)), plan.boundaries); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session), unrepaired(), + pendingTransient(session), unrepaired()), + plan.perChild); + } + + @Test + public void transientInTwoRunsIsIneligible() + { + TimeUUID session = nextTimeUUID(); + + AntiCompactionRunPlanner.Plan plan = planOf(session, T, T, U, U, T, T); + + assertFalse(plan.eligible); + assertEquals(3, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("TRANSIENT appears in 2 runs")); + assertTrue(plan.boundaries.isEmpty()); + assertTrue(plan.perChild.isEmpty()); + } + + /** The three triples must be exactly what {@code createWriterForAntiCompaction} is handed today. */ + @Test + public void repairStatePerLabelMatchesTheRewritePath() + { + TimeUUID session = nextTimeUUID(); + + RepairState full = AntiCompactionRunPlanner.stateFor(F, session); + assertEquals(UNREPAIRED_SSTABLE, full.repairedAt); + assertEquals(session, full.pendingRepair); + assertFalse(full.isTransient); + + RepairState trans = AntiCompactionRunPlanner.stateFor(T, session); + assertEquals(UNREPAIRED_SSTABLE, trans.repairedAt); + assertEquals(session, trans.pendingRepair); + assertTrue(trans.isTransient); + + RepairState unrepaired = AntiCompactionRunPlanner.stateFor(U, session); + assertEquals(UNREPAIRED_SSTABLE, unrepaired.repairedAt); + assertEquals(NO_PENDING_REPAIR, unrepaired.pendingRepair); + assertFalse(unrepaired.isTransient); + + assertNotEquals(full, trans); + assertNotEquals(full, unrepaired); + } + + /** + * Overlapping full and transient ranges are permitted, and full must win -- the same precedence + * {@code antiCompactGroup} applies when it routes a partition to one of its three writers. Getting this + * backwards would mark data transient on a full replica, and transient pending-repair data is DELETED rather + * than promoted when the session finalizes. + */ + @Test + public void fullWinsOverTransientForOverlappingRanges() + { + // tokens 1000, 2000, ... 6000 + List keys = ascendingKeys(6); + Range full = new Range<>(token(2500), token(4500)); // tokens 3000, 4000 + Range trans = new Range<>(token(1500), token(5500)); // tokens 2000..5000, overlapping full + RangesAtEndpoint ranges = rangesAtEndpoint(Collections.singletonList(full), + Collections.singletonList(trans)); + + assertEquals(Arrays.asList(U, T, F, F, T, U), AntiCompactionRunPlanner.labels(keys, ranges)); + + // ...and that shape is TRANSIENT-in-two-runs, so it is ineligible + AntiCompactionRunPlanner.Plan plan = + AntiCompactionRunPlanner.planFromLabels(AntiCompactionRunPlanner.labels(keys, ranges), + keys, nextTimeUUID()); + assertFalse(plan.eligible); + assertEquals(5, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("TRANSIENT appears in 2 runs")); + } + + /** No full ranges and no transient ranges at all: every partition is UNREPAIRED, and nothing blows up. */ + @Test + public void emptyRangesLabelEverythingUnrepaired() + { + List keys = ascendingKeys(4); + RangesAtEndpoint empty = rangesAtEndpoint(Collections.emptyList(), Collections.emptyList()); + + assertEquals(Arrays.asList(U, U, U, U), AntiCompactionRunPlanner.labels(keys, empty)); + } + + // ---------------------------------------------------------------------------------------------------- + // End to end: a real compressed sstable, ranges derived from its own index keys + // ---------------------------------------------------------------------------------------------------- + + @Test + public void planOnRealSSTableCutsAtTheFirstOwnedKey() throws Throwable + { + SSTableReader parent = compressedSSTable(40); + List keys = indexKeys(parent); + assertEquals(40, keys.size()); + TimeUUID session = nextTimeUUID(); + + // (token[19], token[39]] owns exactly keys 20..39 + RangesAtEndpoint ranges = fullOnly(new Range<>(keys.get(19).getToken(), keys.get(39).getToken())); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertNull(plan.ineligibleReason); + assertEquals(2, plan.runCount); + assertEquals(Arrays.asList(keys.get(20)), plan.boundaries); + assertNotEquals(keys.get(19), plan.boundaries.get(0)); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session)), plan.perChild); + } + + @Test + public void planOnRealSSTableFindsTheThreeRunStraddle() throws Throwable + { + SSTableReader parent = compressedSSTable(40); + List keys = indexKeys(parent); + TimeUUID session = nextTimeUUID(); + + // (token[9], token[24]] owns exactly keys 10..24 + RangesAtEndpoint ranges = fullOnly(new Range<>(keys.get(9).getToken(), keys.get(24).getToken())); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(3, plan.runCount); + assertEquals(Arrays.asList(keys.get(10), keys.get(25)), plan.boundaries); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session), unrepaired()), plan.perChild); + } + + @Test + public void planOnRealSSTableHandlesAFullAndATransientRun() throws Throwable + { + SSTableReader parent = compressedSSTable(40); + List keys = indexKeys(parent); + TimeUUID session = nextTimeUUID(); + + Range full = new Range<>(keys.get(9).getToken(), keys.get(19).getToken()); // keys 10..19 + Range trans = new Range<>(keys.get(19).getToken(), keys.get(29).getToken()); // keys 20..29 + RangesAtEndpoint ranges = rangesAtEndpoint(Collections.singletonList(full), + Collections.singletonList(trans)); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, session); + + assertTrue(plan.ineligibleReason, plan.eligible); + assertEquals(4, plan.runCount); + assertEquals(Arrays.asList(keys.get(10), keys.get(20), keys.get(30)), plan.boundaries); + assertEquals(Arrays.asList(unrepaired(), pendingFull(session), pendingTransient(session), unrepaired()), + plan.perChild); + } + + @Test + public void planOnRealSSTableRejectsInterleavedFullRanges() throws Throwable + { + SSTableReader parent = compressedSSTable(40); + List keys = indexKeys(parent); + + // two owned islands with unowned partitions between them: U F U F U + Range firstIsland = new Range<>(keys.get(4).getToken(), keys.get(9).getToken()); // keys 5..9 + Range secondIsland = new Range<>(keys.get(19).getToken(), keys.get(24).getToken()); // keys 20..24 + RangesAtEndpoint ranges = rangesAtEndpoint(Arrays.asList(firstIsland, secondIsland), + Collections.emptyList()); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, nextTimeUUID()); + + assertFalse(plan.eligible); + assertEquals(5, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("FULL appears in 2 runs")); + assertTrue(plan.boundaries.isEmpty()); + assertTrue(plan.perChild.isEmpty()); + } + + @Test + public void planOnRealSSTableRejectsAFullyOwnedSSTable() throws Throwable + { + SSTableReader parent = compressedSSTable(20); + List keys = indexKeys(parent); + + RangesAtEndpoint ranges = fullOnly(new Range<>(parent.getPartitioner().getMinimumToken(), + keys.get(keys.size() - 1).getToken())); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, nextTimeUUID()); + + assertFalse(plan.eligible); + assertEquals(1, plan.runCount); + assertTrue(plan.ineligibleReason, plan.ineligibleReason.contains("entire sstable is FULL")); + } + + /** + * An uncompressed sstable must be REPORTED ineligible, not throw: {@code plan} is called on every sstable of + * every anticompaction group, and a throw there would fail repairs on any uncompressed table. + */ + @Test + public void planReportsUncompressedSSTableAsIneligible() throws Throwable + { + createTable("CREATE TABLE %s (pk text, ck int, val text, PRIMARY KEY (pk, ck)) " + + "WITH compression = {'enabled': 'false'}"); + disableCompaction(); + for (int p = 0; p < 10; p++) + execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, ?)", key(p), 0, "v"); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + assertEquals(1, cfs.getLiveSSTables().size()); + SSTableReader parent = cfs.getLiveSSTables().iterator().next(); + assertFalse(parent.compression); + assertFalse(ZeroCopySSTableSplitter.isSupported(parent)); + + List keys = indexKeys(parent); + RangesAtEndpoint ranges = fullOnly(new Range<>(keys.get(4).getToken(), keys.get(9).getToken())); + + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, nextTimeUUID()); + + assertFalse(plan.eligible); + assertEquals(0, plan.runCount); + assertTrue(plan.ineligibleReason, + plan.ineligibleReason.contains("not a compressed BIG-format sstable")); + assertTrue(plan.boundaries.isEmpty()); + assertTrue(plan.perChild.isEmpty()); + } + + // ---------------------------------------------------------------------------------------------------- + // Helpers + // ---------------------------------------------------------------------------------------------------- + + private static AntiCompactionRunPlanner.Plan planOf(TimeUUID session, AntiCompactionRunPlanner.Label... labels) + { + return AntiCompactionRunPlanner.planFromLabels(Arrays.asList(labels), ascendingKeys(labels.length), session); + } + + /** {@code count} distinct keys with strictly ascending tokens 1000, 2000, ... */ + private static List ascendingKeys(int count) + { + List keys = new ArrayList<>(count); + for (int i = 0; i < count; i++) + keys.add(new BufferDecoratedKey(token(1000L * (i + 1)), ByteBufferUtil.bytes(String.format("k%06d", i)))); + return keys; + } + + private static Token token(long value) + { + return new Murmur3Partitioner.LongToken(value); + } + + private static RepairState unrepaired() + { + return new RepairState(UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false); + } + + private static RepairState pendingFull(TimeUUID session) + { + return new RepairState(UNREPAIRED_SSTABLE, session, false); + } + + private static RepairState pendingTransient(TimeUUID session) + { + return new RepairState(UNREPAIRED_SSTABLE, session, true); + } + + private static RangesAtEndpoint fullOnly(Range range) + { + return rangesAtEndpoint(Collections.singletonList(range), Collections.emptyList()); + } + + private static RangesAtEndpoint rangesAtEndpoint(List> full, List> trans) + { + InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local); + for (Range range : full) + builder.add(Replica.fullReplica(local, range)); + for (Range range : trans) + builder.add(Replica.transientReplica(local, range)); + return builder.build(); + } + + /** + * One compressed sstable holding exactly {@code partitions} partitions. Compression is what makes the + * splitter applicable at all, so a SchemaLoader-style uncompressed table could never reach the eligible + * path. + */ + private SSTableReader compressedSSTable(int partitions) throws Throwable + { + createTable("CREATE TABLE %s (pk text, ck int, val text, PRIMARY KEY (pk, ck)) " + + "WITH compression = {'class': 'LZ4Compressor', 'chunk_length_in_kb': '4'}"); + disableCompaction(); + for (int p = 0; p < partitions; p++) + execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, ?)", key(p), 0, "value"); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + assertEquals(1, cfs.getLiveSSTables().size()); + SSTableReader parent = cfs.getLiveSSTables().iterator().next(); + assertTrue(parent.compression); + assertTrue(ZeroCopySSTableSplitter.isSupported(parent)); + return parent; + } + + /** The sstable's partition keys in on-disk (token) order -- the exact sequence the planner walks. */ + private static List indexKeys(SSTableReader sstable) + { + List keys = new ArrayList<>(); + try (KeyIterator it = new KeyIterator(sstable.descriptor, sstable.metadata())) + { + while (it.hasNext()) + keys.add(it.next()); + } + for (int i = 1; i < keys.size(); i++) + assertTrue("index keys must be strictly ascending", keys.get(i - 1).compareTo(keys.get(i)) < 0); + return keys; + } + + private static String key(int p) + { + return String.format("k%06d", p); + } +} diff --git a/test/unit/org/apache/cassandra/db/compaction/ZeroCopyAntiCompactionFuzzTest.java b/test/unit/org/apache/cassandra/db/compaction/ZeroCopyAntiCompactionFuzzTest.java new file mode 100644 index 000000000000..ef8ea1510d82 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/compaction/ZeroCopyAntiCompactionFuzzTest.java @@ -0,0 +1,925 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.compaction; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import com.google.common.collect.Lists; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.Util; +import org.apache.cassandra.config.Config.FlushCompression; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.Refs; + +import static org.apache.cassandra.service.ActiveRepairService.NO_PENDING_REPAIR; +import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABLE; +import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Randomised test of anticompaction over random range sets, covering both the + * {@link ZeroCopySSTableSplitter}-based path (chosen by {@link AntiCompactionRunPlanner}) and the unchanged + * three-rewriter fallback. Every iteration builds one compressed sstable, invents a full/transient range set, + * runs {@code performAnticompaction} over it, and applies one oracle. + * + *

The oracle

+ * Whichever path ran: + *
    + *
  1. every parent partition appears exactly once across the resulting sstables, with byte-for-byte the + * same content (compared as the fully-detailed rendering of the partition-level deletion, the static row + * and every unfiltered); and
    2. + *
  2. each partition's {@code pendingRepair} / {@code isTransient} is exactly what its token implies from the + * range set -- FULL to {@code (sessionID, false)}, TRANSIENT to {@code (sessionID, true)}, everything else + * to {@code (null, false)} -- and nothing is ever marked repaired.
    3. + *
+ * That is path-agnostic on purpose, so the test stays valid whichever side of the gate an iteration lands on. On + * top of it the gate itself is cross-checked: labels, run count, boundaries and per-child repair state are all + * recomputed here from {@code Range.contains} alone and the planner's verdict must agree; the + * {@code BytesZeroCopyAnticompaction} meter must move iff the verdict was "eligible"; and the output sstable count + * must be the run count when the split ran and the number of distinct labels when the rewrite ran. The + * recomputation deliberately uses the naive scan rather than the {@code OrderedRangeContainmentChecker} both + * production paths share, so agreement means something. + * + *

Why the generator writes no tombstones and never overwrites

+ * The fallback path pushes every partition through a {@link CompactionController} and so legitimately drops data + * shadowed by a deletion, where the zero-copy path copies chunks verbatim and retains it. One content oracle can + * only hold for both if the data contains nothing either path may change: hence one INSERT per {@code (pk, ck)}, + * no DELETEs, no nulls, no TTLs, and an explicit {@code gc_grace_seconds}. Tombstone retention is + * {@code ZeroCopySSTableSplitterFuzzTest}'s job; the subject here is routing and completeness. + * + *

What is randomised

+ * The compressor and {@code chunk_length_in_kb}, {@code column_index_size} (so wide partitions really do carry a + * promoted index the planner's Index.db walk has to skip), the partition count, and wide versus narrow partitions. + * The range set is randomised by shape: the whole sstable, a prefix, a suffix, a middle span, exactly one + * partition, an endpoint landing exactly on the first or last partition's token, ranges covering no partition at + * all (in a token gap, or entirely below or above the sstable's span), a full range abutting a transient one, a + * transient range nested in a full one and vice versa, several disjoint full ranges, and a vnode-like alternating + * layout with more runs than the planner retains detail for. Iteration {@code i} always gets shape + * {@code i % shapes}, encoded in the low digits of that iteration's seed so a seed alone still replays it. + * + *

Reproducing a failure

+ * Every assertion message carries the whole configuration, range set included, plus the iteration's seed. A bare + * {@code -Dfoo=bar} on the ant command line does not reach the forked test JVM, so these must go through + * {@code -Dtest.jvm.args}: + * 
+ *   ant testsome -Duse.jdk11=true \
+ *       -Dtest.name=org.apache.cassandra.db.compaction.ZeroCopyAntiCompactionFuzzTest \
+ *       -Dtest.methods=fuzzRangeSets \
+ *       -Dtest.jvm.args="-Dcassandra.test.zcanticompaction.replaySeed=<seed from the failure message>"
+ *
+ * For a longer soak, raise the iteration count and/or move the base seed: + * 
+ *   -Dtest.jvm.args="-Dcassandra.test.zcanticompaction.iterations=160 -Dcassandra.test.zcanticompaction.seed=99"
+ *
+ * The default of one iteration per shape is deliberately modest so this stays inside a normal unit-test run. + */ +public class ZeroCopyAntiCompactionFuzzTest extends CQLTester +{ + private static final Logger logger = LoggerFactory.getLogger(ZeroCopyAntiCompactionFuzzTest.class); + + private static final String PROP_SEED = "cassandra.test.zcanticompaction.seed"; + private static final String PROP_ITERATIONS = "cassandra.test.zcanticompaction.iterations"; + private static final String PROP_REPLAY_SEED = "cassandra.test.zcanticompaction.replaySeed"; + + /** How the full/transient ranges are placed relative to the sstable's token span. */ + private enum Shape + { + /** Every partition FULL; also the shape that makes {@code mutateFullyContainedSSTables} take over. */ + COVER_ALL, + /** FULL over a token prefix: {@code F U}. */ + PREFIX, + /** FULL over a token suffix: {@code U F}. */ + SUFFIX, + /** FULL over a middle span: {@code U F U}. */ + MIDDLE, + /** FULL over exactly one partition, both endpoints on real partition tokens: {@code U F U}. */ + SINGLE_PARTITION, + /** Right endpoint exactly on the first partition's token: {@code F U}. */ + TOUCH_FIRST, + /** Left endpoint exactly on the second-to-last partition's token: {@code U F}. */ + TOUCH_LAST, + /** A range strictly inside a gap between two adjacent tokens: covers nothing. */ + GAP_MISS, + /** A range entirely below the sstable's token span: covers nothing. */ + BELOW_MISS, + /** A range entirely above the sstable's token span: covers nothing. */ + ABOVE_MISS, + /** FULL immediately followed by TRANSIENT: {@code U F T U}, the widest eligible shape. */ + FULL_THEN_TRANSIENT, + /** TRANSIENT over a middle span, no full ranges at all: {@code U T U}. */ + TRANSIENT_MIDDLE, + /** A TRANSIENT range nested inside a FULL one: full wins, so still {@code U F U}. */ + TRANSIENT_INSIDE_FULL, + /** A FULL range nested inside a TRANSIENT one: {@code U T F T U}, so TRANSIENT is not contiguous. */ + FULL_INSIDE_TRANSIENT, + /** Two disjoint FULL ranges with unrepaired partitions between them. */ + INTERLEAVED_FULL, + /** Alternating single-partition FULL ranges: the vnode layout, with more runs than are retained. */ + VNODE + } + + private static final Shape[] SHAPES = Shape.values(); + + private static final long BASE_SEED = Long.getLong(PROP_SEED, 20260727_0001L); + private static final int ITERATIONS = Integer.getInteger(PROP_ITERATIONS, SHAPES.length); + /** When set, exactly one iteration runs, with this literal seed. */ + private static final Long REPLAY_SEED = Long.getLong(PROP_REPLAY_SEED); + + /** Explicit insert timestamps keep the on-disk layout stable across runs of the same seed. */ + private static final long BASE_TS = 1_600_000_000_000_000L; + + private static final String[] COMPRESSORS = { "LZ4Compressor", "SnappyCompressor", + "DeflateCompressor", "ZstdCompressor" }; + private static final int[] CHUNK_KB = { 4, 8, 16, 32, 64 }; + /** Small values force a promoted index into Index.db, which the planner's walk has to skip over. */ + private static final int[] COLUMN_INDEX_KB = { 1, 2, 4, 64 }; + private static final int[] COLUMN_INDEX_CACHE_KB = { 0, 2, 99999 }; + + /** Enough distinct tokens for every shape's index arithmetic to have room. */ + private static final int MIN_DISTINCT_TOKENS = 8; + + private int eligibleIterations; + private int ineligibleIterations; + private final Map verdictByShape = new TreeMap<>(); + + @Test + public void fuzzRangeSets() throws Throwable + { + int savedIndexSize = DatabaseDescriptor.getColumnIndexSizeInKiB(); + int savedCacheSize = DatabaseDescriptor.getColumnIndexCacheSizeInKiB(); + FlushCompression savedFlushCompression = DatabaseDescriptor.getFlushCompression(); + boolean savedZeroCopy = DatabaseDescriptor.getZeroCopyAnticompactionEnabled(); + try + { + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(true); + + if (REPLAY_SEED != null) + { + logger.info("Replaying a single zero-copy anticompaction fuzz iteration, seed {}", REPLAY_SEED); + runGuarded(REPLAY_SEED); + } + else + { + logger.info("Zero-copy anticompaction fuzz: {} iterations from base seed {} over {} range shapes", + ITERATIONS, BASE_SEED, SHAPES.length); + for (int i = 0; i < ITERATIONS; i++) + runGuarded(seedForIteration(i)); + } + } + finally + { + DatabaseDescriptor.setColumnIndexSize(savedIndexSize); + DatabaseDescriptor.setColumnIndexCacheSize(savedCacheSize); + DatabaseDescriptor.setFlushCompression(savedFlushCompression); + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(savedZeroCopy); + } + + logger.info("Zero-copy anticompaction fuzz done: {} iterations took the zero-copy split, {} fell back to " + + "the rewrite path. Per shape: {}", eligibleIterations, ineligibleIterations, verdictByShape); + + // Without at least one eligible iteration the oracle above would only ever have exercised the + // pre-existing rewrite path, i.e. this test would be vacuous with respect to the feature it covers. + assertTrue("no iteration reached the zero-copy split path (per shape: " + verdictByShape + "); this fuzz " + + "is no longer testing the feature it exists for", eligibleIterations > 0); + assertEquals("some iteration was never classified as zero-copy or fallback", + REPLAY_SEED != null ? 1 : ITERATIONS, eligibleIterations + ineligibleIterations); + } + + /** + * Iteration {@code i} always gets shape {@code i % SHAPES.length}, but the shape is derived from the + * seed rather than from {@code i}, so quoting the seed back is enough to replay the whole iteration. + */ + private static long seedForIteration(int i) + { + long base = scramble(BASE_SEED + i) >>> 8; // non-negative, and still far apart between iterations + return base * SHAPES.length + (i % SHAPES.length); + } + + private static Shape shapeForSeed(long seed) + { + return SHAPES[(int) Math.floorMod(seed, (long) SHAPES.length)]; + } + + private void runGuarded(long seed) throws Throwable + { + Config cfg = new Config(seed); + try + { + runIteration(cfg); + } + catch (Throwable t) + { + throw new AssertionError("zero-copy anticompaction fuzz iteration FAILED\n" + cfg + '\n' + + replayHint(seed), t); + } + } + + private static String replayHint(long seed) + { + // a bare -D does not reach the forked test JVM, hence -Dtest.jvm.args + return "replay this case alone with:\n" + + " ant testsome -Duse.jdk11=true" + + " -Dtest.name=org.apache.cassandra.db.compaction.ZeroCopyAntiCompactionFuzzTest" + + " -Dtest.methods=fuzzRangeSets" + + " -Dtest.jvm.args=\"-D" + PROP_REPLAY_SEED + '=' + seed + '"'; + } + + // ------------------------------------------------------------------------------------------------ + // One iteration + // ------------------------------------------------------------------------------------------------ + + private void runIteration(Config cfg) throws Throwable + { + Random rnd = new Random(cfg.seed); + + cfg.shape = shapeForSeed(cfg.seed); + cfg.compressor = COMPRESSORS[rnd.nextInt(COMPRESSORS.length)]; + cfg.chunkKb = CHUNK_KB[rnd.nextInt(CHUNK_KB.length)]; + cfg.columnIndexKb = COLUMN_INDEX_KB[rnd.nextInt(COLUMN_INDEX_KB.length)]; + cfg.columnIndexCacheKb = COLUMN_INDEX_CACHE_KB[rnd.nextInt(COLUMN_INDEX_CACHE_KB.length)]; + cfg.wide = rnd.nextBoolean(); + cfg.partitions = 16 + rnd.nextInt(25); + cfg.totalBytes = 300_000 + rnd.nextInt(400_000); + + long perPartition = Math.max(200, cfg.totalBytes / cfg.partitions); + cfg.rowsPerPartition = cfg.wide ? 6 + rnd.nextInt(19) : 1 + rnd.nextInt(2); + cfg.valueBytes = (int) Math.min(24_000, Math.max(48, perPartition / cfg.rowsPerPartition)); + + DatabaseDescriptor.setColumnIndexSize(cfg.columnIndexKb); + DatabaseDescriptor.setColumnIndexCacheSize(cfg.columnIndexCacheKb); + // flush_compression defaults to `fast`, which silently replaces any compressor that does not advertise + // FAST_COMPRESSION with CompressionParams.DEFAULT -- LZ4 at 16 KiB. Without this, every non-LZ4 + // iteration below would quietly test the same single configuration. + DatabaseDescriptor.setFlushCompression(FlushCompression.table); + + createTable(ddl(cfg)); + disableCompaction(); + writeData(cfg, rnd); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + TableMetadata metadata = cfs.metadata(); + assertTrue("this test builds its ranges out of Murmur3 long tokens; CQLTester is supposed to force " + + "Murmur3Partitioner but the table is on " + cfs.getPartitioner(), + cfs.getPartitioner() instanceof Murmur3Partitioner); + + Set live = cfs.getLiveSSTables(); + assertEquals("expected exactly one sstable after the flush, got " + live, 1, live.size()); + SSTableReader parent = live.iterator().next(); + + assertTrue("the generator produced an uncompressed sstable, so the whole compressor matrix is void", + parent.compression); + assertTrue("a compressed BIG sstable must be splittable", ZeroCopySSTableSplitter.isSupported(parent)); + // If flush_compression silently downgraded the table's compression this is where it shows up. + assertEquals("the table's chunk_length_in_kb did not survive to the sstable; flush_compression has " + + "replaced the requested compressor and this iteration would test nothing", + cfg.chunkKb * 1024, parent.getCompressionMetadata().chunkLength()); + assertEquals("the table's compressor did not survive to the sstable; flush_compression has replaced it " + + "and this iteration would test nothing", + cfg.compressor, + parent.getCompressionMetadata().parameters.getSstableCompressor().getClass().getSimpleName()); + + // ---- the parent, as read before anything touches it: the oracle's left-hand side ---- + Map before = new HashMap<>(); + List keysInOrder = new ArrayList<>(); + readPartitions(parent, metadata, before, keysInOrder); + assertEquals("the generator did not write one partition per pk", cfg.partitions, keysInOrder.size()); + cfg.parentPartitions = keysInOrder.size(); + + List distinctTokens = new ArrayList<>(new TreeSet<>(tokensOf(keysInOrder))); + assertTrue("only " + distinctTokens.size() + " distinct tokens; the range shapes need at least " + + MIN_DISTINCT_TOKENS, distinctTokens.size() >= MIN_DISTINCT_TOKENS); + + // ---- the range set ---- + TimeUUID sessionID = nextTimeUUID(); + Set> fullRanges = new LinkedHashSet<>(); + Set> transientRanges = new LinkedHashSet<>(); + buildRanges(cfg.shape, rnd, distinctTokens, fullRanges, transientRanges); + transientRanges.removeAll(fullRanges); // RangesAtEndpoint.Builder rejects one range being both + ensureIntersectsSSTable(parent, distinctTokens, fullRanges, transientRanges); + cfg.fullRanges = fullRanges.toString(); + cfg.transientRanges = transientRanges.toString(); + + InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local); + for (Range range : fullRanges) + builder.add(new Replica(local, range, true)); + for (Range range : transientRanges) + builder.add(new Replica(local, range, false)); + RangesAtEndpoint ranges = builder.build(); + + // ---- what the range set implies, computed here from Range.contains alone ---- + List expectedLabels = new ArrayList<>(keysInOrder.size()); + for (DecoratedKey key : keysInOrder) + expectedLabels.add(labelOf(key.getToken(), fullRanges, transientRanges)); + + List runStarts = runStarts(expectedLabels); + int expectedRunCount = runStarts.size(); + int fullRuns = countRuns(expectedLabels, runStarts, AntiCompactionRunPlanner.Label.FULL); + int transientRuns = countRuns(expectedLabels, runStarts, AntiCompactionRunPlanner.Label.TRANSIENT); + boolean expectedEligible = expectedRunCount >= 2 && fullRuns <= 1 && transientRuns <= 1; + Set labelsPresent = new TreeSet<>(expectedLabels); + + cfg.labels = expectedLabels.toString(); + cfg.runCount = expectedRunCount; + cfg.expectedEligible = expectedEligible; + verdictByShape.put(cfg.shape.name(), (expectedEligible ? "zero-copy" : "fallback") + + " runs=" + expectedRunCount); + + // ---- the planner must agree, on every detail the split depends on ---- + AntiCompactionRunPlanner.Plan plan = AntiCompactionRunPlanner.plan(parent, ranges, sessionID); + assertEquals(cfg + " -- planner disagrees about eligibility (" + plan + ')', + expectedEligible, plan.eligible); + assertEquals(cfg + " -- planner counted the wrong number of runs (" + plan + ')', + expectedRunCount, plan.runCount); + if (expectedEligible) + { + List expectedBoundaries = new ArrayList<>(); + for (int b = 1; b < runStarts.size(); b++) + expectedBoundaries.add(keysInOrder.get(runStarts.get(b))); + assertEquals(cfg + " -- wrong split boundaries", expectedBoundaries, plan.boundaries); + + List expectedStates = new ArrayList<>(); + for (int start : runStarts) + expectedStates.add(expectedState(expectedLabels.get(start), sessionID)); + assertEquals(cfg + " -- wrong per-child repair state", expectedStates, plan.perChild); + } + else + { + assertNotNull(cfg + " -- an ineligible plan must say why", plan.ineligibleReason); + } + + // ---- run the anticompaction through the real public entry point ---- + long zcBytesBefore = cfs.metric.bytesZeroCopyAnticompaction.getCount(); + try + { + ActiveRepairService.instance.registerParentRepairSession(sessionID, local, Lists.newArrayList(cfs), + ranges.ranges(), true, UNREPAIRED_SSTABLE, + true, PreviewKind.NONE); + Set sstables = new HashSet<>(live); + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); + Refs refs = Refs.ref(sstables)) + { + assertNotNull(cfg + " -- could not mark the sstable compacting", txn); + CompactionManager.instance.performAnticompaction(cfs, ranges, refs, txn, sessionID, () -> false); + } + } + finally + { + ActiveRepairService.instance.removeParentRepairSession(sessionID); + } + long zcBytesAfter = cfs.metric.bytesZeroCopyAnticompaction.getCount(); + + // ---- which path actually ran ---- + if (expectedEligible) + { + eligibleIterations++; + assertTrue(cfg + " -- the plan was eligible but BytesZeroCopyAnticompaction did not move (" + + zcBytesBefore + " -> " + zcBytesAfter + "), so the rewrite path ran instead", + zcBytesAfter > zcBytesBefore); + } + else + { + ineligibleIterations++; + assertEquals(cfg + " -- the plan was ineligible but BytesZeroCopyAnticompaction moved, so the gate " + + "let a zero-copy split through", zcBytesBefore, zcBytesAfter); + } + + // The split produces exactly one sstable per run; the rewrite produces exactly one per non-empty + // destination, i.e. one per label that any partition carries. (The whole-sstable-FULL case that + // mutateFullyContainedSSTables serves is a single label too, so it needs no special case here.) + int expectedOutputs = expectedEligible ? expectedRunCount : labelsPresent.size(); + Util.assertOnDiskState(cfs, expectedOutputs); + + // An sstable whose whole token span sits inside one range never gets rewritten or split at all: + // mutateFullyContainedSSTables rewrites its metadata in place and drops it from the transaction. + boolean fullyContained = fullyContained(parent, fullRanges) || fullyContained(parent, transientRanges); + if (fullyContained) + { + assertEquals(cfg + " -- every partition of a fully contained sstable must carry the same label, " + + "otherwise the in-place metadata mutation mislabels some of them", + 1, labelsPresent.size()); + assertTrue(cfg + " -- a fully contained sstable must be kept and only have its metadata mutated", + cfs.getLiveSSTables().contains(parent)); + } + else + { + assertFalse(cfg + " -- the parent was not obsoleted; its data now exists twice", + cfs.getLiveSSTables().contains(parent)); + } + + // ---- THE ORACLE ---- + Map after = new HashMap<>(); + Map expectedByKey = new HashMap<>(); + for (int i = 0; i < keysInOrder.size(); i++) + expectedByKey.put(hex(keysInOrder.get(i)), expectedLabels.get(i)); + + for (SSTableReader output : cfs.getLiveSSTables()) + { + assertFalse(cfg + " -- " + output.descriptor + " was marked repaired; anticompaction only ever " + + "produces pending-repair or unrepaired sstables", output.isRepaired()); + + List outputKeys = new ArrayList<>(); + Map outputPartitions = new HashMap<>(); + readPartitions(output, metadata, outputPartitions, outputKeys); + assertFalse(cfg + " -- " + output.descriptor + " is empty", outputKeys.isEmpty()); + + for (DecoratedKey key : outputKeys) + { + String hex = hex(key); + AntiCompactionRunPlanner.Label expected = expectedByKey.get(hex); + assertNotNull(cfg + " -- " + output.descriptor + " holds a partition the parent never had: " + key, + expected); + + // (2) the repair state of the sstable a partition landed in must match its token's label + ZeroCopySSTableSplitter.RepairState want = expectedState(expected, sessionID); + assertEquals(cfg + " -- " + key + " is labelled " + expected + " but landed in " + + output.descriptor + " with pendingRepair=" + output.getPendingRepair(), + want.pendingRepair, output.getPendingRepair()); + assertEquals(cfg + " -- " + key + " is labelled " + expected + " but landed in " + + output.descriptor + " with isTransient=" + output.isTransient(), + want.isTransient, output.isTransient()); + + // (1a) exactly once + String previous = after.put(hex, outputPartitions.get(hex)); + assertNull(cfg + " -- " + key + " appears in more than one output sstable (again in " + + output.descriptor + ')', previous); + } + } + + // (1b) all of them, content-identical + assertEquals(cfg + " -- the outputs do not hold exactly the parent's partitions", + new TreeSet<>(before.keySet()), new TreeSet<>(after.keySet())); + for (Map.Entry entry : before.entrySet()) + assertEquals(cfg + " -- partition " + entry.getKey() + " changed content", + entry.getValue(), after.get(entry.getKey())); + + for (SSTableReader output : cfs.getLiveSSTables()) + { + assertFalse(cfg + " -- " + output.descriptor + " is marked compacted", output.isMarkedCompacted()); + assertEquals(cfg + " -- leaked reference on " + output.descriptor, + 1, output.selfRef().globalCount()); + } + assertEquals(cfg + " -- sstables left marked compacting", 0, cfs.getTracker().getCompacting().size()); + } + + // ------------------------------------------------------------------------------------------------ + // The independent labelling / run oracle + // ------------------------------------------------------------------------------------------------ + + /** + * Deliberately the naive form -- a linear scan of the raw ranges with {@code Range.contains} -- + * so it shares nothing with the production {@code OrderedRangeContainmentChecker} (normalize plus a + * forward-only cursor) that both the planner and the rewrite path use. Full wins over transient, which is + * the precedence {@code antiCompactGroup} routes by. + */ + private static AntiCompactionRunPlanner.Label labelOf(Token token, + Collection> full, + Collection> trans) + { + for (Range range : full) + if (range.contains(token)) + return AntiCompactionRunPlanner.Label.FULL; + for (Range range : trans) + if (range.contains(token)) + return AntiCompactionRunPlanner.Label.TRANSIENT; + return AntiCompactionRunPlanner.Label.UNREPAIRED; + } + + /** The index of the first partition of each contiguous run of identical labels. */ + private static List runStarts(List labels) + { + List starts = new ArrayList<>(); + AntiCompactionRunPlanner.Label previous = null; + for (int i = 0; i < labels.size(); i++) + { + if (labels.get(i) != previous) + { + starts.add(i); + previous = labels.get(i); + } + } + return starts; + } + + private static int countRuns(List labels, + List runStarts, + AntiCompactionRunPlanner.Label label) + { + int count = 0; + for (int start : runStarts) + if (labels.get(start) == label) + count++; + return count; + } + + /** The triples {@code createWriterForAntiCompaction} is called with, spelled out rather than delegated. */ + private static ZeroCopySSTableSplitter.RepairState expectedState(AntiCompactionRunPlanner.Label label, + TimeUUID sessionID) + { + switch (label) + { + case FULL: + return new ZeroCopySSTableSplitter.RepairState(UNREPAIRED_SSTABLE, sessionID, false); + case TRANSIENT: + return new ZeroCopySSTableSplitter.RepairState(UNREPAIRED_SSTABLE, sessionID, true); + default: + return new ZeroCopySSTableSplitter.RepairState(UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false); + } + } + + // ------------------------------------------------------------------------------------------------ + // Range-set generation + // ------------------------------------------------------------------------------------------------ + + private static void buildRanges(Shape shape, + Random rnd, + List tokens, + Set> full, + Set> trans) + { + int n = tokens.size(); + switch (shape) + { + case COVER_ALL: + full.add(range(Long.MIN_VALUE, tokens.get(n - 1))); + break; + case PREFIX: + full.add(range(Long.MIN_VALUE, tokens.get(pick(rnd, 0, n - 3)))); + break; + case SUFFIX: + full.add(range(tokens.get(pick(rnd, 0, n - 2)), Long.MAX_VALUE)); + break; + case MIDDLE: + { + int i = pick(rnd, 0, n - 4); + int j = pick(rnd, i + 1, n - 2); + full.add(range(tokens.get(i), tokens.get(j))); + break; + } + case SINGLE_PARTITION: + { + int i = pick(rnd, 1, n - 2); + full.add(range(tokens.get(i - 1), tokens.get(i))); + break; + } + case TOUCH_FIRST: + full.add(range(Long.MIN_VALUE, tokens.get(0))); + break; + case TOUCH_LAST: + full.add(range(tokens.get(n - 2), tokens.get(n - 1))); + break; + case GAP_MISS: + full.add(gapRange(tokens)); + break; + case BELOW_MISS: + // strictly below every partition; ensureIntersectsSSTable() adds what validation needs + full.add(range(Long.MIN_VALUE, tokens.get(0) - 1)); + break; + case ABOVE_MISS: + full.add(range(tokens.get(n - 1), Long.MAX_VALUE)); + break; + case FULL_THEN_TRANSIENT: + { + int i = pick(rnd, 0, n - 4); + int j = pick(rnd, i + 1, n - 3); + int k = pick(rnd, j + 1, n - 2); + full.add(range(tokens.get(i), tokens.get(j))); + trans.add(range(tokens.get(j), tokens.get(k))); + break; + } + case TRANSIENT_MIDDLE: + { + int i = pick(rnd, 0, n - 4); + int j = pick(rnd, i + 1, n - 2); + trans.add(range(tokens.get(i), tokens.get(j))); + break; + } + case TRANSIENT_INSIDE_FULL: + { + int i = pick(rnd, 0, n - 5); + int j = pick(rnd, i + 3, n - 2); + full.add(range(tokens.get(i), tokens.get(j))); + trans.add(range(tokens.get(i + 1), tokens.get(j - 1))); + break; + } + case FULL_INSIDE_TRANSIENT: + { + int i = pick(rnd, 0, n - 5); + int j = pick(rnd, i + 3, n - 2); + trans.add(range(tokens.get(i), tokens.get(j))); + full.add(range(tokens.get(i + 1), tokens.get(j - 1))); + break; + } + case INTERLEAVED_FULL: + { + int a = pick(rnd, 0, n - 5); + int b = pick(rnd, a + 1, n - 4); + int c = pick(rnd, b + 1, n - 3); + int d = pick(rnd, c + 1, n - 2); + full.add(range(tokens.get(a), tokens.get(b))); + full.add(range(tokens.get(c), tokens.get(d))); + break; + } + case VNODE: + for (int i = 0; i + 1 <= n - 2 && full.size() < 6; i += 2) + full.add(range(tokens.get(i), tokens.get(i + 1))); + break; + default: + throw new AssertionError("unhandled shape " + shape); + } + } + + /** + * {@code validateSSTableBoundsForAnticompaction} (CompactionManager) throws outright if no range even + * intersects the sstable's bounds, which the deliberately-missing shapes would otherwise trip. Add a range + * that lives strictly inside a gap between two adjacent tokens: it satisfies validation without covering a + * single partition, so the labelling -- and therefore the oracle -- is untouched. + */ + private static void ensureIntersectsSSTable(SSTableReader parent, + List tokens, + Set> full, + Set> trans) + { + if (intersectsBounds(parent, full, trans)) + return; + + Range gap = gapRange(tokens); + if (full.isEmpty()) + trans.add(gap); + else + full.add(gap); + assertTrue("could not make the range set intersect the sstable bounds even with the gap range " + gap, + intersectsBounds(parent, full, trans)); + } + + /** The {@code findSSTablesToAnticompact} predicate: is the whole token span inside a single range? */ + private static boolean fullyContained(SSTableReader parent, Set> ranges) + { + if (ranges.isEmpty()) + return false; + Token first = parent.first.getToken(); + Token last = parent.last.getToken(); + for (Range range : Range.normalize(ranges)) + { + if (range.contains(first) && range.contains(last)) + return true; + } + return false; + } + + private static boolean intersectsBounds(SSTableReader parent, + Set> full, + Set> trans) + { + List> all = new ArrayList<>(full); + all.addAll(trans); + Bounds bounds = new Bounds<>(parent.first.getToken(), parent.last.getToken()); + for (Range range : Range.normalize(all)) + { + if ((range.contains(bounds.left) && range.contains(bounds.right)) || range.intersects(bounds)) + return true; + } + return false; + } + + /** A range strictly between two adjacent partition tokens, so it can never contain a partition. */ + private static Range gapRange(List tokens) + { + for (int i = 0; i + 1 < tokens.size(); i++) + { + if (tokens.get(i + 1) - tokens.get(i) >= 2) + return range(tokens.get(i), tokens.get(i + 1) - 1); + } + throw new AssertionError("no gap of two or more between any adjacent tokens: " + tokens); + } + + private static Range range(long left, long right) + { + assertTrue("refusing to build the wraparound/full-ring range (" + left + ", " + right + ']', left < right); + return new Range<>(new Murmur3Partitioner.LongToken(left), new Murmur3Partitioner.LongToken(right)); + } + + private static int pick(Random rnd, int lo, int hi) + { + assertTrue("empty index range [" + lo + ',' + hi + "]; the shape needs more distinct tokens", lo <= hi); + return lo + rnd.nextInt(hi - lo + 1); + } + + // ------------------------------------------------------------------------------------------------ + // Schema, data and reading + // ------------------------------------------------------------------------------------------------ + + private static String ddl(Config cfg) + { + return "CREATE TABLE %s (pk text, ck int, v blob, s text static, PRIMARY KEY (pk, ck))" + + " WITH compression = {'class': '" + cfg.compressor + + "', 'chunk_length_in_kb': " + cfg.chunkKb + '}' + // far in the future, so no tombstone this test writes could ever become droppable and let the + // rewrite path legitimately diverge from the copy path + + " AND gc_grace_seconds = 864000"; + } + + /** + * One INSERT per {@code (pk, ck)} and one per static row, all at distinct explicit timestamps: nothing is + * overwritten, nothing is deleted, so neither anticompaction path is permitted to change the content. + */ + private void writeData(Config cfg, Random rnd) throws Throwable + { + long ts = BASE_TS; + for (int p = 0; p < cfg.partitions; p++) + { + String pk = String.format("p%05d", p); + for (int r = 0; r < cfg.rowsPerPartition; r++) + execute("INSERT INTO %s (pk, ck, v) VALUES (?, ?, ?) USING TIMESTAMP ?", + pk, r, blob(rnd, cfg.valueBytes, rnd.nextInt(4) == 0), ts++); + if (rnd.nextBoolean()) + execute("INSERT INTO %s (pk, s) VALUES (?, ?) USING TIMESTAMP ?", pk, text(rnd, 24), ts++); + } + } + + private static ByteBuffer blob(Random rnd, int size, boolean compressible) + { + byte[] bytes = new byte[size]; + if (compressible) + { + Arrays.fill(bytes, (byte) ('a' + rnd.nextInt(26))); + for (int i = 0; i < bytes.length; i += 512) + bytes[i] = (byte) rnd.nextInt(); + } + else + { + rnd.nextBytes(bytes); // near-incompressible, so the sstable really spans many chunks + } + return ByteBuffer.wrap(bytes); + } + + private static String text(Random rnd, int length) + { + char[] chars = new char[length]; + for (int i = 0; i < length; i++) + chars[i] = (char) ('a' + rnd.nextInt(26)); + return new String(chars); + } + + /** + * Reads every partition of {@code sstable}, appending a fully-detailed rendering of its content to + * {@code contents} (keyed by the hex of the partition key) and its key to {@code keys}, in token order. + */ + private static void readPartitions(SSTableReader sstable, + TableMetadata metadata, + Map contents, + List keys) + { + try (ISSTableScanner scanner = sstable.getScanner()) + { + while (scanner.hasNext()) + { + try (UnfilteredRowIterator partition = scanner.next()) + { + // clone the key: the scanner's buffers are not guaranteed to outlive the iterator + DecoratedKey key = sstable.getPartitioner() + .decorateKey(ByteBufferUtil.clone(partition.partitionKey().getKey())); + StringBuilder sb = new StringBuilder(); + sb.append("deletion=").append(partition.partitionLevelDeletion()); + sb.append(" static=").append(partition.staticRow().toString(metadata, true)); + while (partition.hasNext()) + sb.append("\n ").append(partition.next().toString(metadata, true)); + + String previous = contents.put(hex(key), sb.toString()); + assertNull("the same partition key appears twice inside " + sstable.descriptor + ": " + key, + previous); + keys.add(key); + } + } + } + } + + private static List tokensOf(List keys) + { + List tokens = new ArrayList<>(keys.size()); + for (DecoratedKey key : keys) + tokens.add(((Murmur3Partitioner.LongToken) key.getToken()).token); + return tokens; + } + + private static String hex(DecoratedKey key) + { + return ByteBufferUtil.bytesToHex(key.getKey()); + } + + /** splitmix64, so consecutive base seeds give uncorrelated iterations. */ + private static long scramble(long seed) + { + long z = seed + 0x9E3779B97F4A7C15L; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + + /** Everything needed to understand -- and replay -- one iteration. Mutated as the iteration progresses. */ + private static final class Config + { + final long seed; + + Shape shape; + String compressor = "?"; + int chunkKb = -1; + int columnIndexKb = -1; + int columnIndexCacheKb = -1; + boolean wide; + int partitions = -1; + int rowsPerPartition = -1; + int valueBytes = -1; + long totalBytes = -1; + + int parentPartitions = -1; + String fullRanges = "?"; + String transientRanges = "?"; + String labels = "?"; + int runCount = -1; + boolean expectedEligible; + + Config(long seed) + { + this.seed = seed; + } + + @Override + public String toString() + { + return "seed=" + seed + + " shape=" + shape + + " compressor=" + compressor + + " chunkKb=" + chunkKb + + " columnIndexKb=" + columnIndexKb + + " columnIndexCacheKb=" + columnIndexCacheKb + + " wide=" + wide + + " partitions=" + partitions + + " rowsPerPartition=" + rowsPerPartition + + " valueBytes=" + valueBytes + + " totalBytes=" + totalBytes + + " parentPartitions=" + parentPartitions + + " runCount=" + runCount + + " expectedEligible=" + expectedEligible + + "\n fullRanges=" + fullRanges + + "\n transientRanges=" + transientRanges + + "\n labels=" + labels; + } + } +} diff --git a/test/unit/org/apache/cassandra/db/compaction/ZeroCopyAntiCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/ZeroCopyAntiCompactionTest.java new file mode 100644 index 000000000000..9c274fa5b127 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/compaction/ZeroCopyAntiCompactionTest.java @@ -0,0 +1,700 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.compaction; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.Refs; + +import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * End-to-end wiring test for the zero-copy anticompaction path + * ({@link CompactionManager#zeroCopyAntiCompact}, {@link AntiCompactionRunPlanner}, + * {@link ZeroCopySSTableSplitter}). Every test drives the real + * {@link CompactionManager#performAnticompaction} against a real registered parent repair session, so the whole + * chain -- {@code validateSSTableBoundsForAnticompaction}, {@code mutateFullyContainedSSTables}, + * {@code doAntiCompaction}'s per-group {@code txn.split}, the carve-out, the 1-to-N transaction replacement and + * the accounting guard -- is exercised. + * + *

How "which path ran" is observed

+ * {@code cfs.metric.bytesZeroCopyAnticompaction} is marked only after a successful zero-copy commit, and + * CQLTester makes a fresh table (hence fresh {@code TableMetrics}) per test, so its per-table count is an exact, + * non-flaky witness: {@code > 0} means the split ran, {@code 0} the rewrite. The output sstable count is a second, + * independent witness, since the paths differ structurally for the same input: the rewrite produces at most one + * sstable per repair bucket (full / transient / unrepaired) where the split produces one child per contiguous label + * run, so {@code U F U} is 2 rewritten but 3 split and {@code U F U T U} is 3 versus 5. Both are asserted + * in every test, in both directions, so this fails if the gate silently stops or starts engaging. + * + *

Correctness assertions, applied identically to every case

+ * {@link #assertOutcome} snapshots every partition of the parent before the run (key -> a full textual rendering + * including partition deletions, row liveness info, row deletions, every cell value and timestamp) and compares it + * against the union of the outputs afterwards, failing on a missing key, an extra key, a key in two outputs at + * once, or any content difference. Repair state is checked per partition key rather than per sstable, + * since a partition routed into the wrong bucket is the critical failure mode, and {@link Util#assertOnDiskState} + * proves the parent's files are really gone. + */ +public class ZeroCopyAntiCompactionTest extends CQLTester +{ + /** Enough partitions to give every run several compression chunks of its own. */ + private static final int PARTITIONS = 200; + private static final int ROWS_PER_PARTITION = 5; + private static final int VALUE_BYTES = 500; + + /** What the ranges say a partition should become. Computed by the test, independently of the planner. */ + private enum Expect + { FULL, TRANSIENT, UNREPAIRED } + + private boolean savedZeroCopyEnabled; + + @Before + public void saveZeroCopyFlag() + { + savedZeroCopyEnabled = DatabaseDescriptor.getZeroCopyAnticompactionEnabled(); + } + + @After + public void restoreZeroCopyFlag() + { + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(savedZeroCopyEnabled); + } + + // ---------------------------------------------------------------------------------------------------- + // The happy path + // ---------------------------------------------------------------------------------------------------- + + /** + * The single-contiguous-run shape the gate exists for: {@code UNREPAIRED, FULL, UNREPAIRED}. The zero-copy + * split must run and must produce one child per run (3), each with the right repair state for every key it + * holds, and together holding exactly the parent's data. + */ + @Test + public void singleFullRunIsSplitZeroCopy() throws Throwable + { + createCompressedTable(""); + insertPartitions(); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertTrue("the whole point of this test is a compressed BIG sstable", + ZeroCopySSTableSplitter.isSupported(parent)); + + List keys = keysInTokenOrder(cfs); + assertEquals(keys.get(0), parent.first); + assertEquals(keys.get(keys.size() - 1), parent.last); + + // (keys[59].token, keys[139].token] covers keys[60..139] exactly: U(0..59) F(60..139) U(140..199). + Collection> full = Collections.singleton(rangeCovering(keys, 60, 140)); + RangesAtEndpoint ranges = rangesAtEndpoint(full, Collections.emptySet()); + + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(true); + Outputs before = collect(Collections.singleton(parent)); + assertEquals(PARTITIONS, before.partitions); + + TimeUUID sessionID = nextTimeUUID(); + anticompact(cfs, ranges, sessionID); + + assertTrue("the zero-copy split did not run: metric is " + zeroCopyBytes(cfs), + zeroCopyBytes(cfs) > 0); + // 3 runs -> 3 children. The rewrite path would have produced 2 (one pending, one unrepaired). + assertOutcome(cfs, parent, before, full, Collections.emptySet(), sessionID, 3); + assertReopenableAndVerifiable(cfs); + } + + /** + * Both a full and a transient range, laid out as {@code UNREPAIRED, FULL, UNREPAIRED, TRANSIENT, + * UNREPAIRED} -- 5 runs, still eligible because FULL and TRANSIENT each occupy exactly one run. Getting + * {@code isTransient} the wrong way round is a real correctness bug (a non-transient pending child is + * promoted to repaired at session finalize instead of being dropped), so it is asserted per key. + */ + @Test + public void fullAndTransientRunsAreSplitZeroCopy() throws Throwable + { + createCompressedTable(""); + insertPartitions(); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertTrue(ZeroCopySSTableSplitter.isSupported(parent)); + + List keys = keysInTokenOrder(cfs); + Collection> full = Collections.singleton(rangeCovering(keys, 40, 90)); + Collection> trans = Collections.singleton(rangeCovering(keys, 120, 170)); + RangesAtEndpoint ranges = rangesAtEndpoint(full, trans); + + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(true); + Outputs before = collect(Collections.singleton(parent)); + assertEquals(PARTITIONS, before.partitions); + + TimeUUID sessionID = nextTimeUUID(); + anticompact(cfs, ranges, sessionID); + + assertTrue("the zero-copy split did not run", zeroCopyBytes(cfs) > 0); + // 5 runs -> 5 children. The rewrite path would have produced 3. + assertOutcome(cfs, parent, before, full, trans, sessionID, 5); + assertReopenableAndVerifiable(cfs); + + // and, explicitly: exactly one transient output, holding exactly the transient range's partitions + int transientSSTables = 0; + for (SSTableReader sstable : cfs.getLiveSSTables()) + { + if (sstable.isTransient()) + { + transientSSTables++; + assertTrue("a transient child must be pending repair", sstable.isPendingRepair()); + assertEquals(sessionID, sstable.getPendingRepair()); + } + } + assertEquals(1, transientSSTables); + } + + // ---------------------------------------------------------------------------------------------------- + // The gate: everything that must NOT take the zero-copy path + // ---------------------------------------------------------------------------------------------------- + + /** + * Interleaved full ranges -- FULL in two runs, i.e. {@code U F U F U} -- is what vnodes produce and is + * exactly what the gate rejects, because the split can only emit contiguous key ranges. The rewrite path + * must run instead, and the result must still be completely correct. + */ + @Test + public void interleavedFullRangesFallBackToTheRewritePath() throws Throwable + { + createCompressedTable(""); + insertPartitions(); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertTrue("the sstable itself is eligible; only the range layout must make it ineligible", + ZeroCopySSTableSplitter.isSupported(parent)); + + List keys = keysInTokenOrder(cfs); + List> full = new ArrayList<>(); + full.add(rangeCovering(keys, 20, 40)); + full.add(rangeCovering(keys, 100, 120)); + RangesAtEndpoint ranges = rangesAtEndpoint(full, Collections.emptySet()); + + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(true); + Outputs before = collect(Collections.singleton(parent)); + + TimeUUID sessionID = nextTimeUUID(); + anticompact(cfs, ranges, sessionID); + + assertEquals("the zero-copy split must not run for interleaved ranges", 0, zeroCopyBytes(cfs)); + // The rewrite routes by token, so the two FULL runs land in ONE pending sstable and the three + // UNREPAIRED runs land in ONE unrepaired sstable. The split would have produced 5. + assertOutcome(cfs, parent, before, full, Collections.emptySet(), sessionID, 2); + } + + /** An uncompressed sstable has no compression chunks to copy, so {@code isSupported} is false. */ + @Test + public void uncompressedSSTableFallsBackToTheRewritePath() throws Throwable + { + createTable("CREATE TABLE %s (pk text, ck int, val text, PRIMARY KEY (pk, ck)) " + + "WITH compression = {'enabled': 'false'}"); + disableCompaction(); + insertPartitions(); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertFalse("an uncompressed sstable must not be splittable", + ZeroCopySSTableSplitter.isSupported(parent)); + + List keys = keysInTokenOrder(cfs); + Collection> full = Collections.singleton(rangeCovering(keys, 60, 140)); + RangesAtEndpoint ranges = rangesAtEndpoint(full, Collections.emptySet()); + + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(true); + Outputs before = collect(Collections.singleton(parent)); + + TimeUUID sessionID = nextTimeUUID(); + anticompact(cfs, ranges, sessionID); + + assertEquals("an uncompressed sstable must not take the zero-copy path", 0, zeroCopyBytes(cfs)); + assertOutcome(cfs, parent, before, full, Collections.emptySet(), sessionID, 2); + } + + /** + * The kill switch. Exactly the eligible layout of {@link #singleFullRunIsSplitZeroCopy}, but with + * {@code zero_copy_anticompaction_enabled = false}: the split must not run at all (not even the planner's + * Index.db walk), and the outcome must be the unchanged rewrite result. + */ + @Test + public void killSwitchDisablesTheZeroCopyPath() throws Throwable + { + createCompressedTable(""); + insertPartitions(); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertTrue("with the flag on this sstable would be split", ZeroCopySSTableSplitter.isSupported(parent)); + + List keys = keysInTokenOrder(cfs); + Collection> full = Collections.singleton(rangeCovering(keys, 60, 140)); + RangesAtEndpoint ranges = rangesAtEndpoint(full, Collections.emptySet()); + + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(false); + Outputs before = collect(Collections.singleton(parent)); + + TimeUUID sessionID = nextTimeUUID(); + anticompact(cfs, ranges, sessionID); + + assertEquals("the kill switch did not stop the zero-copy path", 0, zeroCopyBytes(cfs)); + // 2, not the 3 children the split produces for this same layout. + assertOutcome(cfs, parent, before, full, Collections.emptySet(), sessionID, 2); + } + + // ---------------------------------------------------------------------------------------------------- + // The accepted behaviour change + // ---------------------------------------------------------------------------------------------------- + + /** + * Pins the accepted behaviour change: the zero-copy path copies compression chunks verbatim and therefore + * RETAINS droppable tombstones and shadowed data the rewriting anticompaction would have purged. Retention, + * never loss -- nothing can be resurrected -- and deliberately not gated on the droppable-tombstone ratio. + *

+ * The parent carries a partition-level and a row-level tombstone that are genuinely droppable when the + * anticompaction runs ({@code gc_grace_seconds = 0}, and the run is more than a second after the deletes, so + * {@code localDeletionTime < gcBefore}), asserted via {@link SSTableReader#getDroppableTombstonesBefore} so this + * cannot pass vacuously. Both must still be there afterwards. If someone later "fixes" this by purging, this + * test is the record that the retention was intentional. + */ + @Test + public void purgeableTombstonesSurviveTheZeroCopySplit() throws Throwable + { + createCompressedTable(" AND gc_grace_seconds = 0"); + insertPartitions(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + List keys = keysInTokenOrder(cfs); + // Both deleted partitions sit in the middle of the token order, so the full range below contains them. + String deletedPartition = keyOf(keys.get(100)); + String partiallyDeleted = keyOf(keys.get(101)); + execute("DELETE FROM %s WHERE pk = ?", deletedPartition); + execute("DELETE FROM %s WHERE pk = ? AND ck = ?", partiallyDeleted, 2); + flush(); + + SSTableReader parent = onlySSTable(cfs); + assertTrue(ZeroCopySSTableSplitter.isSupported(parent)); + + // gcBefore is computed from the wall clock when the anticompaction starts, and purging requires + // localDeletionTime < gcBefore, so the deletes must be strictly in the past for them to be droppable. + Uninterruptibles.sleepUninterruptibly(1100, TimeUnit.MILLISECONDS); + + Collection> full = Collections.singleton(rangeCovering(keys, 60, 140)); + RangesAtEndpoint ranges = rangesAtEndpoint(full, Collections.emptySet()); + + DatabaseDescriptor.setZeroCopyAnticompactionEnabled(true); + Outputs before = collect(Collections.singleton(parent)); + assertEquals(PARTITIONS, before.partitions); + DeletionTime parentTombstone = before.partitionDeletion.get(hexOf(deletedPartition)); + assertFalse("the fully deleted partition should carry a partition level tombstone in the parent", + parentTombstone.isLive()); + // Non-vacuity check. Assert the exact predicate CompactionController's purge evaluator applies + // (localDeletionTime < gcBefore) rather than SSTableReader.getDroppableTombstonesBefore: that reads + // estimatedTombstoneDropTime, and StreamingTombstoneHistogramBuilder.update rounds every point UP to + // the next roundSeconds boundary (ceilKey(point, roundSeconds), 60s by default per SSTable.java:81). + // A tombstone written a second ago therefore lands in a future bucket and the estimate is legitimately + // 0 here, which says nothing about whether the tombstone is actually droppable. + assertTrue("the partition tombstone is not droppable yet, so this test would pass vacuously:" + + " localDeletionTime=" + parentTombstone.localDeletionTime() + + " gcBefore=" + FBUtilities.nowInSeconds(), + parentTombstone.localDeletionTime() < FBUtilities.nowInSeconds()); + + TimeUUID sessionID = nextTimeUUID(); + anticompact(cfs, ranges, sessionID); + + assertTrue("the zero-copy split did not run", zeroCopyBytes(cfs) > 0); + // The content comparison inside assertOutcome already proves byte-for-byte retention of both + // tombstones; the explicit assertions below make the intent unmissable. + Outputs after = assertOutcome(cfs, parent, before, full, Collections.emptySet(), sessionID, 3); + + DeletionTime survived = after.partitionDeletion.get(hexOf(deletedPartition)); + assertNotNull("the fully deleted partition disappeared entirely", survived); + assertFalse("DECISION 2: a droppable partition tombstone must SURVIVE the zero-copy split", + survived.isLive()); + assertEquals("the surviving partition tombstone must be bit-identical", + before.partitionDeletion.get(hexOf(deletedPartition)), survived); + assertEquals("DECISION 2: the droppable row tombstone must SURVIVE the zero-copy split", + before.content.get(hexOf(partiallyDeleted)), + after.content.get(hexOf(partiallyDeleted))); + } + + // ---------------------------------------------------------------------------------------------------- + // Driving the real anticompaction + // ---------------------------------------------------------------------------------------------------- + + /** + * Runs {@link CompactionManager#performAnticompaction} over every live sstable, exactly as + * {@code PendingAntiCompaction} does: a registered parent repair session, the sstables marked + * {@code ANTICOMPACTION} in the tracker, and a {@link Refs} that {@code performAnticompaction} consumes + * itself (which is what finally unlinks the obsoleted parent's files). + */ + private void anticompact(ColumnFamilyStore cfs, RangesAtEndpoint ranges, TimeUUID sessionID) throws Exception + { + Set sstables = ImmutableSet.copyOf(cfs.getLiveSSTables()); + assertFalse("nothing to anticompact", sstables.isEmpty()); + + ActiveRepairService.instance.registerParentRepairSession(sessionID, + InetAddressAndPort.getByName("127.0.0.1"), + Lists.newArrayList(cfs), + ranges.ranges(), + true, + ActiveRepairService.UNREPAIRED_SSTABLE, + true, + PreviewKind.NONE); + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.ANTICOMPACTION); + Refs refs = Refs.ref(sstables)) + { + assertNotNull("could not mark the sstables compacting", txn); + CompactionManager.instance.performAnticompaction(cfs, ranges, refs, txn, sessionID, () -> false); + } + finally + { + ActiveRepairService.instance.removeParentRepairSession(sessionID); + } + } + + /** + * Every outcome assertion that must hold no matter which path ran: + *

    + *
  1. the parent is gone from the live set, is marked compacted, and no orphan Data.db survives + * {@code waitForDeletions} ({@link Util#assertOnDiskState});
    2. + *
  2. the live sstable count is exactly {@code expectedSSTables} -- which differs between the two paths + * and so doubles as a witness of which one ran;
    3. + *
  3. no partition is lost, duplicated or altered: same key set, same content, same total count;
    4. + *
  4. the repair state of the sstable holding each key matches what the ranges say for that key.
    5. + *
+ * + * @return the outputs, so a caller can make extra assertions about them + */ + private Outputs assertOutcome(ColumnFamilyStore cfs, + SSTableReader parent, + Outputs before, + Collection> full, + Collection> trans, + TimeUUID sessionID, + int expectedSSTables) + { + assertFalse("the parent is still live, so it was not obsoleted", cfs.getLiveSSTables().contains(parent)); + assertTrue("the parent was not marked compacted", parent.isMarkedCompacted()); + // waits for deletions, asserts the live count, and asserts that every *Data.db on disk belongs to a + // live sstable -- i.e. that the parent's files really are gone + Util.assertOnDiskState(cfs, expectedSSTables); + assertEquals("sstables were left marked compacting", 0, cfs.getTracker().getCompacting().size()); + + Outputs after = collect(cfs.getLiveSSTables()); + + assertEquals("partitions were lost or duplicated", before.partitions, after.partitions); + assertEquals("the set of partition keys changed", before.content.keySet(), after.content.keySet()); + for (Map.Entry entry : before.content.entrySet()) + { + assertEquals("the content of partition " + entry.getKey() + " changed", + entry.getValue(), after.content.get(entry.getKey())); + } + + for (Map.Entry entry : after.token.entrySet()) + { + String key = entry.getKey(); + Expect expected = expectedFor(entry.getValue(), full, trans); + SSTableReader owner = after.owner.get(key); + String context = "partition " + key + " (expected " + expected + ") in " + owner.descriptor; + + // repairedAt is never set by anticompaction; the promotion happens later, at session finalize. + assertFalse(context + ": must not be repaired", owner.isRepaired()); + assertEquals(context + ": repairedAt", ActiveRepairService.UNREPAIRED_SSTABLE, owner.getRepairedAt()); + switch (expected) + { + case FULL: + assertEquals(context + ": pendingRepair", sessionID, owner.getPendingRepair()); + assertFalse(context + ": isTransient", owner.isTransient()); + break; + case TRANSIENT: + assertEquals(context + ": pendingRepair", sessionID, owner.getPendingRepair()); + assertTrue(context + ": isTransient", owner.isTransient()); + break; + default: + assertNull(context + ": must not be pending repair", owner.getPendingRepair()); + assertFalse(context + ": isTransient", owner.isTransient()); + break; + } + } + return after; + } + + /** + * Nothing may depend on in-memory state: reopen each output purely from its on-disk components and run the + * extended {@link Verifier} over it, which walks Data.db linearly from the first index position, validates + * Digest.crc32, the index, the summary and the bloom filter, and throws on any inconsistency. This is the + * assertion that catches a child whose rebuilt components disagree with its copied Data.db. + */ + private void assertReopenableAndVerifiable(ColumnFamilyStore cfs) throws Exception + { + for (SSTableReader live : cfs.getLiveSSTables()) + { + SSTableReader reopened = SSTableReader.open(live.descriptor, live.getComponents(), cfs.metadata); + try + { + assertEquals(live.first, reopened.first); + assertEquals(live.last, reopened.last); + assertEquals(live.getPendingRepair(), reopened.getPendingRepair()); + assertEquals(live.isTransient(), reopened.isTransient()); + assertEquals(live.getRepairedAt(), reopened.getRepairedAt()); + try (Verifier verifier = new Verifier(cfs, reopened, true, + Verifier.options().extendedVerification(true).build())) + { + verifier.verify(); + } + } + finally + { + reopened.selfRef().release(); + } + } + } + + // ---------------------------------------------------------------------------------------------------- + // Content snapshots + // ---------------------------------------------------------------------------------------------------- + + /** Everything the assertions need about one set of sstables, keyed by hex partition key. */ + private static final class Outputs + { + /** Full textual rendering of the partition: deletions, liveness info, cells, timestamps. */ + final Map content = new HashMap<>(); + /** Which sstable held the partition. */ + final Map owner = new HashMap<>(); + final Map token = new HashMap<>(); + final Map partitionDeletion = new HashMap<>(); + int partitions; + } + + /** + * Scans every sstable and records each partition. A partition appearing in two sstables fails here, which + * is how duplication is detected: the outputs of an anticompaction are disjoint by construction. + */ + private static Outputs collect(Collection sstables) + { + Outputs out = new Outputs(); + for (SSTableReader sstable : sstables) + { + try (ISSTableScanner scanner = sstable.getScanner()) + { + while (scanner.hasNext()) + { + try (UnfilteredRowIterator partition = scanner.next()) + { + DecoratedKey key = partition.partitionKey(); + String hex = ByteBufferUtil.bytesToHex(key.getKey()); + DeletionTime deletion = partition.partitionLevelDeletion(); + // describe() consumes the iterator, so it must happen inside this block + String description = describe(partition, sstable.metadata()); + SSTableReader previous = out.owner.get(hex); + assertNull("partition " + hex + " is in both " + previous + " and " + sstable.descriptor, + out.content.put(hex, description)); + out.owner.put(hex, sstable); + out.token.put(hex, key.getToken()); + out.partitionDeletion.put(hex, deletion); + out.partitions++; + } + } + } + } + assertTrue("nothing was collected", out.partitions > 0); + return out; + } + + /** + * A canonical rendering of one partition. {@code toString(metadata, true)} is the full-detail form: primary key + * liveness info (timestamp, ttl, local expiration), the row deletion if any, and every cell via + * {@code AbstractCell.toString()}, which includes the cell timestamp and marks tombstones. So comparing these + * strings compares rows, cells, timestamps and deletions, not just keys -- and unlike the iterators they can be + * retained safely after the scanner and the parent's files are gone. + */ + private static String describe(UnfilteredRowIterator partition, TableMetadata metadata) + { + StringBuilder sb = new StringBuilder(); + sb.append("partitionDeletion=").append(partition.partitionLevelDeletion()); + sb.append(" static=").append(partition.staticRow().toString(metadata, true)); + while (partition.hasNext()) + sb.append('\n').append(partition.next().toString(metadata, true)); + return sb.toString(); + } + + // ---------------------------------------------------------------------------------------------------- + // Fixtures + // ---------------------------------------------------------------------------------------------------- + + private void createCompressedTable(String extraOptions) throws Throwable + { + // small chunks plus near-incompressible values, so the sstable really spans many compression chunks + createTable("CREATE TABLE %s (pk text, ck int, val text, PRIMARY KEY (pk, ck)) " + + "WITH compression = {'class': 'LZ4Compressor', 'chunk_length_in_kb': '4'}" + extraOptions); + disableCompaction(); + } + + private void insertPartitions() throws Throwable + { + for (int p = 0; p < PARTITIONS; p++) + for (int c = 0; c < ROWS_PER_PARTITION; c++) + execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, ?)", key(p), c, randomText(VALUE_BYTES)); + } + + private static String key(int p) + { + return String.format("k%06d", p); + } + + /** Near-incompressible payload. */ + private static String randomText(int length) + { + ThreadLocalRandom random = ThreadLocalRandom.current(); + char[] chars = new char[length]; + for (int i = 0; i < length; i++) + chars[i] = (char) ('!' + random.nextInt(94)); + return new String(chars); + } + + /** + * The partition keys in on-disk (token) order, derived by decorating them directly rather than by reading + * the sstable, so ranges can be chosen before anything is written and so the test is partitioner agnostic. + * The callers cross-check the result against the parent's {@code first} / {@code last}. + */ + private static List keysInTokenOrder(ColumnFamilyStore cfs) + { + List keys = new ArrayList<>(PARTITIONS); + for (int p = 0; p < PARTITIONS; p++) + keys.add(cfs.getPartitioner().decorateKey(ByteBufferUtil.bytes(key(p)))); + keys.sort(DecoratedKey::compareTo); + return keys; + } + + /** + * A range covering exactly {@code keys[fromInclusive .. toExclusive - 1]}. Ranges are half-open + * {@code (left, right]}, so the left bound is the token of the key just before the first one wanted. + * {@code fromInclusive} must be > 0 and {@code toExclusive} <= {@code keys.size()}, which is what + * leaves an unrepaired run on each side and keeps {@code mutateFullyContainedSSTables} from claiming the + * sstable via the metadata-only path. + */ + private static Range rangeCovering(List keys, int fromInclusive, int toExclusive) + { + assertTrue("leave an unrepaired prefix", fromInclusive > 0); + assertTrue("leave an unrepaired suffix", toExclusive < keys.size()); + return new Range<>(keys.get(fromInclusive - 1).getToken(), keys.get(toExclusive - 1).getToken()); + } + + private static RangesAtEndpoint rangesAtEndpoint(Collection> full, + Collection> trans) + { + InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(local); + for (Range range : full) + builder.add(Replica.fullReplica(local, range)); + for (Range range : trans) + builder.add(Replica.transientReplica(local, range)); + return builder.build(); + } + + /** What the ranges say a token becomes; full wins over transient, as the anticompaction routing does. */ + private static Expect expectedFor(Token token, Collection> full, Collection> trans) + { + for (Range range : full) + { + if (range.contains(token)) + return Expect.FULL; + } + for (Range range : trans) + { + if (range.contains(token)) + return Expect.TRANSIENT; + } + return Expect.UNREPAIRED; + } + + /** + * The witness of which path ran: marked only on a successful zero-copy commit, and zero for a fresh table. + */ + private static long zeroCopyBytes(ColumnFamilyStore cfs) + { + return cfs.metric.bytesZeroCopyAnticompaction.getCount(); + } + + private static SSTableReader onlySSTable(ColumnFamilyStore cfs) + { + Set live = cfs.getLiveSSTables(); + assertEquals("expected exactly one sstable", 1, live.size()); + return live.iterator().next(); + } + + private static String keyOf(DecoratedKey key) throws Exception + { + return ByteBufferUtil.string(key.getKey()); + } + + private static String hexOf(String partitionKey) + { + return ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(partitionKey)); + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterArithmeticTest.java b/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterArithmeticTest.java new file mode 100644 index 000000000000..186bf6d39993 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterArithmeticTest.java @@ -0,0 +1,1099 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.sstable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.junit.Test; + +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter.ChunkRange; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter.CopyPlan; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +/** + * Pure arithmetic tests for {@link ZeroCopySSTableSplitter}. No sstables, no disk, no schema, no config -- + * every method exercised here is static and operates only on primitives, so this test is where the off-by-one + * bugs in the chunk-run computation have to be caught. + * + *

The properties under test, restated independently of the implementation: + *

    + *
  • {@code i = lo / L} -- the chunk containing the child's first live byte;
    • + *
  • {@code j = (hi - 1) / L} -- the chunk containing the child's LAST live byte. Not {@code hi / L}: + * {@code hi} is exclusive, so a {@code hi} that lands exactly on a chunk boundary must NOT pull in the + * chunk that starts there;
    • + *
  • {@code C = j - i + 1}, {@code Dp = hi - i*L}, {@code shift = i*L}, {@code dead = lo mod L};
    • + *
  • {@code (C-1)*L < Dp <= C*L} -- the last chunk holds at least one live byte and at most a full chunk + * of them;
    • + *
  • {@code Dp - dead == hi - lo} -- the child's live span is exactly the parent's;
    • + *
  • everything is computed in long arithmetic; an {@code int} product {@code k*L} overflows past 2 GiB.
    • + *
+ */ +public class ZeroCopySSTableSplitterArithmeticTest +{ + private static final int K = 1024; + private static final int L4 = 4 * K; + private static final int L16 = 16 * K; + private static final int L64 = 64 * K; + private static final int[] REAL_CHUNK_LENGTHS = { L4, L16, L64 }; + + /** Fixed so a sweep failure reproduces; the value is echoed in every sweep failure message. */ + private static final long SEED = 20260726L; + + /** + * The alignment {@code copyPlan} works to, restated here rather than read from the class under test: these + * tests are the definition of it. It has to match {@link org.apache.cassandra.io.util.Reflink#RANGE_ALIGNMENT}. + */ + private static final long A = 64 * 1024; + + // ------------------------------------------------------------------------------------------------ + // chunkIndexFor / firstChunk: boundary, one before, one after + // ------------------------------------------------------------------------------------------------ + + @Test + public void chunkIndexForOnAroundAndBetweenBoundaries() + { + for (int L : REAL_CHUNK_LENGTHS) + { + assertEquals("first byte of the file", 0, ZeroCopySSTableSplitter.chunkIndexFor(0, L)); + assertEquals("last byte of chunk 0", 0, ZeroCopySSTableSplitter.chunkIndexFor(L - 1, L)); + assertEquals("first byte of chunk 1", 1, ZeroCopySSTableSplitter.chunkIndexFor(L, L)); + assertEquals("second byte of chunk 1", 1, ZeroCopySSTableSplitter.chunkIndexFor(L + 1, L)); + + for (long k = 0; k < 5; k++) + { + long base = k * L; + assertEquals("boundary k=" + k + " L=" + L, k, ZeroCopySSTableSplitter.chunkIndexFor(base, L)); + assertEquals("boundary+1 k=" + k + " L=" + L, k, ZeroCopySSTableSplitter.chunkIndexFor(base + 1, L)); + assertEquals("boundary-1 k=" + k + " L=" + L, + Math.max(0, k - 1), ZeroCopySSTableSplitter.chunkIndexFor(Math.max(0, base - 1), L)); + assertEquals("mid-chunk k=" + k + " L=" + L, k, ZeroCopySSTableSplitter.chunkIndexFor(base + L / 2, L)); + assertEquals("last byte k=" + k + " L=" + L, k, ZeroCopySSTableSplitter.chunkIndexFor(base + L - 1, L)); + } + } + } + + @Test + public void firstChunkIsChunkIndexFor() + { + Random rnd = new Random(SEED); + for (int L : REAL_CHUNK_LENGTHS) + { + for (int t = 0; t < 500; t++) + { + long lo = nextLong(rnd, 1L << 36); + assertEquals("lo=" + lo + " L=" + L, + ZeroCopySSTableSplitter.chunkIndexFor(lo, L), + ZeroCopySSTableSplitter.firstChunk(lo, L)); + assertEquals("lo=" + lo + " L=" + L, lo / L, ZeroCopySSTableSplitter.firstChunk(lo, L)); + } + } + } + + /** Division, not a power-of-two bit mask: a masking shortcut would give the wrong answer here. */ + @Test + public void chunkArithmeticIsPlainDivisionNotAMask() + { + assertEquals(3, ZeroCopySSTableSplitter.chunkIndexFor(10, 3)); + assertEquals(1, ZeroCopySSTableSplitter.deadPrefixBytes(10, 3)); + assertEquals(3, ZeroCopySSTableSplitter.lastChunk(10, 3)); // (10-1)/3 + assertEquals(2, ZeroCopySSTableSplitter.lastChunk(9, 3)); // (9-1)/3 -- boundary, not 3 + assertEquals(0, ZeroCopySSTableSplitter.chunkIndexFor(999, 1000)); + assertEquals(1, ZeroCopySSTableSplitter.chunkIndexFor(1000, 1000)); + } + + // ------------------------------------------------------------------------------------------------ + // lastChunk: the (hi - 1)/L vs hi/L distinction + // ------------------------------------------------------------------------------------------------ + + @Test + public void lastChunkUsesHiMinusOneNotHi() + { + for (int L : REAL_CHUNK_LENGTHS) + { + // hi exactly on a boundary: the naive hi/L would be one too far. + for (long k = 1; k <= 5; k++) + { + long hi = k * L; + assertEquals("hi==" + k + "*L must stop at chunk " + (k - 1) + " (L=" + L + ')', + k - 1, ZeroCopySSTableSplitter.lastChunk(hi, L)); + assertNotEquals("lastChunk must not be hi/L for a boundary hi", + hi / L, ZeroCopySSTableSplitter.lastChunk(hi, L)); + + // one byte before the boundary is still in the previous chunk + assertEquals(k - 1, ZeroCopySSTableSplitter.lastChunk(hi - 1, L)); + // one byte after crosses into chunk k + assertEquals(k, ZeroCopySSTableSplitter.lastChunk(hi + 1, L)); + } + + assertEquals("a one-byte file lives entirely in chunk 0", 0, ZeroCopySSTableSplitter.lastChunk(1, L)); + } + } + + /** + * The whole point of {@code (hi-1)/L}: the child that ends exactly on a chunk boundary must copy one + * chunk fewer than the child that ends one byte later, and the {@code (C-1)*L < Dp} invariant is what + * would break if it did not. + */ + @Test + public void hiOnChunkBoundaryDoesNotPullInAnExtraChunk() + { + for (int L : REAL_CHUNK_LENGTHS) + { + ChunkRange exact = ZeroCopySSTableSplitter.chunkRange(0, 2L * L, L); + assertEquals(0, exact.firstChunk); + assertEquals("hi == 2L must end at chunk 1", 1, exact.lastChunk); + assertEquals(2, exact.chunkCount); + assertEquals(2L * L, exact.dataLength); + assertRangeInvariants("exact L=" + L, 0, 2L * L, L, exact); + + ChunkRange onePast = ZeroCopySSTableSplitter.chunkRange(0, 2L * L + 1, L); + assertEquals("one byte past the boundary needs a third chunk", 2, onePast.lastChunk); + assertEquals(3, onePast.chunkCount); + assertEquals(2L * L + 1, onePast.dataLength); + assertRangeInvariants("onePast L=" + L, 0, 2L * L + 1, L, onePast); + + ChunkRange oneBefore = ZeroCopySSTableSplitter.chunkRange(0, 2L * L - 1, L); + assertEquals(1, oneBefore.lastChunk); + assertEquals(2, oneBefore.chunkCount); + assertRangeInvariants("oneBefore L=" + L, 0, 2L * L - 1, L, oneBefore); + + assertEquals("boundary and one-byte-before must copy the same chunk run", + exact.chunkCount, oneBefore.chunkCount); + assertEquals(exact.chunkCount + 1, onePast.chunkCount); + } + } + + // ------------------------------------------------------------------------------------------------ + // Degenerate children + // ------------------------------------------------------------------------------------------------ + + @Test + public void singleChunkChildren() + { + for (int L : REAL_CHUNK_LENGTHS) + { + // whole chunk 0 + ChunkRange whole = ZeroCopySSTableSplitter.chunkRange(0, L, L); + assertEquals(0, whole.firstChunk); + assertEquals(0, whole.lastChunk); + assertEquals(1, whole.chunkCount); + assertEquals(L, whole.dataLength); + assertEquals(0, whole.shift); + assertEquals(0, whole.deadPrefixBytes); + assertRangeInvariants("whole chunk 0 L=" + L, 0, L, L, whole); + + // a child living entirely inside chunk 7 + long lo = 7L * L + 10; + long hi = 7L * L + 4000; + ChunkRange inside = ZeroCopySSTableSplitter.chunkRange(lo, hi, L); + assertEquals(7, inside.firstChunk); + assertEquals(7, inside.lastChunk); + assertEquals(1, inside.chunkCount); + assertEquals(7L * L, inside.shift); + assertEquals(10, inside.deadPrefixBytes); + assertEquals(4000, inside.dataLength); + assertRangeInvariants("inside chunk 7 L=" + L, lo, hi, L, inside); + + // the very last live byte of chunk 7 + ChunkRange toEnd = ZeroCopySSTableSplitter.chunkRange(7L * L, 8L * L, L); + assertEquals(7, toEnd.firstChunk); + assertEquals(7, toEnd.lastChunk); + assertEquals(1, toEnd.chunkCount); + assertEquals(L, toEnd.dataLength); + assertRangeInvariants("chunk 7 exactly L=" + L, 7L * L, 8L * L, L, toEnd); + } + } + + @Test + public void singleByteChildren() + { + for (int L : REAL_CHUNK_LENGTHS) + { + long[] los = { 0, 1, L - 1, L, L + 1, 5L * L, 5L * L + L / 2, 5L * L + L - 1 }; + for (long lo : los) + { + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(lo, lo + 1, L); + assertEquals("a one-byte child spans exactly one chunk (lo=" + lo + " L=" + L + ')', + 1, r.chunkCount); + assertEquals(r.firstChunk, r.lastChunk); + assertEquals("Dp == dead + 1", r.deadPrefixBytes + 1, r.dataLength); + assertRangeInvariants("single byte lo=" + lo + " L=" + L, lo, lo + 1, L, r); + } + } + } + + // ------------------------------------------------------------------------------------------------ + // Rejections -- an invalid range must throw, never produce a bogus ChunkRange + // ------------------------------------------------------------------------------------------------ + + @Test + public void emptyRangeIsRejected() + { + for (int L : REAL_CHUNK_LENGTHS) + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(0, 0, L)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(1000, 1000, L)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(3L * L, 3L * L, L)) + .isInstanceOf(IllegalArgumentException.class); + } + } + + @Test + public void invertedRangeIsRejected() + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(100, 99, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(1L << 30, 1, L64)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void negativePositionsAreRejected() + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkIndexFor(-1, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.firstChunk(-1, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.deadPrefixBytes(-1, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(-1, 10, L4)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void nonPositiveHiIsRejectedByLastChunk() + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.lastChunk(0, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.lastChunk(-1, L4)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void nonPositiveChunkLengthIsRejected() + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkIndexFor(0, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkIndexFor(0, -4096)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.lastChunk(10, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.deadPrefixBytes(10, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.childDataLength(10, 0, 0)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.chunkRange(0, 10, 0)) + .isInstanceOf(IllegalArgumentException.class); + } + + /** {@code hi} at or below the start of the first chunk means the child has no live bytes at all. */ + @Test + public void nonPositiveChildDataLengthIsRejected() + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.childDataLength(L4, 1, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.childDataLength(L4 - 1, 1, L4)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.childDataLength(0, 0, L4)) + .isInstanceOf(IllegalArgumentException.class); + } + + // ------------------------------------------------------------------------------------------------ + // deadPrefixBytes + // ------------------------------------------------------------------------------------------------ + + @Test + public void deadPrefixIsLoModChunkLengthAndZeroExactlyWhenAligned() + { + Random rnd = new Random(SEED); + for (int L : REAL_CHUNK_LENGTHS) + { + for (long k = 0; k < 4; k++) + { + assertEquals("aligned lo must have no dead prefix", + 0, ZeroCopySSTableSplitter.deadPrefixBytes(k * L, L)); + if (k > 0) + assertEquals(L - 1, ZeroCopySSTableSplitter.deadPrefixBytes(k * L - 1, L)); + assertEquals(1, ZeroCopySSTableSplitter.deadPrefixBytes(k * L + 1, L)); + } + + for (int t = 0; t < 2000; t++) + { + long lo = nextLong(rnd, 1L << 36); + long dead = ZeroCopySSTableSplitter.deadPrefixBytes(lo, L); + assertEquals("lo=" + lo + " L=" + L, lo % L, dead); + assertTrue("dead prefix must be < L: lo=" + lo + " L=" + L, dead >= 0 && dead < L); + assertEquals("dead == 0 iff lo is chunk aligned: lo=" + lo + " L=" + L, + lo % L == 0, dead == 0); + // and it is exactly the distance from the start of the first chunk + assertEquals(lo - ZeroCopySSTableSplitter.firstChunk(lo, L) * (long) L, dead); + } + } + } + + // ------------------------------------------------------------------------------------------------ + // childDataLength invariant, swept + // ------------------------------------------------------------------------------------------------ + + @Test + public void childDataLengthInvariantHoldsOverASweep() + { + for (int L : REAL_CHUNK_LENGTHS) + { + for (long i = 0; i < 8; i++) + { + long chunkStart = i * L; + // every interesting hi in the chunks [i, i+3] + for (long span = 1; span <= 3L * L; span += Math.max(1, L / 8)) + { + checkChildDataLength(chunkStart + span, i, L); + } + // and the exact boundaries + for (long c = 1; c <= 4; c++) + { + checkChildDataLength(chunkStart + c * L, i, L); + checkChildDataLength(chunkStart + c * L - 1, i, L); + checkChildDataLength(chunkStart + c * L + 1, i, L); + } + } + } + } + + private static void checkChildDataLength(long hi, long firstChunk, int L) + { + long dp = ZeroCopySSTableSplitter.childDataLength(hi, firstChunk, L); + assertEquals("Dp = hi - i*L (hi=" + hi + " i=" + firstChunk + " L=" + L + ')', + hi - firstChunk * (long) L, dp); + long lastChunk = ZeroCopySSTableSplitter.lastChunk(hi, L); + long c = lastChunk - firstChunk + 1; + assertTrue("(C-1)*L < Dp violated: C=" + c + " L=" + L + " Dp=" + dp, + (c - 1) * (long) L < dp); + assertTrue("Dp <= C*L violated: C=" + c + " L=" + L + " Dp=" + dp, + dp <= c * (long) L); + } + + // ------------------------------------------------------------------------------------------------ + // Exhaustive tiny sweep -- cheap and total, catches any off-by-one immediately + // ------------------------------------------------------------------------------------------------ + + @Test + public void exhaustiveTinyChunkLengthSweep() + { + for (int L : new int[]{ 1, 2, 3, 8, 16 }) + { + for (long lo = 0; lo <= 4L * L; lo++) + { + for (long hi = lo + 1; hi <= 4L * L + 3; hi++) + { + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(lo, hi, L); + assertRangeInvariants("tiny L=" + L + " lo=" + lo + " hi=" + hi, lo, hi, L, r); + } + } + } + } + + // ------------------------------------------------------------------------------------------------ + // Brute-force sweep over realistic chunk lengths + // ------------------------------------------------------------------------------------------------ + + @Test + public void bruteForceSweepOverRealisticChunkLengths() + { + long seed = SEED; + Random rnd = new Random(seed); + int checked = 0; + try + { + for (int L : REAL_CHUNK_LENGTHS) + { + long[] offsets = interestingOffsets(L); + for (int a = 0; a < offsets.length; a++) + { + for (int b = 0; b < offsets.length; b++) + { + long lo = offsets[a]; + long hi = offsets[b]; + if (hi <= lo) + continue; + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(lo, hi, L); + assertRangeInvariants("systematic L=" + L + " lo=" + lo + " hi=" + hi, lo, hi, L, r); + checked++; + } + } + + for (int t = 0; t < 20000; t++) + { + long lo = nextLong(rnd, 1L << 38); + long hi = lo + 1 + nextLong(rnd, 6L * L); + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(lo, hi, L); + assertRangeInvariants("random L=" + L + " lo=" + lo + " hi=" + hi, lo, hi, L, r); + checked++; + } + } + } + catch (AssertionError | RuntimeException e) + { + throw new AssertionError("sweep failed with seed=" + seed + " after " + checked + + " cases: " + e, e); + } + assertTrue("sweep should have checked a lot of cases, got " + checked, checked > 50000); + } + + /** Chunk boundaries, and one byte either side of them, at small, medium and very large chunk indices. */ + private static long[] interestingOffsets(int L) + { + long[] chunkIndices = { 0, 1, 2, 3, 17, 1000, 65535, 65536, 65537, 1048576 }; + long[] deltas = { -2, -1, 0, 1, 2, L / 2, L - 1 }; + List out = new ArrayList<>(); + for (long k : chunkIndices) + { + for (long d : deltas) + { + long v = k * L + d; + if (v >= 0) + out.add(v); + } + } + long[] arr = new long[out.size()]; + for (int i = 0; i < arr.length; i++) + arr[i] = out.get(i); + return arr; + } + + // ------------------------------------------------------------------------------------------------ + // Overflow: positions beyond Integer.MAX_VALUE + // ------------------------------------------------------------------------------------------------ + + @Test + public void largeOffsetsAreComputedInLongArithmetic() + { + final long fortyGiB = 40L * 1024 * 1024 * 1024; // 42949672960, way past Integer.MAX_VALUE + + for (int L : REAL_CHUNK_LENGTHS) + { + long expectedChunk = fortyGiB / L; + assertEquals("40GiB / " + L, expectedChunk, ZeroCopySSTableSplitter.chunkIndexFor(fortyGiB, L)); + assertTrue("chunk index for 40GiB must be positive", expectedChunk > 0); + + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(fortyGiB + 123, fortyGiB + 123 + 5L * L, L); + assertRangeInvariants("40GiB L=" + L, fortyGiB + 123, fortyGiB + 123 + 5L * L, L, r); + assertTrue("shift must not overflow: " + r.shift, r.shift > Integer.MAX_VALUE); + assertEquals(r.firstChunk * (long) L, r.shift); + } + } + + /** + * The concrete trap: {@code (int) (k * L)} is exactly 0 when {@code k * L == 2^32}, and negative when + * {@code k * L} lands in {@code [2^31, 2^32)}. Both would silently produce a bogus shift. + */ + @Test + public void chunkTimesChunkLengthDoesNotOverflowInt() + { + // k * L == 2^32 exactly -> an int product would be 0 + checkNoIntOverflow(1L << 32, L64, 65536); + checkNoIntOverflow(1L << 32, L16, 262144); + checkNoIntOverflow(1L << 32, L4, 1048576); + + // k * L == 2^31 exactly -> an int product would be Integer.MIN_VALUE + checkNoIntOverflow(1L << 31, L64, 32768); + checkNoIntOverflow(1L << 31, L16, 131072); + checkNoIntOverflow(1L << 31, L4, 524288); + + // k * L somewhere in the negative half of the int range + checkNoIntOverflow(3L << 30, L64, (3L << 30) / L64); + } + + private static void checkNoIntOverflow(long alignedLo, int L, long expectedChunk) + { + assertEquals("test setup: lo must be chunk aligned", 0, alignedLo % L); + assertEquals("chunk index at lo=" + alignedLo + " L=" + L, + expectedChunk, ZeroCopySSTableSplitter.firstChunk(alignedLo, L)); + // fixture sanity: an int-truncated product really would give the wrong answer here + assertNotEquals("test fixture is pointless unless the int product overflows", + alignedLo, (long) (int) (expectedChunk * L)); + + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(alignedLo, alignedLo + 3L * L, L); + assertEquals("shift must be the exact long product", alignedLo, r.shift); + assertTrue("shift must be positive, got " + r.shift, r.shift > 0); + assertEquals("aligned lo means no dead prefix", 0, r.deadPrefixBytes); + assertEquals(3, r.chunkCount); + assertEquals(3L * L, r.dataLength); + assertRangeInvariants("intOverflow lo=" + alignedLo + " L=" + L, + alignedLo, alignedLo + 3L * L, L, r); + + // and one byte in from the alignment, so dead prefix and shift are both large + ChunkRange off = ZeroCopySSTableSplitter.chunkRange(alignedLo + 1, alignedLo + 1 + L, L); + assertEquals(alignedLo, off.shift); + assertEquals(1, off.deadPrefixBytes); + assertEquals(2, off.chunkCount); + assertEquals(L + 1, off.dataLength); + assertRangeInvariants("intOverflow+1 lo=" + (alignedLo + 1) + " L=" + L, + alignedLo + 1, alignedLo + 1 + L, L, off); + } + + /** Nothing anywhere in the arithmetic may go negative for very large but legal inputs. */ + @Test + public void veryLargePositionsStayPositive() + { + long huge = 1L << 45; // 32 TiB + for (int L : REAL_CHUNK_LENGTHS) + { + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(huge + 7, huge + 7 + 2L * L, L); + assertTrue(r.firstChunk > 0); + assertTrue(r.lastChunk >= r.firstChunk); + assertTrue(r.chunkCount > 0); + assertTrue(r.dataLength > 0); + assertTrue(r.shift > 0); + assertTrue(r.deadPrefixBytes >= 0); + assertRangeInvariants("huge L=" + L, huge + 7, huge + 7 + 2L * L, L, r); + } + } + + // ------------------------------------------------------------------------------------------------ + // Adjacent children and the shared boundary chunk + // ------------------------------------------------------------------------------------------------ + + @Test + public void adjacentChildrenShareTheBoundaryChunkOnlyWhenUnaligned() + { + for (int L : REAL_CHUNK_LENGTHS) + { + // unaligned boundary -> the chunk containing it is copied into BOTH children + assertBoundary(0, 3L * L + 17, 7L * L, L, true); + assertBoundary(L / 3, 3L * L + 1, 4L * L, L, true); + assertBoundary(0, L - 1, 2L * L, L, true); + + // aligned boundary -> no shared chunk + assertBoundary(0, 3L * L, 7L * L, L, false); + assertBoundary(L / 3, 4L * L, 9L * L + 5, L, false); + assertBoundary(0, L, 2L * L, L, false); + } + } + + @Test + public void sharedBoundaryChunkSweep() + { + long seed = SEED + 1; + Random rnd = new Random(seed); + try + { + for (int L : REAL_CHUNK_LENGTHS) + { + for (int t = 0; t < 5000; t++) + { + long lo = nextLong(rnd, 1L << 34); + long mid = lo + 1 + nextLong(rnd, 4L * L); + long hi = mid + 1 + nextLong(rnd, 4L * L); + assertBoundary(lo, mid, hi, L, mid % L != 0); + } + } + } + catch (AssertionError | RuntimeException e) + { + throw new AssertionError("boundary sweep failed with seed=" + seed + ": " + e, e); + } + } + + /** + * A run of adjacent children partitions the parent, so every child's {@code hi} is the next child's + * {@code lo}. Verify the whole chain: no gaps, no negative overlap, and duplication bounded by exactly + * one chunk per unaligned interior boundary. + */ + @Test + public void chainOfAdjacentChildrenCoversTheParentExactly() + { + for (int L : REAL_CHUNK_LENGTHS) + { + long[] cuts = { 0, L / 2, L, 3L * L, 3L * L + 1, 6L * L, 6L * L + L - 1, 10L * L }; + List ranges = new ArrayList<>(); + for (int i = 0; i + 1 < cuts.length; i++) + ranges.add(ZeroCopySSTableSplitter.chunkRange(cuts[i], cuts[i + 1], L)); + + long liveBytes = 0; + long shared = 0; + for (int i = 0; i < ranges.size(); i++) + { + ChunkRange r = ranges.get(i); + assertRangeInvariants("chain[" + i + "] L=" + L, cuts[i], cuts[i + 1], L, r); + liveBytes += r.dataLength - r.deadPrefixBytes; + + if (i > 0) + { + ChunkRange prev = ranges.get(i - 1); + assertEquals("children must be contiguous", prev.hi, r.lo); + assertTrue("chunk runs must be non-decreasing", r.firstChunk >= prev.lastChunk); + if (r.lo % L == 0) + { + assertEquals("aligned boundary must not share a chunk", + prev.lastChunk + 1, r.firstChunk); + assertEquals(0, r.deadPrefixBytes); + } + else + { + assertEquals("unaligned boundary must share exactly one chunk", + prev.lastChunk, r.firstChunk); + assertEquals(r.lo % L, r.deadPrefixBytes); + shared++; + } + } + } + assertEquals("the chain's live bytes must equal the parent's span", + cuts[cuts.length - 1] - cuts[0], liveBytes); + assertTrue("shared chunks are bounded by the number of interior boundaries", + shared <= ranges.size() - 1); + assertTrue("this fixture is supposed to contain unaligned boundaries", shared > 0); + } + } + + private static void assertBoundary(long lo, long mid, long hi, int L, boolean expectShared) + { + ChunkRange left = ZeroCopySSTableSplitter.chunkRange(lo, mid, L); + ChunkRange right = ZeroCopySSTableSplitter.chunkRange(mid, hi, L); + assertRangeInvariants("left L=" + L + " [" + lo + ',' + mid + ')', lo, mid, L, left); + assertRangeInvariants("right L=" + L + " [" + mid + ',' + hi + ')', mid, hi, L, right); + + String ctx = "L=" + L + " lo=" + lo + " mid=" + mid + " hi=" + hi + " mid%L=" + (mid % L); + assertEquals("expectShared must match alignment: " + ctx, mid % L != 0, expectShared); + + if (expectShared) + { + assertEquals("unaligned boundary shares the chunk: " + ctx, left.lastChunk, right.firstChunk); + assertTrue("the shared chunk gives the right child a dead prefix: " + ctx, + right.deadPrefixBytes > 0); + } + else + { + assertEquals("aligned boundary shares no chunk: " + ctx, left.lastChunk + 1, right.firstChunk); + assertEquals("aligned boundary means no dead prefix: " + ctx, 0, right.deadPrefixBytes); + assertNotEquals(left.lastChunk, right.firstChunk); + } + + // in both cases the right child's shift never goes backwards and the left child's run ends + // at or after the byte before the boundary + assertTrue(right.shift >= left.shift); + assertEquals(right.lo - right.shift, right.deadPrefixBytes); + assertTrue("left child must contain the byte before the boundary", + left.lastChunk == (mid - 1) / L); + } + + // ------------------------------------------------------------------------------------------------ + // ChunkRange value semantics + // ------------------------------------------------------------------------------------------------ + + @Test + public void chunkRangeIsAValue() + { + ChunkRange a = ZeroCopySSTableSplitter.chunkRange(1234, 98765, L4); + ChunkRange b = ZeroCopySSTableSplitter.chunkRange(1234, 98765, L4); + ChunkRange c = ZeroCopySSTableSplitter.chunkRange(1234, 98766, L4); + ChunkRange d = ZeroCopySSTableSplitter.chunkRange(1234, 98765, L16); + + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + assertNotEquals(a, c); + assertNotEquals(a, d); + assertFalse(a.equals(null)); + assertFalse(a.equals("not a ChunkRange")); + assertTrue(a.toString().contains("lo=1234")); + } + + // ------------------------------------------------------------------------------------------------ + // chooseByByteShare -- also pure arithmetic (package-visible test hook) + // ------------------------------------------------------------------------------------------------ + + @Test + public void chooseByByteShareOnAUniformLayout() + { + long[] positions = new long[10]; + for (int i = 0; i < positions.length; i++) + positions[i] = i * 100L; + long uncompressedLength = 1000; + + assertArrayEqualsInt(new int[]{ 0 }, + ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, 1)); + assertArrayEqualsInt(new int[]{ 0, 5 }, + ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, 2)); + assertArrayEqualsInt(new int[]{ 0, 3, 7 }, + ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, 3)); + assertArrayEqualsInt(new int[]{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, 10)); + } + + @Test + public void chooseByByteShareNeverEmitsAnEmptyRun() + { + long seed = SEED + 2; + Random rnd = new Random(seed); + try + { + for (int t = 0; t < 2000; t++) + { + int n = 1 + (int) nextLong(rnd, 200); + long[] positions = new long[n]; + long p = nextLong(rnd, 1L << 20); + for (int i = 0; i < n; i++) + { + positions[i] = p; + p += 1 + nextLong(rnd, 100000); + } + long uncompressedLength = p + 1; + int numChildren = 1 + (int) nextLong(rnd, n); + + int[] runStarts = ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, numChildren); + + String ctx = "n=" + n + " numChildren=" + numChildren; + assertEquals(ctx, numChildren, runStarts.length); + assertEquals(ctx + " first run must start at 0", 0, runStarts[0]); + for (int m = 1; m < numChildren; m++) + { + assertTrue(ctx + " runs must be strictly increasing at " + m, + runStarts[m] > runStarts[m - 1]); + assertTrue(ctx + " run start out of range at " + m, + runStarts[m] >= 0 && runStarts[m] < n); + } + assertTrue(ctx + " the last run must be non-empty", runStarts[numChildren - 1] <= n - 1); + } + } + catch (AssertionError | RuntimeException e) + { + throw new AssertionError("chooseByByteShare sweep failed with seed=" + seed + ": " + e, e); + } + } + + /** + * The load-bearing test for the streaming selector. + * + *

{@code RunSelector} exists because materialising a {@code long} per partition is a hard ceiling on how + * large an sstable can be split -- a terabyte of small partitions is tens of gigabytes of heap for an array + * whose every access is sequential. It is also much harder to read than {@link + * ZeroCopySSTableSplitter#chooseByByteShare}, which is kept precisely so that this test can assert the two + * agree exactly, run start for run start, on randomised layouts. Anything that makes them disagree is a + * regression in the streaming version, not a new policy. + * + *

The sweep is shaped to hit the two clamps that are the whole difficulty, because they are the only + * places the array version reaches somewhere other than the cursor: + *

    + *
  • {@code numChildren == n} and near it, which forces the tail-room clamp on nearly every run;
    • + *
  • partitions large enough that one of them spans several byte-share targets, which forces the + * non-empty clamp and, with it, the deferred offset resolution.
    • + *
+ */ + @Test + public void runSelectorAgreesWithChooseByByteShare() + { + long seed = SEED + 7; + Random rnd = new Random(seed); + String ctx = ""; + try + { + for (int t = 0; t < 4000; t++) + { + int n = 1 + (int) nextLong(rnd, 120); + // A mix of tiny and huge partitions: a partition wider than total/numChildren is what forces + // several targets onto one record, hence the non-empty clamp. + boolean lumpy = (t % 3) == 0; + long[] positions = new long[n]; + long p = nextLong(rnd, 1L << 20); + for (int i = 0; i < n; i++) + { + positions[i] = p; + p += 1 + nextLong(rnd, lumpy && (i % 7) == 0 ? 5_000_000 : 1000); + } + long uncompressedLength = p + 1 + nextLong(rnd, 1000); + + // exercise the extremes as well as the middle + int numChildren; + if (t % 4 == 0) + numChildren = n; // every run on the tail-room clamp + else if (t % 4 == 1) + numChildren = Math.max(1, n - 1); + else if (t % 4 == 2) + numChildren = 1; + else + numChildren = 1 + (int) nextLong(rnd, n); + + ctx = "n=" + n + " numChildren=" + numChildren + " lumpy=" + lumpy; + int[] expected = ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, numChildren); + + ZeroCopySSTableSplitter.RunSelector selector = + new ZeroCopySSTableSplitter.RunSelector(uncompressedLength, numChildren, n); + for (int i = 0; i < n; i++) + selector.offer(i, positions[i]); + ZeroCopySSTableSplitter.Runs runs = selector.finish(); + + assertArrayEquals(ctx, expected, runs.runStarts); + assertEquals(ctx, n, runs.partitionCount); + + // and the offsets it carries have to be the ones those run starts point at, since build() takes + // every child's lo straight from them + for (int m = 0; m < numChildren; m++) + assertEquals(ctx + " offset of run " + m, positions[expected[m]], runs.runPositions[m]); + } + } + catch (AssertionError | RuntimeException e) + { + throw new AssertionError("RunSelector sweep failed with seed=" + seed + " at " + ctx + ": " + e, e); + } + } + + /** + * Byte shares chosen by {@code chooseByByteShare} must be consumable by {@code chunkRange}: every run is + * non-empty, so every {@code [lo, hi)} it implies is a legal child. + */ + @Test + public void chooseByByteShareProducesLegalChunkRanges() + { + int n = 137; + long[] positions = new long[n]; + long p = 0; + Random rnd = new Random(SEED + 3); + for (int i = 0; i < n; i++) + { + positions[i] = p; + p += 1 + nextLong(rnd, 40000); + } + long uncompressedLength = p + 1; + + for (int numChildren = 1; numChildren <= 16; numChildren++) + { + int[] runStarts = ZeroCopySSTableSplitter.chooseByByteShare(positions, uncompressedLength, numChildren); + for (int L : REAL_CHUNK_LENGTHS) + { + long previousHi = -1; + for (int b = 0; b < runStarts.length; b++) + { + int from = runStarts[b]; + int to = (b + 1 < runStarts.length) ? runStarts[b + 1] : n; + assertTrue("empty run " + b, from < to); + long lo = positions[from]; + long hi = (to < n) ? positions[to] : uncompressedLength; + if (previousHi >= 0) + assertEquals("runs must be contiguous", previousHi, lo); + previousHi = hi; + ChunkRange r = ZeroCopySSTableSplitter.chunkRange(lo, hi, L); + assertRangeInvariants("share K=" + numChildren + " b=" + b + " L=" + L, lo, hi, L, r); + } + assertEquals("the runs must cover the whole parent", uncompressedLength, previousHi); + } + } + } + + // ------------------------------------------------------------------------------------------------ + // copyPlan: the physical half, i.e. the alignment extent sharing needs + // ------------------------------------------------------------------------------------------------ + + /** Without alignment the plan must be exactly what the splitter did before extent sharing existed. */ + @Test + public void copyPlanWithoutAlignmentIsTheOldBehaviour() + { + for (long copyFrom : new long[]{ 0, 1, 4095, A, A + 1, 3 * A - 7, 1L << 40, (1L << 40) + 12345 }) + { + for (long physical : new long[]{ 1, 4096, A - 1, A, A + 1, 1 << 20, 3L << 30 }) + { + CopyPlan plan = ZeroCopySSTableSplitter.copyPlan(copyFrom, physical, false, false); + String ctx = "from=" + copyFrom + " physical=" + physical; + assertEquals(ctx + " srcStart", copyFrom, plan.srcStart); + assertEquals(ctx + " pad", 0, plan.headPadBytes); + assertEquals(ctx + " childLength", physical, plan.childLength); + assertEquals(ctx + " cloneLength", 0, plan.cloneLength); + assertEquals(ctx + " tailLength", physical, plan.tailLength()); + } + } + } + + /** + * The three properties the ioctl actually demands, over every residue of the alignment: the source offset + * is aligned, the destination offset is aligned (it is always 0), and the cloned length is aligned. Plus + * the two the format demands: the child's byte 0 comes from at or before {@code O(i)}, and the pad is + * exactly the distance between them. + */ + @Test + public void copyPlanAlignsEveryResidue() + { + Random rnd = new Random(SEED + 11); + for (int trial = 0; trial < 20000; trial++) + { + // A base far enough out that a 32-bit intermediate would have overflowed long ago + long copyFrom = trial < A ? trial : (1L << 42) + nextLong(rnd, 1L << 30); + long physical = 1 + nextLong(rnd, 1L << 26); + CopyPlan plan = ZeroCopySSTableSplitter.copyPlan(copyFrom, physical, true, true); + String ctx = "from=" + copyFrom + " physical=" + physical + ' ' + plan; + + assertEquals(ctx + " -- srcStart must be alignment aligned", 0, plan.srcStart % A); + assertEquals(ctx + " -- cloneLength must be alignment aligned", 0, plan.cloneLength % A); + assertEquals(ctx + " -- pad is the distance from srcStart to O(i)", + copyFrom - plan.srcStart, plan.headPadBytes); + assertTrue(ctx + " -- pad must be under one alignment unit", plan.headPadBytes < A); + assertTrue(ctx + " -- srcStart must not overshoot O(i)", plan.srcStart <= copyFrom); + assertEquals(ctx + " -- childLength", plan.headPadBytes + physical, plan.childLength); + + // the clone must never read past the child's last live byte, i.e. into the parent's trailing slack + assertTrue(ctx + " -- clone overruns the run", plan.cloneLength <= plan.childLength); + assertTrue(ctx + " -- tail must be under one alignment unit", plan.tailLength() < A); + assertEquals(ctx + " -- clone + tail must cover the child exactly", + plan.childLength, plan.cloneLength + plan.tailLength()); + // and the range read from the parent is exactly [srcStart, O(i) + physical) + assertEquals(ctx + " -- range end", copyFrom + physical, plan.srcStart + plan.childLength); + } + } + + /** + * Aligning without sharing is what a test does on a filesystem that cannot share extents: identical layout, + * nothing cloned. The layout has to be independent of the mechanism or that test proves nothing. + */ + @Test + public void copyPlanCanAlignWithoutCloning() + { + for (long copyFrom : new long[]{ 0, 1, 999, A - 1, A, A + 1, 5 * A + 4097 }) + { + CopyPlan shared = ZeroCopySSTableSplitter.copyPlan(copyFrom, 1 << 20, true, true); + CopyPlan copied = ZeroCopySSTableSplitter.copyPlan(copyFrom, 1 << 20, true, false); + String ctx = "from=" + copyFrom; + assertEquals(ctx + " srcStart", shared.srcStart, copied.srcStart); + assertEquals(ctx + " pad", shared.headPadBytes, copied.headPadBytes); + assertEquals(ctx + " childLength", shared.childLength, copied.childLength); + assertEquals(ctx + " nothing cloned", 0, copied.cloneLength); + assertEquals(ctx + " everything tail", copied.childLength, copied.tailLength()); + } + } + + /** A run whose whole length is under one alignment unit has nothing to clone, but still gets its pad. */ + @Test + public void copyPlanBelowOneAlignmentUnitClonesNothing() + { + CopyPlan plan = ZeroCopySSTableSplitter.copyPlan(A + 100, 200, true, true); + assertEquals(A, plan.srcStart); + assertEquals(100, plan.headPadBytes); + assertEquals(300, plan.childLength); + assertEquals(0, plan.cloneLength); + assertEquals(300, plan.tailLength()); + } + + /** Exactly one alignment unit, and one byte either side of it. */ + @Test + public void copyPlanAtTheAlignmentBoundary() + { + // copyFrom already aligned: no pad, and the whole run is cloneable when it is a whole number of units + assertEquals(new CopyPlan(2 * A, 0, 3 * A, 3 * A), + ZeroCopySSTableSplitter.copyPlan(2 * A, 3 * A, true, true)); + // one byte short of a unit: the last (partial) unit is the tail + assertEquals(new CopyPlan(2 * A, 0, 3 * A - 1, 2 * A), + ZeroCopySSTableSplitter.copyPlan(2 * A, 3 * A - 1, true, true)); + // one byte over: the extra byte is the tail + assertEquals(new CopyPlan(2 * A, 0, 3 * A + 1, 3 * A), + ZeroCopySSTableSplitter.copyPlan(2 * A, 3 * A + 1, true, true)); + // pad and tail together, both maximal + assertEquals(new CopyPlan(0, A - 1, 3 * A - 1, 2 * A), + ZeroCopySSTableSplitter.copyPlan(A - 1, 2 * A, true, true)); + } + + @Test + public void copyPlanRejectsNonsense() + { + assertThatThrownBy(() -> ZeroCopySSTableSplitter.copyPlan(-1, 1024, true, true)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("negative copyFrom"); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.copyPlan(0, 0, true, true)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("non-positive physicalBytes"); + assertThatThrownBy(() -> ZeroCopySSTableSplitter.copyPlan(0, -4096, false, false)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("non-positive physicalBytes"); + } + + // ------------------------------------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------------------------------------ + + /** + * Recompute every field of a {@link ChunkRange} independently of the implementation and cross-check the + * standalone helpers against it. + */ + private static void assertRangeInvariants(String ctx, long lo, long hi, int L, ChunkRange r) + { + long i = lo / L; + long j = (hi - 1) / L; + long c = j - i + 1; + long dp = hi - i * (long) L; + long shift = i * (long) L; + long dead = lo % L; + + assertEquals(ctx + " lo", lo, r.lo); + assertEquals(ctx + " hi", hi, r.hi); + assertEquals(ctx + " chunkLength", L, r.chunkLength); + assertEquals(ctx + " firstChunk", i, r.firstChunk); + assertEquals(ctx + " lastChunk", j, r.lastChunk); + assertEquals(ctx + " chunkCount", c, r.chunkCount); + assertEquals(ctx + " dataLength", dp, r.dataLength); + assertEquals(ctx + " shift", shift, r.shift); + assertEquals(ctx + " deadPrefixBytes", dead, r.deadPrefixBytes); + + // structural invariants + assertTrue(ctx + " firstChunk <= lastChunk", r.firstChunk <= r.lastChunk); + assertTrue(ctx + " chunkCount >= 1", r.chunkCount >= 1); + assertTrue(ctx + " dataLength > 0", r.dataLength > 0); + assertTrue(ctx + " shift >= 0", r.shift >= 0); + assertTrue(ctx + " dead prefix in [0, L)", r.deadPrefixBytes >= 0 && r.deadPrefixBytes < L); + assertTrue(ctx + " shift <= lo", r.shift <= lo); + assertEquals(ctx + " lo - shift == dead", r.deadPrefixBytes, lo - r.shift); + + // the load-bearing invariant: the last chunk holds at least one live byte and no more than a chunk + assertTrue(ctx + " (C-1)*L < Dp [C=" + r.chunkCount + " Dp=" + r.dataLength + ']', + (r.chunkCount - 1) * (long) L < r.dataLength); + assertTrue(ctx + " Dp <= C*L [C=" + r.chunkCount + " Dp=" + r.dataLength + ']', + r.dataLength <= r.chunkCount * (long) L); + + // the child's live span is exactly the parent's + assertEquals(ctx + " Dp - dead == hi - lo", hi - lo, r.dataLength - r.deadPrefixBytes); + + // the standalone helpers must agree with the aggregate + assertEquals(ctx + " firstChunk()", r.firstChunk, ZeroCopySSTableSplitter.firstChunk(lo, L)); + assertEquals(ctx + " lastChunk()", r.lastChunk, ZeroCopySSTableSplitter.lastChunk(hi, L)); + assertEquals(ctx + " childDataLength()", r.dataLength, + ZeroCopySSTableSplitter.childDataLength(hi, r.firstChunk, L)); + assertEquals(ctx + " deadPrefixBytes()", r.deadPrefixBytes, + ZeroCopySSTableSplitter.deadPrefixBytes(lo, L)); + assertEquals(ctx + " chunkIndexFor(lo)", r.firstChunk, ZeroCopySSTableSplitter.chunkIndexFor(lo, L)); + assertEquals(ctx + " chunkIndexFor(hi-1)", r.lastChunk, + ZeroCopySSTableSplitter.chunkIndexFor(hi - 1, L)); + + // and the value is reproducible + assertEquals(ctx + " reproducible", r, ZeroCopySSTableSplitter.chunkRange(lo, hi, L)); + } + + private static void assertArrayEqualsInt(int[] expected, int[] actual) + { + assertEquals("length " + java.util.Arrays.toString(actual), expected.length, actual.length); + for (int i = 0; i < expected.length; i++) + assertEquals("index " + i + " of " + java.util.Arrays.toString(actual), expected[i], actual[i]); + } + + /** Uniform in [0, bound). {@code Math.floorMod} keeps it non-negative even for {@code Long.MIN_VALUE}. */ + private static long nextLong(Random rnd, long bound) + { + return Math.floorMod(rnd.nextLong(), bound); + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterFuzzTest.java b/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterFuzzTest.java new file mode 100644 index 000000000000..99d95d30b457 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterFuzzTest.java @@ -0,0 +1,1291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.zip.CRC32; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.Config.FlushCompression; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Randomised end-to-end test of {@link ZeroCopySSTableSplitter}. + *

+ * The oracle never changes: the concatenation of the children, read in order, must equal the parent, + * partition for partition and unfiltered for unfiltered. Everything else that is asserted (chunk + * arithmetic, physical file lengths, rebased index positions, the digest) is a cross-check derived + * independently from the parent's own {@code CompressionMetadata} and {@code Index.db}, not from the values + * the splitter reports. + * + *

Reproducing a failure

+ * Every iteration derives its own seed and every assertion message carries the whole configuration plus that + * seed. To replay exactly one failing iteration, and nothing else: + * 
+ *   ant testsome -Duse.jdk11=true \
+ *       -Dtest.name=org.apache.cassandra.io.sstable.ZeroCopySSTableSplitterFuzzTest \
+ *       -Dtest.methods=fuzz \
+ *       -Dcassandra.test.zerocopysplitter.replaySeed=<seed from the failure message>
+ *
+ * For a long soak run, raise the iteration count and/or move the base seed: + * 
+ *   -Dcassandra.test.zerocopysplitter.iterations=500
+ *   -Dcassandra.test.zerocopysplitter.seed=12345
+ *
+ * The default of {@value #DEFAULT_ITERATIONS} iterations is deliberately modest so this stays inside a normal + * unit-test run. + */ +public class ZeroCopySSTableSplitterFuzzTest extends CQLTester +{ + private static final Logger logger = LoggerFactory.getLogger(ZeroCopySSTableSplitterFuzzTest.class); + + private static final String PROP_SEED = "cassandra.test.zerocopysplitter.seed"; + private static final String PROP_ITERATIONS = "cassandra.test.zerocopysplitter.iterations"; + private static final String PROP_REPLAY_SEED = "cassandra.test.zerocopysplitter.replaySeed"; + + static final int DEFAULT_ITERATIONS = 24; + + private static final long BASE_SEED = Long.getLong(PROP_SEED, 20260726_0001L); + private static final int ITERATIONS = Integer.getInteger(PROP_ITERATIONS, DEFAULT_ITERATIONS); + /** When set, exactly one iteration runs, with this literal seed. */ + private static final Long REPLAY_SEED = Long.getLong(PROP_REPLAY_SEED); + + /** Explicit insert timestamps keep the on-disk layout stable across runs of the same seed. */ + private static final long PAST_TS = 1_600_000_000_000_000L; + /** Strictly greater than any wall-clock timestamp a DELETE will get during this test. */ + private static final long FUTURE_TS = 2_000_000_000_000_000L; + + /** Keep a single iteration's sstable small enough that one flush is one sstable. */ + private static final long MAX_TABLE_BYTES = 1_200_000L; + + private static final String[] COMPRESSORS = { "LZ4Compressor", "SnappyCompressor", "DeflateCompressor", + "ZstdCompressor", null /* uncompressed: must be refused */ }; + private static final int[] CHUNK_KB = { 4, 8, 16, 32, 64 }; + /** 0 disables the raw-chunk fallback; > 1 makes most chunks store raw, which is the sharpest edge case. */ + private static final double[] MIN_COMPRESS_RATIO = { 0.0, 0.0, 1.0, 2.0, 8.0 }; + private static final int[] COLUMN_INDEX_KB = { 1, 2, 4, 16, 64 }; + private static final int[] COLUMN_INDEX_CACHE_KB = { 0, 2, 99999 }; + + // ------------------------------------------------------------------------------------------------ + // Tests + // ------------------------------------------------------------------------------------------------ + + @Test + public void fuzz() throws Throwable + { + int savedIndexSize = DatabaseDescriptor.getColumnIndexSizeInKiB(); + int savedCacheSize = DatabaseDescriptor.getColumnIndexCacheSizeInKiB(); + try + { + if (REPLAY_SEED != null) + { + logger.info("Replaying a single ZeroCopySSTableSplitter fuzz iteration, seed {}", REPLAY_SEED); + runGuarded(REPLAY_SEED); + return; + } + + logger.info("ZeroCopySSTableSplitter fuzz: {} iterations from base seed {}", ITERATIONS, BASE_SEED); + for (int i = 0; i < ITERATIONS; i++) + runGuarded(scramble(BASE_SEED + i)); + } + finally + { + DatabaseDescriptor.setColumnIndexSize(savedIndexSize); + DatabaseDescriptor.setColumnIndexCacheSize(savedCacheSize); + } + } + + /** + * The deliberately adversarial generator: every partition is calibrated to be exactly {@code L}, + * {@code L - 1} or {@code L + 1} bytes, so partition boundaries land exactly on, one byte before, and one + * byte after a compression-chunk boundary. That is precisely where {@code (hi-1)/L} vs {@code hi/L}, + * {@code lo mod L} and the {@code O(j+1) - O(i)} physical length are most likely to be off by one. + *

+ * Splitting at every partition maximises the number of such boundaries exercised. + */ + @Test + public void straddlesChunkBoundaries() throws Throwable + { + int savedIndexSize = DatabaseDescriptor.getColumnIndexSizeInKiB(); + int savedCacheSize = DatabaseDescriptor.getColumnIndexCacheSizeInKiB(); + try + { + // partitions are one row each, so no promoted index; keep the grid coarse and predictable + DatabaseDescriptor.setColumnIndexSize(64); + DatabaseDescriptor.setColumnIndexCacheSize(2); + + long seed = REPLAY_SEED != null ? REPLAY_SEED : BASE_SEED; + int overhead = calibrateOverhead(new Random(seed)); + boolean anyConverged = false; + + for (int chunkKb : new int[]{ 4, 16 }) + { + for (int delta : new int[]{ -1, 0, 1 }) + { + long scenarioSeed = scramble(seed + chunkKb * 1000L + delta); + try + { + anyConverged |= runStraddleScenario(chunkKb, delta, overhead, scenarioSeed); + } + catch (Throwable t) + { + throw new AssertionError(String.format("straddle scenario FAILED: chunkKb=%d delta=%d " + + "overhead=%d seed=%d%n%s", + chunkKb, delta, overhead, scenarioSeed, + replayHint(scenarioSeed, "straddlesChunkBoundaries")), t); + } + } + } + + assertTrue("the adversarial generator never converged on an exact partition size; it is no longer " + + "producing chunk-straddling partitions and this test has silently stopped testing anything", + anyConverged); + } + finally + { + DatabaseDescriptor.setColumnIndexSize(savedIndexSize); + DatabaseDescriptor.setColumnIndexCacheSize(savedCacheSize); + } + } + + /** + * The implementation refuses an uncompressed parent rather than emitting a child with a misaligned CRC.db. + * If that ever changes this test is the reminder to extend the fuzz loop to cover it. + */ + @Test + public void uncompressedParentIsRefused() throws Throwable + { + createTable("CREATE TABLE %s (pk text PRIMARY KEY, v blob) WITH compression = {'enabled': 'false'}"); + disableCompaction(); + for (int i = 0; i < 8; i++) + execute("INSERT INTO %s (pk, v) VALUES (?, ?) USING TIMESTAMP ?", + String.format("p%05d", i), ByteBuffer.wrap(new byte[512]), PAST_TS + i); + flush(); + + SSTableReader parent = onlySSTable(getCurrentColumnFamilyStore(), "uncompressed refusal"); + assertFalse("an uncompressed sstable must not be reported as supported", parent.compression); + assertFalse("isSupported() must be false for an uncompressed parent", + ZeroCopySSTableSplitter.isSupported(parent)); + + try + { + ZeroCopySSTableSplitter.split(parent, 2, null); + fail("expected UnsupportedOperationException for an uncompressed parent"); + } + catch (UnsupportedOperationException e) + { + assertTrue("refusal message must start with the public constant, got: " + e.getMessage(), + e.getMessage().startsWith(ZeroCopySSTableSplitter.UNCOMPRESSED_UNSUPPORTED_MESSAGE)); + } + } + + // ------------------------------------------------------------------------------------------------ + // One fuzz iteration + // ------------------------------------------------------------------------------------------------ + + private void runGuarded(long seed) throws Throwable + { + Config cfg = new Config(seed); + try + { + runIteration(cfg); + } + catch (Throwable t) + { + throw new AssertionError("ZeroCopySSTableSplitter fuzz iteration FAILED\n" + cfg + '\n' + + replayHint(seed, "fuzz"), t); + } + } + + private static String replayHint(long seed, String method) + { + return "replay this case alone with:\n" + + " ant testsome -Duse.jdk11=true" + + " -Dtest.name=org.apache.cassandra.io.sstable.ZeroCopySSTableSplitterFuzzTest" + + " -Dtest.methods=" + method + + " -D" + PROP_REPLAY_SEED + '=' + seed; + } + + private void runIteration(Config cfg) throws Throwable + { + Random rnd = new Random(cfg.seed); + + cfg.compressor = COMPRESSORS[rnd.nextInt(COMPRESSORS.length)]; + cfg.chunkKb = CHUNK_KB[rnd.nextInt(CHUNK_KB.length)]; + cfg.minCompressRatio = MIN_COMPRESS_RATIO[rnd.nextInt(MIN_COMPRESS_RATIO.length)]; + cfg.columnIndexKb = COLUMN_INDEX_KB[rnd.nextInt(COLUMN_INDEX_KB.length)]; + cfg.columnIndexCacheKb = COLUMN_INDEX_CACHE_KB[rnd.nextInt(COLUMN_INDEX_CACHE_KB.length)]; + cfg.clusterings = rnd.nextInt(3); // 0, 1 or 2 clustering columns + cfg.reverse0 = cfg.clusterings >= 1 && rnd.nextBoolean(); + cfg.reverse1 = cfg.clusterings >= 2 && rnd.nextBoolean(); + cfg.hasStatic = cfg.clusterings >= 1 && rnd.nextBoolean(); + cfg.hasMap = rnd.nextBoolean(); + cfg.hasSet = rnd.nextBoolean(); + + DatabaseDescriptor.setColumnIndexSize(cfg.columnIndexKb); + DatabaseDescriptor.setColumnIndexCacheSize(cfg.columnIndexCacheKb); + + int chunkLength = cfg.chunkKb * 1024; + // big chunks + big partitions would blow the byte budget; scale the partition count to compensate + cfg.partitions = cfg.chunkKb >= 32 ? 6 + rnd.nextInt(10) : 8 + rnd.nextInt(30); + + // flush_compression defaults to `fast`, which silently replaces any compressor that does not + // advertise FAST_COMPRESSION with CompressionParams.DEFAULT -- LZ4 at 16 KiB (BigTableWriter.java:127-151). + // Without this the whole compressor/chunk-length matrix below would be a no-op for every + // non-LZ4 iteration and the fuzz would only ever exercise one configuration. + DatabaseDescriptor.setFlushCompression(FlushCompression.table); + + createTable(ddl(cfg)); + disableCompaction(); + writeRandomData(cfg, rnd, chunkLength); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs, cfg.toString()); + + if (cfg.compressor == null) + { + // the uncompressed variant of the fuzz: assert the refusal, exactly and every time + assertFalse("uncompressed parent reported as supported", ZeroCopySSTableSplitter.isSupported(parent)); + try + { + ZeroCopySSTableSplitter.split(parent, 2, null); + fail("expected UnsupportedOperationException for an uncompressed parent"); + } + catch (UnsupportedOperationException e) + { + assertTrue("bad refusal message: " + e.getMessage(), + e.getMessage().startsWith(ZeroCopySSTableSplitter.UNCOMPRESSED_UNSUPPORTED_MESSAGE)); + } + return; + } + + assertTrue("compressed parent reported as unsupported", ZeroCopySSTableSplitter.isSupported(parent)); + ParentIndex index = readIndex(parent); + assertEquals("the generator did not write one partition per pk", cfg.partitions, index.size()); + cfg.parentPartitions = index.size(); + cfg.parentUncompressedLength = parent.uncompressedLength(); + cfg.parentChunkLength = parent.getCompressionMetadata().chunkLength(); + assertEquals("the table's chunk_length_in_kb did not survive to the sstable", + chunkLength, cfg.parentChunkLength); + + // ---- split-point selection ------------------------------------------------------------------ + boolean byKeys = rnd.nextBoolean(); + cfg.splitMode = byKeys ? "boundaries" : "numChildren"; + cfg.useTxn = rnd.nextBoolean(); + // Half the iterations use the ALIGNED layout, in which every child's Data.db starts with up to 64 KiB + // of the parent's previous chunk so that its extents could be shared with the parent by FICLONERANGE. + // Forced rather than left to the filesystem: no CI box can share extents, and the layout is the part + // that has to survive every compressor, chunk length and raw-chunk threshold in this matrix. A padded + // range that is copied instead of shared produces a byte-identical child, so this covers the layout + // fully and the ioctl not at all. + cfg.alignedLayout = rnd.nextBoolean(); + ZeroCopySSTableSplitter.forceAlignedLayoutForTesting = cfg.alignedLayout; + // ...and a quarter of them skip Digest.crc32 entirely, which is a supported configuration and therefore + // has to hold for every compressor, chunk length and raw-chunk threshold in the matrix, not just for the + // one case a dedicated test would pick. + cfg.writeDigest = rnd.nextInt(4) != 0; + DatabaseDescriptor.setZeroCopySplitDigestEnabled(cfg.writeDigest); + + LifecycleTransaction txn = cfg.useTxn ? LifecycleTransaction.offline(OperationType.UNKNOWN) : null; + ZeroCopySSTableSplitter.Result result = null; + try + { + int[] expectedRunStarts; + if (byKeys) + { + cfg.boundaryIndices = pickBoundaryIndices(rnd, index, chunkLength); + List boundaries = new ArrayList<>(cfg.boundaryIndices.length); + for (int idx : cfg.boundaryIndices) + boundaries.add(index.keys[idx]); + // boundary keys are existing keys, so run b+1 starts exactly at that key's record index + expectedRunStarts = cfg.boundaryIndices; + result = ZeroCopySSTableSplitter.split(parent, boundaries, txn); + } + else + { + cfg.numChildren = 1 + rnd.nextInt(Math.min(6, index.size())); + expectedRunStarts = null; + result = ZeroCopySSTableSplitter.split(parent, cfg.numChildren, txn); + } + + cfg.actualChildren = result.children.size(); + verify(parent, index, result, cfg, expectedRunStarts); + } + finally + { + ZeroCopySSTableSplitter.forceAlignedLayoutForTesting = false; + DatabaseDescriptor.setZeroCopySplitDigestEnabled(true); + releaseChildren(result); + if (txn != null) + { + // closing an unfinished offline transaction aborts it, which deletes everything trackNew'd on + // it: that doubles as this iteration's cleanup and proves trackNew really registered the + // children with the LogTransaction. + try + { + txn.close(); + } + catch (Throwable t) + { + logger.warn("failed aborting the split transaction", t); + } + LifecycleTransaction.waitForDeletions(); + } + deleteChildFiles(result); + } + } + + // ------------------------------------------------------------------------------------------------ + // Verification: the oracle plus independent structural cross-checks + // ------------------------------------------------------------------------------------------------ + + private void verify(SSTableReader parent, + ParentIndex index, + ZeroCopySSTableSplitter.Result result, + Config cfg, + int[] expectedRunStarts) throws Exception + { + String ctx = cfg.toString(); + List children = result.children; + assertFalse(ctx + " -- split produced no children", children.isEmpty()); + + // The runs the split is REQUIRED to produce, derived here from the boundary indices alone. Empty runs + // (a boundary at record 0, or two boundaries resolving to the same record) must yield no child. + List expectedRuns = null; + if (expectedRunStarts == null) + { + assertEquals(ctx + " -- chooseByByteShare must always produce exactly numChildren non-empty runs", + cfg.numChildren, children.size()); + } + else + { + expectedRuns = new ArrayList<>(); + int previous = 0; + for (int start : expectedRunStarts) + { + if (start > previous) + expectedRuns.add(new int[]{ previous, start }); + previous = start; + } + if (index.size() > previous) + expectedRuns.add(new int[]{ previous, index.size() }); + assertEquals(ctx + " -- wrong number of children for boundaries " + + Arrays.toString(expectedRunStarts), expectedRuns.size(), children.size()); + } + + CompressionMetadata parentMeta = parent.getCompressionMetadata(); + int chunkLength = parentMeta.chunkLength(); + int parentChunkCount = (int) ((parentMeta.dataLength + chunkLength - 1) / chunkLength); + + long physicalSum = 0; + long deadSum = 0; + long partitionSum = 0; + int cursor = 0; + + for (int b = 0; b < children.size(); b++) + { + ZeroCopySSTableSplitter.Child child = children.get(b); + String cctx = ctx + " -- child " + b + '/' + children.size() + ' ' + child; + + // an empty child is not representable: IndexSummaryBuilder.build and getPositionsForRanges both assert + assertTrue(cctx + " -- empty child", child.partitionCount > 0); + assertTrue(cctx + " -- claims more partitions than remain in the parent", + cursor + child.partitionCount <= index.size()); + int from = cursor; + int to = (int) (cursor + child.partitionCount); + cursor = to; + + if (expectedRuns != null) + { + assertEquals(cctx + " -- child does not start at the requested boundary " + + Arrays.toString(expectedRunStarts), expectedRuns.get(b)[0], from); + assertEquals(cctx + " -- child does not end at the requested boundary " + + Arrays.toString(expectedRunStarts), expectedRuns.get(b)[1], to); + } + + assertEquals(cctx + " -- wrong first key", index.keys[from], child.first); + assertEquals(cctx + " -- wrong last key", index.keys[to - 1], child.last); + + // ---- chunk arithmetic, recomputed from the parent, not read back from the child ---- + long lo = index.positions[from]; + long hi = to < index.size() ? index.positions[to] : parentMeta.dataLength; + ZeroCopySSTableSplitter.ChunkRange expected = ZeroCopySSTableSplitter.chunkRange(lo, hi, chunkLength); + assertEquals(cctx + " -- firstChunk for [" + lo + ',' + hi + ')', expected.firstChunk, child.firstChunk); + assertEquals(cctx + " -- lastChunk for [" + lo + ',' + hi + ')', expected.lastChunk, child.lastChunk); + assertEquals(cctx + " -- dataLength", expected.dataLength, child.dataLength); + assertEquals(cctx + " -- shift", expected.shift, child.shift); + assertEquals(cctx + " -- deadPrefixBytes", expected.deadPrefixBytes, child.deadPrefixBytes); + assertEquals(cctx + " -- deadPrefixBytes must equal lo mod L", lo % chunkLength, child.deadPrefixBytes); + assertTrue(cctx + " -- (C-1)*L < Dp invariant broken", + (expected.chunkCount - 1) * (long) chunkLength < child.dataLength); + assertTrue(cctx + " -- Dp <= C*L invariant broken", + child.dataLength <= expected.chunkCount * (long) chunkLength); + + long copyFrom = chunkOffset(parentMeta, expected.firstChunk, parentChunkCount, chunkLength); + long copyTo = chunkOffset(parentMeta, expected.lastChunk + 1, parentChunkCount, chunkLength); + assertEquals(cctx + " -- physicalBytes must be exactly O(j+1) - O(i)", + copyTo - copyFrom, child.physicalBytes); + + // ---- the child's files on disk ---- + // A child aligned for extent sharing carries a head pad of O(i) mod 64 KiB, and its physical + // lengths are all measured from there rather than from 0. Zero on a filesystem that cannot share. + long pad = child.headPadBytes; + assertTrue(cctx + " -- head pad must be under one alignment unit", pad >= 0 && pad < 64 * 1024); + assertTrue(cctx + " -- head pad must be O(i) mod alignment, or nothing", + pad == 0 || pad == copyFrom % (64 * 1024)); + // If the layout was forced aligned, the pad is not optional: it is exactly O(i) mod A. Without this + // the forcing could quietly stop working and half the fuzz matrix would test the plain layout twice. + if (cfg.alignedLayout) + assertEquals(cctx + " -- forced aligned layout did not pad", copyFrom % (64 * 1024), pad); + assertEquals(cctx + " -- onDiskLength", pad + child.physicalBytes, child.onDiskLength()); + long onDisk = child.descriptor.fileFor(Component.DATA).length(); + assertEquals(cctx + " -- child Data.db has trailing slack (or is short)", child.onDiskLength(), onDisk); + assertEquals(cctx + " -- child uncompressedLength", child.dataLength, child.reader.uncompressedLength()); + + CompressionMetadata childMeta = new CompressionMetadata(child.descriptor, onDisk); + try + { + assertEquals(cctx + " -- child CompressionInfo dataLength", child.dataLength, childMeta.dataLength); + assertEquals(cctx + " -- child chunkLength", chunkLength, childMeta.chunkLength()); + assertEquals(cctx + " -- child maxCompressedLength", parentMeta.maxCompressedLength(), + childMeta.maxCompressedLength()); + assertEquals(cctx + " -- child offsets[0] must be the head pad", pad, childMeta.chunkFor(0).offset); + // the last chunk must end exactly at the physical end of the file + long lastChunkStart = (long) ((childMeta.dataLength - 1) / chunkLength) * chunkLength; + CompressionMetadata.Chunk lastChunk = childMeta.chunkFor(lastChunkStart); + assertEquals(cctx + " -- child last chunk does not end at EOF", + onDisk, lastChunk.offset + lastChunk.length + 4); + } + finally + { + childMeta.close(); + } + + // Digest.crc32 is optional (zero_copy_split_digest_enabled), and the component set is the authority: + // if it claims the digest the value must be right, and if it does not the file must not exist. + assertEquals(cctx + " -- the digest component must follow the config", + cfg.writeDigest, child.components.contains(Component.DIGEST)); + if (cfg.writeDigest) + { + assertEquals(cctx + " -- Digest.crc32 does not match the child Data.db", + crc32(child.descriptor.fileFor(Component.DATA)), + Long.parseLong(readAll(child.descriptor.fileFor(Component.DIGEST)).trim())); + } + else + { + assertFalse(cctx + " -- Digest.crc32 exists but was not requested", + child.descriptor.fileFor(Component.DIGEST).exists()); + } + + // ---- every index position was rebased by exactly `shift` ---- + for (int r = from; r < to; r++) + { + RowIndexEntry entry = child.reader.getPosition(index.keys[r], SSTableReader.Operator.EQ, false); + assertNotNull(cctx + " -- child cannot find key " + index.keys[r], entry); + assertEquals(cctx + " -- rebased position for record " + r, + index.positions[r] - child.shift, entry.position); + } + assertEquals(cctx + " -- first partition must land at the dead prefix", + child.deadPrefixBytes, + child.reader.getPosition(child.first, SSTableReader.Operator.EQ, false).position); + assertTrue(cctx + " -- the dead prefix must be smaller than one chunk", + child.deadPrefixBytes < chunkLength); + + // ---- children must be disjoint and in token order ---- + if (b > 0) + assertTrue(cctx + " -- children overlap or are out of order", + children.get(b - 1).last.compareTo(child.first) < 0); + assertTrue(cctx + " -- first > last", child.first.compareTo(child.last) <= 0); + + physicalSum += child.physicalBytes; + deadSum += child.deadPrefixBytes; + partitionSum += child.partitionCount; + } + + assertEquals(ctx + " -- children do not cover every parent partition", index.size(), partitionSum); + assertEquals(ctx + " -- totalPhysicalBytesCopied", physicalSum, result.totalPhysicalBytesCopied); + assertEquals(ctx + " -- totalDeadPrefixBytes", deadSum, result.totalDeadPrefixBytes); + + // ---- THE ORACLE ---- + assertConcatenatedChildrenEqualParent(parent, children, ctx); + } + + /** concatenated children == parent, exactly. */ + private static void assertConcatenatedChildrenEqualParent(SSTableReader parent, + List children, + String ctx) + { + try (ISSTableScanner parentScanner = parent.getScanner()) + { + long seen = 0; + for (int b = 0; b < children.size(); b++) + { + ZeroCopySSTableSplitter.Child child = children.get(b); + long inChild = 0; + try (ISSTableScanner childScanner = child.reader.getScanner()) + { + while (childScanner.hasNext()) + { + assertTrue(ctx + " -- child " + b + " has partitions the parent does not, after " + seen, + parentScanner.hasNext()); + try (UnfilteredRowIterator expected = parentScanner.next(); + UnfilteredRowIterator actual = childScanner.next()) + { + assertPartitionEquals(expected, actual, ctx + " -- child " + b + " partition " + seen); + } + inChild++; + seen++; + } + } + assertEquals(ctx + " -- child " + b + " scanned a different number of partitions than it reported", + child.partitionCount, inChild); + } + assertFalse(ctx + " -- the children are missing trailing parent partitions (saw " + seen + ')', + parentScanner.hasNext()); + } + } + + private static void assertPartitionEquals(UnfilteredRowIterator expected, UnfilteredRowIterator actual, String ctx) + { + assertEquals(ctx + " -- partition key", expected.partitionKey(), actual.partitionKey()); + String key = " (" + expected.partitionKey() + ')'; + assertEquals(ctx + " -- partition level deletion" + key, + expected.partitionLevelDeletion(), actual.partitionLevelDeletion()); + assertEquals(ctx + " -- static row" + key, expected.staticRow(), actual.staticRow()); + assertEquals(ctx + " -- columns" + key, expected.columns(), actual.columns()); + assertEquals(ctx + " -- reverse order" + key, expected.isReverseOrder(), actual.isReverseOrder()); + + int u = 0; + while (expected.hasNext()) + { + assertTrue(ctx + " -- child partition truncated at unfiltered " + u + key, actual.hasNext()); + assertEquals(ctx + " -- unfiltered " + u + key, expected.next(), actual.next()); + u++; + } + assertFalse(ctx + " -- child partition has extra unfiltereds past " + u + key, actual.hasNext()); + } + + // ------------------------------------------------------------------------------------------------ + // Adversarial generator: partitions calibrated to straddle chunk boundaries as tightly as possible + // ------------------------------------------------------------------------------------------------ + + /** @return the constant per-partition serialized overhead of the straddle schema, i.e. size - blobLength. */ + private int calibrateOverhead(Random rnd) throws Throwable + { + int probeBlob = 4096; + int size = straddleTableAndMeasure(probeBlob, 3, "{'class': 'LZ4Compressor', 'chunk_length_in_kb': 16}", rnd); + return size - probeBlob; + } + + /** + * @return true if the generator converged on partitions of exactly {@code L + delta} bytes + */ + private boolean runStraddleScenario(int chunkKb, int delta, int overhead, long seed) throws Throwable + { + Random rnd = new Random(seed); + int chunkLength = chunkKb * 1024; + long target = chunkLength + delta; + String compressor = COMPRESSORS[rnd.nextInt(COMPRESSORS.length - 1)]; // never the uncompressed slot + String compression = String.format("{'class': '%s', 'chunk_length_in_kb': %d}", compressor, chunkKb); + + int partitions = 12; + int blobLength = (int) target - overhead; + if (blobLength <= 0) + { + logger.warn("straddle target {} is smaller than the per-partition overhead {}", target, overhead); + return false; + } + + boolean converged = false; + for (int attempt = 0; attempt < 3 && blobLength > 0; attempt++) + { + int measured = straddleTableAndMeasure(blobLength, partitions, compression, rnd); + if (measured == target) + { + converged = true; + break; + } + blobLength += (int) target - measured; + } + + if (!converged) + { + logger.warn("straddle generator did not converge for chunkKb={} delta={} (overhead={}); the split " + + "oracle still runs, but partitions are not exactly chunk-aligned", chunkKb, delta, overhead); + if (blobLength <= 0) + return false; + } + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + String ctx = String.format("straddle[chunkKb=%d delta=%d target=%d blob=%d compressor=%s converged=%s seed=%d]", + chunkKb, delta, target, blobLength, compressor, converged, seed); + SSTableReader parent = onlySSTable(cfs, ctx); + ParentIndex index = readIndex(parent); + assertEquals(ctx + " -- wrong partition count", partitions, index.size()); + + if (converged) + { + for (int i = 0; i < index.size(); i++) + assertEquals(ctx + " -- partition " + i + " is not at an exact multiple of the target size", + i * target, index.positions[i]); + assertEquals(ctx + " -- parent uncompressedLength", partitions * target, parent.uncompressedLength()); + } + + Config cfg = new Config(seed); + cfg.compressor = compressor; + cfg.chunkKb = chunkKb; + cfg.adversarialNote = ctx; + cfg.parentPartitions = index.size(); + cfg.parentUncompressedLength = parent.uncompressedLength(); + cfg.parentChunkLength = chunkLength; + // Alternate the aligned (extent-shareable) layout with the plain one across the six scenarios, so the + // partitions engineered to land exactly on chunk boundaries are exercised against both. + cfg.alignedLayout = delta >= 0; + ZeroCopySSTableSplitter.forceAlignedLayoutForTesting = cfg.alignedLayout; + + try + { + // (a) one child per partition: every interior boundary is a chunk boundary +/- delta + int[] all = new int[partitions - 1]; + for (int i = 0; i < all.length; i++) + all[i] = i + 1; + runSplitByBoundaries(parent, index, cfg, all); + + // (b) a plain byte-share split over the same adversarial layout + cfg.splitMode = "numChildren"; + cfg.numChildren = 3; + cfg.boundaryIndices = null; + ZeroCopySSTableSplitter.Result byCount = null; + try + { + byCount = ZeroCopySSTableSplitter.split(parent, 3, null); + cfg.actualChildren = byCount.children.size(); + verify(parent, index, byCount, cfg, null); + } + finally + { + releaseChildren(byCount); + deleteChildFiles(byCount); + } + + // (c) a random subset of the same boundaries + int[] subset = pickBoundaryIndices(rnd, index, chunkLength); + runSplitByBoundaries(parent, index, cfg, subset); + } + finally + { + ZeroCopySSTableSplitter.forceAlignedLayoutForTesting = false; + } + + return converged; + } + + private void runSplitByBoundaries(SSTableReader parent, ParentIndex index, Config cfg, int[] indices) + throws Exception + { + cfg.splitMode = "boundaries"; + cfg.numChildren = -1; + cfg.boundaryIndices = indices; + List boundaries = new ArrayList<>(indices.length); + for (int idx : indices) + boundaries.add(index.keys[idx]); + + ZeroCopySSTableSplitter.Result result = null; + try + { + result = ZeroCopySSTableSplitter.split(parent, boundaries, null); + cfg.actualChildren = result.children.size(); + verify(parent, index, result, cfg, indices); + } + finally + { + releaseChildren(result); + deleteChildFiles(result); + } + } + + /** + * Creates a fresh single-row-per-partition table, writes {@code partitions} identically sized partitions and + * flushes. + * + * @return the exact uncompressed size of one partition + */ + private int straddleTableAndMeasure(int blobLength, int partitions, String compression, Random rnd) + throws Throwable + { + createTable("CREATE TABLE %s (pk text PRIMARY KEY, v blob) WITH compression = " + compression); + disableCompaction(); + for (int i = 0; i < partitions; i++) + { + byte[] bytes = new byte[blobLength]; + rnd.nextBytes(bytes); // incompressible, so chunks are stored raw when min_compress_ratio bites + execute("INSERT INTO %s (pk, v) VALUES (?, ?) USING TIMESTAMP ?", + String.format("p%05d", i), ByteBuffer.wrap(bytes), PAST_TS + i); + } + flush(); + + // The probe needs one sstable so that consecutive index positions give one partition's exact size. + // A memtable can flush on its own part way through the loop above (heap pressure from earlier test + // methods in this JVM is enough to trigger it), which leaves two sstables and used to make this an + // order-dependent flake. Consolidate instead of asserting and hoping. + ColumnFamilyStore probeCfs = getCurrentColumnFamilyStore(); + if (probeCfs.getLiveSSTables().size() > 1) + { + compact(); + assertEquals("straddle probe could not consolidate to a single sstable", + 1, probeCfs.getLiveSSTables().size()); + } + + SSTableReader sstable = onlySSTable(probeCfs, "straddle probe"); + ParentIndex index = readIndex(sstable); + assertEquals("straddle probe wrote the wrong number of partitions", partitions, index.size()); + + long size = index.size() > 1 ? index.positions[1] - index.positions[0] + : sstable.uncompressedLength() - index.positions[0]; + for (int i = 1; i < index.size(); i++) + { + long end = i + 1 < index.size() ? index.positions[i + 1] : sstable.uncompressedLength(); + assertEquals("straddle probe partitions are not all the same size; the calibration assumption is broken", + size, end - index.positions[i]); + } + return Math.toIntExact(size); + } + + // ------------------------------------------------------------------------------------------------ + // Schema and data generation + // ------------------------------------------------------------------------------------------------ + + private static String ddl(Config cfg) + { + StringBuilder sb = new StringBuilder("CREATE TABLE %s (pk text"); + if (cfg.clusterings >= 1) + sb.append(", ck0 int"); + if (cfg.clusterings >= 2) + sb.append(", ck1 text"); + sb.append(", v blob, t text, n int"); + if (cfg.hasStatic) + sb.append(", s text static"); + if (cfg.hasMap) + sb.append(", m map"); + if (cfg.hasSet) + sb.append(", st set"); + sb.append(", PRIMARY KEY (pk"); + if (cfg.clusterings >= 1) + sb.append(", ck0"); + if (cfg.clusterings >= 2) + sb.append(", ck1"); + sb.append("))"); + + sb.append(" WITH compression = "); + if (cfg.compressor == null) + { + sb.append("{'enabled': 'false'}"); + } + else + { + sb.append("{'class': '").append(cfg.compressor) + .append("', 'chunk_length_in_kb': ").append(cfg.chunkKb); + if (cfg.minCompressRatio > 0) + sb.append(", 'min_compress_ratio': ").append(cfg.minCompressRatio); + sb.append('}'); + } + + if (cfg.clusterings >= 1) + { + sb.append(" AND CLUSTERING ORDER BY (ck0 ").append(cfg.reverse0 ? "DESC" : "ASC"); + if (cfg.clusterings >= 2) + sb.append(", ck1 ").append(cfg.reverse1 ? "DESC" : "ASC"); + sb.append(')'); + } + return sb.toString(); + } + + private void writeRandomData(Config cfg, Random rnd, int chunkLength) throws Throwable + { + long budget = MAX_TABLE_BYTES; + long pastTs = PAST_TS; // monotonic, and always older than the wall-clock timestamp a DELETE gets + cfg.rowsPerPartition = new int[cfg.partitions]; + cfg.valueBytes = new int[cfg.partitions]; + + for (int p = 0; p < cfg.partitions; p++) + { + String pk = String.format("p%05d", p); + + // A partition-level tombstone that data written afterwards (at FUTURE_TS) survives. + boolean partitionTombstone = rnd.nextInt(10) == 0; + if (partitionTombstone) + execute("DELETE FROM %s WHERE pk = ?", pk); + long rowTs = partitionTombstone ? FUTURE_TS + p * 1000L : pastTs; + + // value size class: tiny / small / much bigger than one chunk + int roll = rnd.nextInt(100); + int valueSize; + int rows; + if (roll < 45) + { + valueSize = rnd.nextInt(200); + rows = cfg.clusterings == 0 ? 1 : 1 + rnd.nextInt(6); + } + else if (roll < 80) + { + valueSize = 200 + rnd.nextInt(2000); + rows = cfg.clusterings == 0 ? 1 : 1 + rnd.nextInt(4); + } + else + { + // deliberately much larger than chunkLength so a single partition spans many chunks + valueSize = chunkLength + rnd.nextInt(2 * chunkLength); + rows = cfg.clusterings == 0 ? 1 : 1 + rnd.nextInt(2); + } + if ((long) valueSize * rows > budget) + { + valueSize = Math.min(valueSize, 256); + rows = Math.min(rows, 2); + } + budget -= (long) valueSize * rows; + cfg.rowsPerPartition[p] = rows; + cfg.valueBytes[p] = valueSize; + + boolean compressible = rnd.nextBoolean(); + for (int r = 0; r < rows; r++) + insertRow(cfg, rnd, pk, r, valueSize, compressible, rowTs + r); + + if (cfg.hasStatic && rnd.nextBoolean()) + execute("INSERT INTO %s (pk, s) VALUES (?, ?) USING TIMESTAMP ?", + pk, text(rnd, 1 + rnd.nextInt(40)), rowTs + rows); + + // only the PAST counter advances; the FUTURE timestamps of a resurrected partition must not leak + // into the next partition or the deletions below would stop biting. + pastTs += rows + 2; + + // deletions run at wall-clock timestamps: they always shadow PAST_TS data and never FUTURE_TS data, + // but either way the tombstones themselves land in the copied blobs. + if (cfg.clusterings >= 1 && rows > 1) + { + if (rnd.nextInt(4) == 0) // row tombstone + { + if (cfg.clusterings == 1) + execute("DELETE FROM %s WHERE pk = ? AND ck0 = ?", pk, rnd.nextInt(rows)); + else + execute("DELETE FROM %s WHERE pk = ? AND ck0 = ? AND ck1 = ?", + pk, rnd.nextInt(rows), "c" + rnd.nextInt(rows)); + } + if (rnd.nextInt(4) == 0) // range tombstone + { + int a = rnd.nextInt(rows); + int b = a + 1 + rnd.nextInt(Math.max(1, rows - a)); + execute("DELETE FROM %s WHERE pk = ? AND ck0 >= ? AND ck0 < ?", pk, a, b); + } + if (rnd.nextInt(4) == 0) // single cell tombstone + { + if (cfg.clusterings == 1) + execute("DELETE t FROM %s WHERE pk = ? AND ck0 = ?", pk, rnd.nextInt(rows)); + else + execute("DELETE t FROM %s WHERE pk = ? AND ck0 = ? AND ck1 = ?", + pk, rnd.nextInt(rows), "c" + rnd.nextInt(rows)); + } + } + else if (cfg.clusterings == 0 && rnd.nextInt(6) == 0) + { + execute("DELETE t FROM %s WHERE pk = ?", pk); + } + } + } + + private void insertRow(Config cfg, Random rnd, String pk, int row, int valueSize, boolean compressible, long ts) + throws Throwable + { + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + + columns.add("pk"); + values.add(pk); + if (cfg.clusterings >= 1) + { + columns.add("ck0"); + values.add(row); + } + if (cfg.clusterings >= 2) + { + columns.add("ck1"); + values.add("c" + row); + } + + columns.add("v"); + values.add(blob(rnd, valueSize, compressible)); + + int textRoll = rnd.nextInt(6); + if (textRoll != 0) + { + columns.add("t"); + // empty and null values both appear + values.add(textRoll == 1 ? "" : textRoll == 2 ? null : text(rnd, 1 + rnd.nextInt(64))); + } + if (rnd.nextInt(3) != 0) + { + columns.add("n"); + values.add(rnd.nextInt(4) == 0 ? null : rnd.nextInt()); + } + if (cfg.hasMap && rnd.nextInt(3) == 0) + { + columns.add("m"); + if (rnd.nextInt(5) == 0) + { + values.add(null); // collection tombstone + } + else + { + // sorted, so the serialized collection matches the element type's comparator + Map map = new TreeMap<>(); + for (int i = 0, n = rnd.nextInt(4); i < n; i++) + map.put(rnd.nextInt(100), text(rnd, 1 + rnd.nextInt(16))); + values.add(map); + } + } + if (cfg.hasSet && rnd.nextInt(3) == 0) + { + columns.add("st"); + Set set = new TreeSet<>(); + for (int i = 0, n = rnd.nextInt(4); i < n; i++) + set.add(text(rnd, 1 + rnd.nextInt(16))); + values.add(set); + } + if (cfg.hasStatic && rnd.nextInt(5) == 0) + { + columns.add("s"); + values.add(text(rnd, 1 + rnd.nextInt(32))); + } + + StringBuilder query = new StringBuilder("INSERT INTO %s ("); + for (int i = 0; i < columns.size(); i++) + query.append(i == 0 ? "" : ", ").append(columns.get(i)); + query.append(") VALUES ("); + for (int i = 0; i < columns.size(); i++) + query.append(i == 0 ? "?" : ", ?"); + query.append(") USING TIMESTAMP ?"); + values.add(ts); + + // a long TTL: long enough that nothing can expire mid-test, but it still writes real expiry info + boolean ttl = rnd.nextInt(4) == 0; + if (ttl) + { + query.append(" AND TTL ?"); + values.add(100_000 + rnd.nextInt(1_000_000)); + } + + execute(query.toString(), values.toArray()); + } + + private static ByteBuffer blob(Random rnd, int size, boolean compressible) + { + byte[] bytes = new byte[size]; + if (compressible) + { + Arrays.fill(bytes, (byte) ('a' + rnd.nextInt(26))); + for (int i = 0; i < bytes.length; i += 512) + bytes[i] = (byte) rnd.nextInt(); + } + else + { + rnd.nextBytes(bytes); + } + return ByteBuffer.wrap(bytes); + } + + private static String text(Random rnd, int length) + { + char[] chars = new char[length]; + for (int i = 0; i < length; i++) + chars[i] = (char) ('!' + rnd.nextInt(90)); + return new String(chars); + } + + // ------------------------------------------------------------------------------------------------ + // Split-point selection for the test side + // ------------------------------------------------------------------------------------------------ + + /** + * Strictly increasing record indices to use as boundary keys. Biased hard towards partitions that start + * exactly on a chunk boundary, or one byte either side of one. Index 0 is occasionally included on purpose: + * it produces an empty leading run, which must yield no child at all. + */ + private static int[] pickBoundaryIndices(Random rnd, ParentIndex index, int chunkLength) + { + int n = index.size(); + if (n < 2) + return new int[0]; // a single partition cannot be cut; the empty list must give exactly one child + int wanted = 1 + rnd.nextInt(Math.min(5, n - 1)); + + List aligned = new ArrayList<>(); + for (int i = 1; i < n; i++) + { + long mod = index.positions[i] % chunkLength; + if (mod == 0 || mod == 1 || mod == chunkLength - 1) + aligned.add(i); + } + + TreeSet chosen = new TreeSet<>(); + boolean preferAligned = !aligned.isEmpty() && rnd.nextInt(3) != 0; + while (chosen.size() < wanted) + { + if (preferAligned && rnd.nextInt(4) != 0) + chosen.add(aligned.get(rnd.nextInt(aligned.size()))); + else + chosen.add(1 + rnd.nextInt(Math.max(1, n - 1))); + if (chosen.size() >= n) + break; + } + if (rnd.nextInt(6) == 0) + chosen.add(0); // empty leading run + + int[] out = new int[chosen.size()]; + int i = 0; + for (int idx : chosen) + out[i++] = idx; + return out; + } + + // ------------------------------------------------------------------------------------------------ + // Plumbing + // ------------------------------------------------------------------------------------------------ + + private static SSTableReader onlySSTable(ColumnFamilyStore cfs, String ctx) + { + Set live = cfs.getLiveSSTables(); + assertEquals(ctx + " -- expected exactly one sstable after the flush, got " + live, 1, live.size()); + return live.iterator().next(); + } + + /** {@code O(k)}, with {@code O(N)} defined as the physical file length, recomputed from the parent. */ + private static long chunkOffset(CompressionMetadata meta, long k, int chunkCount, int chunkLength) + { + if (k == chunkCount) + return meta.compressedFileLength; + return meta.chunkFor(k * (long) chunkLength).offset; + } + + private static final class ParentIndex + { + final DecoratedKey[] keys; + final long[] positions; + + ParentIndex(DecoratedKey[] keys, long[] positions) + { + this.keys = keys; + this.positions = positions; + } + + int size() + { + return keys.length; + } + } + + /** Independent walk of the parent Index.db; nothing here goes through the splitter. */ + private static ParentIndex readIndex(SSTableReader sstable) throws IOException + { + List keys = new ArrayList<>(); + List positions = new ArrayList<>(); + try (RandomAccessReader in = RandomAccessReader.open(sstable.descriptor.fileFor(Component.PRIMARY_INDEX))) + { + long length = in.length(); + while (in.getFilePointer() != length) + { + ByteBuffer key = ByteBufferUtil.readWithShortLength(in); + long position = RowIndexEntry.Serializer.readPosition(in); + int promotedSize = (int) in.readUnsignedVInt(); + if (promotedSize > 0) + in.skipBytesFully(promotedSize); + keys.add(sstable.getPartitioner().decorateKey(key)); + positions.add(position); + } + } + long[] pos = new long[positions.size()]; + for (int i = 0; i < pos.length; i++) + pos[i] = positions.get(i); + return new ParentIndex(keys.toArray(new DecoratedKey[0]), pos); + } + + private static long crc32(org.apache.cassandra.io.util.File file) throws IOException + { + CRC32 crc = new CRC32(); + byte[] buffer = new byte[64 * 1024]; + try (InputStream in = file.newInputStream()) + { + int n; + while ((n = in.read(buffer)) > 0) + crc.update(buffer, 0, n); + } + return crc.getValue(); + } + + private static String readAll(org.apache.cassandra.io.util.File file) throws IOException + { + return new String(java.nio.file.Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static void releaseChildren(ZeroCopySSTableSplitter.Result result) + { + if (result == null) + return; + for (ZeroCopySSTableSplitter.Child child : result.children) + { + try + { + child.reader.selfRef().release(); + } + catch (Throwable t) + { + logger.warn("failed releasing child {}", child.descriptor, t); + } + } + } + + private static void deleteChildFiles(ZeroCopySSTableSplitter.Result result) + { + if (result == null) + return; + for (ZeroCopySSTableSplitter.Child child : result.children) + { + for (Component component : child.components) + { + try + { + child.descriptor.fileFor(component).deleteIfExists(); + } + catch (Throwable t) + { + logger.warn("failed deleting {} of {}", component, child.descriptor, t); + } + } + } + } + + /** splitmix64, so consecutive base seeds give uncorrelated iterations. */ + private static long scramble(long seed) + { + long z = seed + 0x9E3779B97F4A7C15L; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + + /** Everything needed to understand -- and replay -- one iteration. Mutated as the iteration progresses. */ + private static final class Config + { + final long seed; + + String compressor = "?"; + int chunkKb = -1; + double minCompressRatio = -1; + int columnIndexKb = -1; + int columnIndexCacheKb = -1; + int clusterings = -1; + boolean reverse0; + boolean reverse1; + boolean hasStatic; + boolean hasMap; + boolean hasSet; + int partitions = -1; + int[] rowsPerPartition; + int[] valueBytes; + + int parentPartitions = -1; + long parentUncompressedLength = -1; + int parentChunkLength = -1; + + String splitMode = "?"; + int numChildren = -1; + int[] boundaryIndices; + int actualChildren = -1; + boolean useTxn; + boolean alignedLayout; + boolean writeDigest = true; + String adversarialNote; + + Config(long seed) + { + this.seed = seed; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("seed=").append(seed) + .append(" compressor=").append(compressor) + .append(" chunkKb=").append(chunkKb) + .append(" minCompressRatio=").append(minCompressRatio) + .append(" columnIndexKb=").append(columnIndexKb) + .append(" columnIndexCacheKb=").append(columnIndexCacheKb) + .append(" clusterings=").append(clusterings) + .append(" reverse=[").append(reverse0).append(',').append(reverse1).append(']') + .append(" static=").append(hasStatic) + .append(" map=").append(hasMap) + .append(" set=").append(hasSet) + .append(" partitions=").append(partitions) + .append(" parentPartitions=").append(parentPartitions) + .append(" parentUncompressedLength=").append(parentUncompressedLength) + .append(" parentChunkLength=").append(parentChunkLength) + .append(" splitMode=").append(splitMode) + .append(" numChildren=").append(numChildren) + .append(" boundaryIndices=").append(Arrays.toString(boundaryIndices)) + .append(" actualChildren=").append(actualChildren) + .append(" useTxn=").append(useTxn) + .append(" alignedLayout=").append(alignedLayout) + .append(" writeDigest=").append(writeDigest); + if (adversarialNote != null) + sb.append(" adversarial=").append(adversarialNote); + sb.append("\n rowsPerPartition=").append(Arrays.toString(rowsPerPartition)); + sb.append("\n valueBytes=").append(Arrays.toString(valueBytes)); + return sb.toString(); + } + } +} diff --git a/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterTest.java b/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterTest.java new file mode 100644 index 000000000000..070947e682e3 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/sstable/ZeroCopySSTableSplitterTest.java @@ -0,0 +1,1677 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.io.sstable; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import java.util.zip.CRC32; + +import com.google.common.util.concurrent.RateLimiter; + +import org.junit.Test; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.compaction.CompactionInfo; +import org.apache.cassandra.db.compaction.CompactionInterruptedException; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.compaction.Scrubber; +import org.apache.cassandra.db.compaction.Verifier; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.lifecycle.LifecycleTransaction; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.streaming.CassandraOutgoingFile; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.compress.CompressionMetadata; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter.Child; +import org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter.Result; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.io.util.RandomAccessReader; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.utils.BloomFilterSerializer; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.IFilter; +import org.apache.cassandra.utils.OutputHandler; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * End-to-end correctness of {@link ZeroCopySSTableSplitter}: the children must be readable, and their + * concatenation must be indistinguishable from the parent. + * + *

The load bearing assertions are: + *

    + *
  • {@link #assertConcatenatedContentEquals} -- every partition, row, cell, timestamp and deletion of the + * children concatenated in token order equals the parent, exactly;
    • + *
  • {@link #assertPointReads} -- every parent key is found in exactly one child and reads back identically;
    • + *
  • {@link #assertStructure} -- the chunk arithmetic recomputed independently from the parent's Index.db and + * CompressionInfo.db, including "no trailing slack" and "offsets[0] == 0";
    • + *
  • {@link #assertComponents} -- Filter/Summary/Digest/TOC are the ones on disk and are self-consistent.
    • + *
+ */ +public class ZeroCopySSTableSplitterTest extends CQLTester +{ + private static final SSTableReadsListener NOOP = SSTableReadsListener.NOOP_LISTENER; + + // ---------------------------------------------------------------------------------------------------- + // Tests + // ---------------------------------------------------------------------------------------------------- + + /** + * The core test. 80 narrow partitions (no promoted index), 4 children, then everything: content + * equivalence, point reads, structure, components, dead prefixes, and a reopen purely from disk. + */ + @Test + public void splitFourWaysIsEquivalentToTheParent() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(80, 5, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertTrue(parent.compression); + assertTrue(ZeroCopySSTableSplitter.isSupported(parent)); + assertEquals(4096, parent.getCompressionMetadata().chunkLength()); + // more than one chunk, otherwise the whole exercise is trivial + assertTrue(parent.uncompressedLength() > 20L * 4096); + + Result result = ZeroCopySSTableSplitter.split(parent, 4, null); + try + { + assertEquals(4, result.children.size()); + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + + // The dead prefix must genuinely exist for at least one child, and that child must still read. + Child dead = firstChildWithDeadPrefix(result); + assertNotNull("no child started off a chunk boundary; the dead-prefix path was not exercised", dead); + assertTrue(dead.deadPrefixBytes > 0); + RowIndexEntry firstEntry = dead.reader.getPosition(dead.first, SSTableReader.Operator.EQ, false); + assertNotNull(firstEntry); + assertEquals(dead.deadPrefixBytes, firstEntry.position); + assertTrue(firstEntry.position < dead.reader.getCompressionMetadata().chunkLength()); + try (UnfilteredRowIterator expected = parent.rowIterator(dead.first, Slices.ALL, allColumns(cfs), false, NOOP); + UnfilteredRowIterator actual = dead.reader.rowIterator(dead.first, Slices.ALL, allColumns(cfs), false, NOOP)) + { + assertSamePartition(expected, actual); + } + } + finally + { + release(result); + } + + // Reopen purely from the on-disk files: nothing may depend on in-memory state. + List reopened = new ArrayList<>(); + try + { + for (Child child : result.children) + reopened.add(SSTableReader.open(child.descriptor, child.components, cfs.metadata)); + + for (int i = 0; i < reopened.size(); i++) + { + assertEquals(result.children.get(i).first, reopened.get(i).first); + assertEquals(result.children.get(i).last, reopened.get(i).last); + assertEquals(result.children.get(i).dataLength, reopened.get(i).uncompressedLength()); + } + assertConcatenatedContentEquals(parent, reopened); + } + finally + { + for (SSTableReader reader : reopened) + reader.selfRef().release(); + } + } + + /** + * REGRESSION: the parent here is built by a COMPACTION, not by a flush, and is reopened from disk. + * + *

Those two properties together are what every other test here lacks and what every anticompaction target + * has. A compaction-produced sstable carries one more chunk offset than its {@code dataLength} needs: + * {@code SSTableRewriter.doPrepare} syncs the data file twice and {@code CompressedSequentialWriter.flushData} + * appends a chunk unconditionally, even on an empty buffer, so the physical file ends a few bytes past the last + * chunk holding data and {@code chunkCount == ceil(dataLength / chunkLength) + 1}. A flush calls + * {@code flushData} once and has neither property. + * + *

Preemptive open has to be switched on by hand here, which is the deeper reason nothing already present + * caught the regression: {@code sstable_preemptive_open_interval} defaults to disabled, so + * {@code switchWriter(null)} never calls {@code openFinalEarly()} and the second sync never happens. + * {@code test/conf/cassandra.yaml} leaves it unset where the shipped {@code conf/cassandra.yaml} sets 50MiB -- + * so the trailing chunk happens on every real node and on no test. + * + *

The reopen matters as much: {@code CompressionMetadata.Writer.open} trims the offsets table to + * {@code ceil(dataLength / chunkLength)} and resets {@code compressedLength}, so the reader a compaction hands + * back hides the trailing chunk entirely. Only a reader built by {@code CompressionMetadata.create} -- startup, + * {@code nodetool refresh}, streaming receive -- sees the physical file length. + * + *

The bug: the splitter took the end of a child's last chunk to be {@code compressedFileLength} whenever + * {@code lastChunk + 1} reached {@code ceil(dataLength / chunkLength)}, so the LAST child copied the trailing + * chunk as slack. A reader derives a chunk's length from the following offset, so the child's final chunk then + * claimed to be longer than it was and every read of it failed its inline CRC32 -- or, once the inflated length + * crossed {@code maxCompressedLength}, took the raw-chunk branch and returned compressed bytes as row data. + * Digest.crc32 could not catch it, being computed over whatever was written, and the parent was already + * obsoleted by then. + */ + @Test + public void splitOfCompactionProducedParentDoesNotAbsorbTheTrailingChunk() throws Throwable + { + int previousInterval = DatabaseDescriptor.getSSTablePreemptiveOpenIntervalInMiB(); + SSTableReader parent = null; + try + { + // What conf/cassandra.yaml ships, and what test/conf/cassandra.yaml leaves unset. + DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMiB(50); + + createCompressedTable(4); + disableCompaction(); + insertPartitions(60, 5, 480); + flush(); + insertPartitions(60, 5, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + assertEquals("need two sstables to have something to compact", 2, cfs.getLiveSSTables().size()); + cfs.forceMajorCompaction(); + SSTableReader compacted = onlySSTable(cfs); + + parent = SSTableReader.open(compacted.descriptor, compacted.components, cfs.metadata); + + long[] offsets = readChunkOffsets(parent.descriptor); + CompressionMetadata meta = parent.getCompressionMetadata(); + int chunkLength = meta.chunkLength(); + int dataChunks = (int) ((meta.dataLength + chunkLength - 1) / chunkLength); + long physical = parent.descriptor.fileFor(Component.DATA).length(); + + // Guard the guard. If compaction ever stops emitting the trailing chunk, or the reopen stops + // exposing it, this test silently stops testing anything -- so fail loudly instead. + assertEquals("a compaction-produced sstable is expected to carry exactly one trailing " + + "zero-uncompressed-length chunk; without it this test cannot exercise the regression", + dataChunks + 1, offsets.length); + assertEquals("the parent must be the on-disk view, whose length includes the trailing chunk", + physical, meta.compressedFileLength); + assertTrue("the trailing chunk must put the physical end past the last data chunk", + physical > offsets[dataChunks]); + assertTrue("more than one chunk, otherwise the whole exercise is trivial", dataChunks > 20); + + Result result = ZeroCopySSTableSplitter.split(parent, 3, null); + try + { + assertEquals(3, result.children.size()); + + // The last child is the only one that could have swallowed the trailing chunk. + Child last = result.children.get(result.children.size() - 1); + assertEquals("the last child must end at the last DATA chunk", dataChunks - 1, last.lastChunk); + assertEquals("the last child must stop at the end of the last data chunk", + offsets[dataChunks] - offsets[(int) last.firstChunk], last.physicalBytes); + assertEquals("and that must be its exact on-disk length, head pad aside", + last.onDiskLength(), last.descriptor.fileFor(Component.DATA).length()); + assertTrue("the trailing slack must not have been copied", + last.physicalBytes < physical - offsets[(int) last.firstChunk]); + + // The failure mode was confined to the final chunk, so read it: a wrong derived length shows up + // as a CorruptSSTableException here and nowhere else. + try (RandomAccessReader in = last.reader.openDataReader()) + { + in.seek(last.reader.uncompressedLength() - 1); + in.readByte(); + } + + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + } + finally + { + release(result); + } + } + finally + { + if (parent != null) + parent.selfRef().release(); + DatabaseDescriptor.setSSTablePreemptiveOpenIntervalInMiB(previousInterval); + } + } + + /** + * A stop request aborts the copy and leaves nothing behind, and the {@link ZeroCopySSTableSplitter.Progress} + * holder carries what the callers of {@link CompactionInfo.Holder#stop()} need in order to find it. + * + *

This is the wiring behind {@code nodetool stop ANTICOMPACTION}, {@code nodetool stop --id}, TRUNCATE, DROP + * and {@code runWithCompactionsDisabled}. All of them walk {@code CompactionManager.active.getCompactions()} and + * decide from the {@link CompactionInfo}: {@code stopCompaction} matches on {@code getTaskType()}, + * {@code stopCompactionById} on {@code getTaskId()}, {@code interruptCompactionFor} on + * {@code getTableMetadata()} plus the sstables in {@code shouldStop}. Before this existed the split registered + * nothing, so all of them silently found no work to stop -- and truncate reported success while the copy + * carried on. + */ + /** + * The crash-recovery contract: every child is covered by an ADD record in the transaction log, so a start that + * finds the log uncommitted deletes them all and leaves the parent alone. + *

+ * This abandons the transaction without committing or aborting it -- the closest an in-process test can get to + * a {@code kill -9} -- and then runs the boot path, {@code removeUnfinishedLeftovers}. What it does NOT cover + * is the window the tracking has to be early for: a crash BETWEEN two of a child's components. In-process that + * window is unreachable, because {@code build}'s {@code finally} cleans partial children up itself, so no test + * here can distinguish tracking before the copy from tracking after it. The reason the splitter now registers + * before the first byte is the invariant {@code BigTableWriter} states outright -- "must track before any files + * are created" -- and this test is what guards the registration from being dropped altogether. + */ + @Test + public void abandonedSplitIsCleanedUpByTheBootPath() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(80, 5, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + Descriptor parentDescriptor = parent.descriptor; + int sstablesBefore = countDataFiles(parentDescriptor); + + LifecycleTransaction txn = cfs.getTracker().tryModify(parent, OperationType.ANTICOMPACTION); + ZeroCopySSTableSplitter.Result result = ZeroCopySSTableSplitter.split(parent, 4, txn); + assertEquals(4, result.children.size()); + assertEquals("the children are on disk now", sstablesBefore + 4, countDataFiles(parentDescriptor)); + + // Drop every reader the split opened and walk away from the transaction, leaving its log uncommitted -- + // what a power loss between the last child and the commit record leaves behind. + for (ZeroCopySSTableSplitter.Child child : result.children) + child.reader.selfRef().release(); + + assertTrue("the boot path must find work to do", LifecycleTransaction.removeUnfinishedLeftovers(cfs)); + + assertEquals("an uncommitted split's children must not survive a restart", + sstablesBefore, countDataFiles(parentDescriptor)); + for (ZeroCopySSTableSplitter.Child child : result.children) + assertFalse("orphaned child " + child.descriptor, + child.descriptor.fileFor(Component.DATA).exists()); + assertTrue("the parent must still be there, or the range is gone from this replica", + parentDescriptor.fileFor(Component.DATA).exists()); + } + + @Test + public void stopRequestAbortsTheSplitAndLeavesNoFilesBehind() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(80, 5, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + int sstablesBefore = countDataFiles(parent.descriptor); + + ZeroCopySSTableSplitter.Progress progress = + ZeroCopySSTableSplitter.progressFor(parent, RateLimiter.create(Double.MAX_VALUE)); + + // What nodetool stop / truncate / drop look at to decide this operation is theirs to cancel. + CompactionInfo info = progress.getCompactionInfo(); + assertEquals(OperationType.ANTICOMPACTION, info.getTaskType()); + assertEquals(cfs.metadata(), info.getTableMetadata()); + assertNotNull("a null task id would make nodetool stop --id unable to address this", info.getTaskId()); + assertEquals("the parent must be in the info, or interruptCompactionFor cannot match it", + Collections.singleton(parent), info.getSSTables()); + assertEquals(CompactionInfo.Unit.BYTES, info.getUnit()); + assertTrue("total must be positive or compactionstats shows no progress", info.getTotal() > 0); + assertFalse(progress.isStopRequested()); + + progress.stop(); + assertTrue(progress.isStopRequested()); + + try + { + ZeroCopySSTableSplitter.split(parent, 4, null, progress); + fail("a stopped split must raise CompactionInterruptedException rather than finish"); + } + catch (CompactionInterruptedException expected) + { + // exactly what the rewrite path raises when its CompactionIterator is interrupted + } + + assertEquals("an aborted split must not leave child sstables on disk", + sstablesBefore, countDataFiles(parent.descriptor)); + assertEquals("the parent must be untouched", parent, onlySSTable(cfs)); + } + + private static int countDataFiles(Descriptor descriptor) + { + java.io.File[] files = new java.io.File(descriptor.directory.toString()) + .listFiles((dir, name) -> name.endsWith("-Data.db")); + return files == null ? 0 : files.length; + } + + /** + * A split child with a dead prefix is still eligible for entire-SSTable zero-copy streaming. + * + *

Entire-SSTable streaming copies every component file verbatim, so it is legal whenever the + * requested ranges cover all of the child's live data. {@link CassandraOutgoingFile#contained} used to + * compare the requested byte span against the physical data length, which a dead prefix makes + * unreachable ({@code transferLength == uncompressedLength() - deadPrefixBytes}), needlessly refusing + * the fast path until the child was recompacted. The check now measures against the live span, so the + * child is eligible as-is. A genuinely partial range must still fall back to the rewrite path. + */ + @Test + public void childWithDeadPrefixIsEligibleForEntireSSTableStreaming() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(80, 5, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + + Result result = ZeroCopySSTableSplitter.split(parent, 4, null); + try + { + Child dead = firstChildWithDeadPrefix(result); + assertNotNull("no child started off a chunk boundary; the dead-prefix path was not exercised", dead); + assertTrue(dead.deadPrefixBytes > 0); + + SSTableReader child = dead.reader; + + // The first live partition sits at deadPrefixBytes, so getPositionsForRanges() over the whole + // token range yields [deadPrefixBytes, uncompressedLength) -- a span short of the physical length. + long firstPosition = child.getPosition(child.first.getToken().minKeyBound(), SSTableReader.Operator.GT).position; + assertEquals(dead.deadPrefixBytes, firstPosition); + + List> fullRange = Range.normalize(Collections.singletonList( + new Range<>(cfs.getPartitioner().getMinimumToken(), child.last.getToken()))); + List sections = child.getPositionsForRanges(fullRange); + long transferLength = sections.stream().mapToLong(p -> p.upperPosition - p.lowerPosition).sum(); + assertEquals(child.uncompressedLength() - firstPosition, transferLength); + assertTrue("the dead prefix must make the byte span fall short of the physical length", + transferLength < child.uncompressedLength()); + + CassandraOutgoingFile cof = new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, child.ref(), + sections, fullRange, child.estimatedKeys()); + try + { + // The whole live span is requested, so despite the dead prefix the child is eligible. + assertTrue("a dead prefix must not disqualify a fully-covered child", cof.contained(sections, child)); + + // A range covering only part of the child must still fall back to the rewrite path. + List childIndex = readIndex(child.descriptor); + assertTrue("need at least two partitions for a partial range", childIndex.size() >= 2); + Token midToken = child.decorateKey(childIndex.get(childIndex.size() / 2).key).getToken(); + List> partialRange = Range.normalize(Collections.singletonList( + new Range<>(cfs.getPartitioner().getMinimumToken(), midToken))); + List partialSections = child.getPositionsForRanges(partialRange); + assertFalse("a partial range must not be treated as containing the whole sstable", + cof.contained(partialSections, child)); + } + finally + { + cof.finish(); + } + } + finally + { + release(result); + } + } + + /** + * Wide partitions: every partition carries a promoted index blob and spans several compression chunks. + * The blob is copied verbatim, so slice reads (which navigate it) must return identical results, and the + * column-index cache is forced to zero so the blob is re-read from the CHILD's Index.db on every lookup + * (ShallowIndexedEntry) rather than being served from an on-heap copy. + */ + @Test + public void widePartitionsPreserveThePromotedIndex() throws Throwable + { + int previousCacheSize = DatabaseDescriptor.getColumnIndexCacheSizeInKiB(); + DatabaseDescriptor.setColumnIndexCacheSize(0); + try + { + createCompressedTable(4); + disableCompaction(); + int partitions = 12; + int rowsPerPartition = 40; + insertPartitions(partitions, rowsPerPartition, 1000); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + List parentIndex = readIndex(parent.descriptor); + assertEquals(partitions, parentIndex.size()); + + int chunkLength = parent.getCompressionMetadata().chunkLength(); + for (int r = 0; r < parentIndex.size(); r++) + { + assertTrue("partition " + r + " has no promoted index", parentIndex.get(r).promoted != null); + long end = r + 1 < parentIndex.size() ? parentIndex.get(r + 1).position : parent.uncompressedLength(); + assertTrue("partition " + r + " does not span multiple chunks", + end - parentIndex.get(r).position > chunkLength); + } + + Result result = ZeroCopySSTableSplitter.split(parent, 3, null); + try + { + assertEquals(3, result.children.size()); + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + + for (Child child : result.children) + { + RowIndexEntry entry = child.reader.getPosition(child.first, SSTableReader.Operator.EQ, false); + assertNotNull(entry); + assertTrue("child lost the promoted index for " + child.first, entry.isIndexed()); + } + + assertSliceReadsMatch(cfs, parent, result, rowsPerPartition); + } + finally + { + release(result); + } + } + finally + { + DatabaseDescriptor.setColumnIndexCacheSize(previousCacheSize); + } + } + + /** One child: the Data.db copy must be byte identical to the parent's, and there is no dead prefix. */ + @Test + public void singleChildCopiesTheParentByteForByte() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(25, 4, 400); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + + Result result = ZeroCopySSTableSplitter.split(parent, 1, null); + try + { + assertEquals(1, result.children.size()); + Child only = result.children.get(0); + assertEquals(0, only.firstChunk); + assertEquals(0, only.shift); + assertEquals(0, only.deadPrefixBytes); + assertEquals(0, result.totalDeadPrefixBytes); + assertEquals(0, result.duplicatedChunkBytes); + assertEquals(parent.uncompressedLength(), only.dataLength); + assertEquals(parent.descriptor.fileFor(Component.DATA).length(), only.physicalBytes); + + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + + assertArrayEquals("a one-way split must reproduce Data.db exactly", + Files.readAllBytes(parent.descriptor.fileFor(Component.DATA).toPath()), + Files.readAllBytes(only.descriptor.fileFor(Component.DATA).toPath())); + assertEquals(readDigest(parent.descriptor), readDigest(only.descriptor)); + } + finally + { + release(result); + } + } + + /** + * A single-partition sstable is still splittable one way, and cannot be split further. Also covers the + * "very first and very last partition are the same partition" boundary. + */ + @Test + public void singlePartitionSSTable() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(1, 12, 900); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertEquals(1, readIndex(parent.descriptor).size()); + assertTrue(parent.uncompressedLength() > parent.getCompressionMetadata().chunkLength()); + + try + { + ZeroCopySSTableSplitter.split(parent, 2, null); + fail("a single-partition sstable cannot be split two ways"); + } + catch (IllegalArgumentException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("cannot split")); + } + + Result result = ZeroCopySSTableSplitter.split(parent, 1, null); + try + { + assertEquals(1, result.children.size()); + assertEquals(1, result.children.get(0).partitionCount); + assertEquals(parent.first, result.children.get(0).first); + assertEquals(parent.last, result.children.get(0).last); + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + } + finally + { + release(result); + } + } + + /** + * One child per partition, all of them inside a single compression chunk. Every child then copies the + * same physical chunk and only its own Index.db entry keeps it apart; the concatenation must still be + * exactly the parent. + */ + @Test + public void oneChildPerPartitionInsideASingleChunk() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(3, 1, 100); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertTrue("expected the whole sstable to fit in one chunk", + parent.uncompressedLength() <= parent.getCompressionMetadata().chunkLength()); + + try + { + ZeroCopySSTableSplitter.split(parent, 4, null); + fail("expected a refusal for more children than partitions"); + } + catch (IllegalArgumentException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("cannot split")); + } + + try + { + ZeroCopySSTableSplitter.split(parent, 0, null); + fail("expected a refusal for numChildren < 1"); + } + catch (IllegalArgumentException e) + { + // expected + } + + Result result = ZeroCopySSTableSplitter.split(parent, 3, null); + try + { + assertEquals(3, result.children.size()); + for (Child child : result.children) + { + assertEquals(1, child.partitionCount); + assertEquals(0, child.firstChunk); + assertEquals(0, child.lastChunk); + assertEquals(0, child.shift); + } + // children 1 and 2 start inside chunk 0, so they must carry a dead prefix + assertEquals(0, result.children.get(0).deadPrefixBytes); + assertTrue(result.children.get(1).deadPrefixBytes > 0); + assertTrue(result.children.get(2).deadPrefixBytes > 0); + + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + } + finally + { + release(result); + } + } + + /** The explicit-boundary form, including the "boundary range contains no partition" case. */ + @Test + public void explicitBoundaries() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(60, 4, 400); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + List parentIndex = readIndex(parent.descriptor); + assertEquals(60, parentIndex.size()); + + DecoratedKey first = parent.decorateKey(parentIndex.get(17).key); + DecoratedKey second = parent.decorateKey(parentIndex.get(41).key); + + Result result = ZeroCopySSTableSplitter.split(parent, Arrays.asList(first, second), null); + try + { + assertEquals(3, result.children.size()); + assertEquals(17, result.children.get(0).partitionCount); + assertEquals(24, result.children.get(1).partitionCount); + assertEquals(19, result.children.get(2).partitionCount); + assertEquals(first, result.children.get(1).first); + assertEquals(second, result.children.get(2).first); + + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + } + finally + { + release(result); + } + + // A boundary equal to the very first key leaves an empty leading run: no child is emitted for it. + Result degenerate = ZeroCopySSTableSplitter.split(parent, Collections.singletonList(parent.first), null); + try + { + assertEquals(1, degenerate.children.size()); + assertEquals(60, degenerate.children.get(0).partitionCount); + assertConcatenatedContentEquals(parent, readers(degenerate)); + } + finally + { + release(degenerate); + } + + try + { + ZeroCopySSTableSplitter.split(parent, Arrays.asList(second, first), null); + fail("expected non-increasing boundaries to be rejected"); + } + catch (IllegalArgumentException e) + { + assertTrue(e.getMessage(), e.getMessage().contains("strictly increasing")); + } + } + + /** + * A split boundary that lands exactly on a compression chunk boundary: the second child then has no dead + * prefix and the two children share no chunk at all. + *

+ * Every partition here has an identical serialised size S (fixed width key, fixed width value, fixed + * timestamp), so partition r starts at r*S and some r = L / gcd(S, L) <= L is necessarily a multiple + * of the 1 KiB chunk length -- which is why 1200 partitions are written. + */ + @Test + public void splitBoundaryOnAChunkBoundary() throws Throwable + { + createCompressedTable(1); + disableCompaction(); + int partitions = 1200; + String value = fixedText(24); + for (int p = 0; p < partitions; p++) + execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, ?) USING TIMESTAMP 1000", key(p), 0, value); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + int chunkLength = parent.getCompressionMetadata().chunkLength(); + assertEquals(1024, chunkLength); + + List parentIndex = readIndex(parent.descriptor); + assertEquals(partitions, parentIndex.size()); + long size = parentIndex.get(1).position - parentIndex.get(0).position; + for (int r = 1; r < parentIndex.size(); r++) + assertEquals("partitions were expected to be identically sized", + size, parentIndex.get(r).position - parentIndex.get(r - 1).position); + + int aligned = -1; + for (int r = 1; r < parentIndex.size(); r++) + { + if (parentIndex.get(r).position % chunkLength == 0) + { + aligned = r; + break; + } + } + assertTrue("no partition start landed on a chunk boundary (partition size " + size + ')', aligned > 0); + + DecoratedKey boundary = parent.decorateKey(parentIndex.get(aligned).key); + Result result = ZeroCopySSTableSplitter.split(parent, Collections.singletonList(boundary), null); + try + { + assertEquals(2, result.children.size()); + Child head = result.children.get(0); + Child tail = result.children.get(1); + + assertEquals(aligned, head.partitionCount); + assertEquals(partitions - aligned, tail.partitionCount); + assertEquals(0, tail.deadPrefixBytes); + assertEquals(parentIndex.get(aligned).position, tail.shift); + assertEquals("an aligned boundary must not duplicate a chunk", head.lastChunk + 1, tail.firstChunk); + assertEquals(0, result.duplicatedChunkBytes); + // the head's Data.db ends exactly where the tail's begins: no byte is in both children + assertEquals(parent.descriptor.fileFor(Component.DATA).length(), + head.physicalBytes + tail.physicalBytes); + + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + } + finally + { + release(result); + } + } + + /** + * FACT 7: Scrubber and Verifier both walk Data.db linearly from position 0 and were patched to start at + * the first index position instead. So both must now ACCEPT a child that carries a dead prefix. + */ + @Test + public void verifierAndScrubberAcceptAChildWithADeadPrefix() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(40, 4, 500); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + + Result result = ZeroCopySSTableSplitter.split(parent, 3, null); + SSTableReader consumedByTxn = null; + try + { + Child dead = firstChildWithDeadPrefix(result); + assertNotNull("no child started off a chunk boundary", dead); + + // Extended verification reads every partition off Data.db linearly and validates Digest.crc32, + // the index, the summary and the bloom filter. It throws CorruptSSTableException on failure. + for (Child child : result.children) + { + try (Verifier verifier = new Verifier(cfs, child.reader, true, + Verifier.options().extendedVerification(true).build())) + { + verifier.verify(); + } + } + + // Scrubber rewrites the child from a linear Data.db walk; every partition must come back good. + // LifecycleTransaction.offline() hands the reader to a dummy Tracker that owns and releases it + // (LifecycleTransaction.java:143-149), so this child must not be released again below. + consumedByTxn = dead.reader; + Scrubber.ScrubResult scrubResult; + try (LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.SCRUB, dead.reader); + Scrubber scrubber = new Scrubber(cfs, txn, false, true)) + { + scrubResult = scrubber.scrubWithResult(); + } + assertEquals(dead.partitionCount, scrubResult.goodPartitions); + assertEquals(0, scrubResult.badPartitions); + assertEquals(0, scrubResult.emptyPartitions); + } + finally + { + releaseExcept(result, consumedByTxn); + LifecycleTransaction.waitForDeletions(); + } + } + + /** + * The ALIGNED layout, which is what extent sharing costs: a child's Data.db starts with up to 64 KiB of the + * parent's previous compression chunk, so its {@code offsets[0]} is that pad instead of 0 and every physical + * offset in it is shifted. + * + *

The layout is forced on rather than requiring a filesystem that can share extents -- no laptop and no CI + * box can, and this must not be a test that only runs on xfs. The layout belongs to {@code copyPlan}, not to + * the mechanism: a padded range that gets copied produces a byte-identical child, so copying here exercises + * exactly the file a reflink would have produced. What forcing it does NOT cover is the ioctl itself. + * + *

Everything is asserted through the ordinary readers, because the point is that nothing downstream notices. + * The one consumer that did, and had to be fixed, is {@code MmappedRegions}: it placed segments at a cumulative + * sum of chunk lengths seeded at physical 0, so a padded file's last chunk ran off the end of the last mapped + * region. {@code test/conf/cassandra.yaml} sets {@code disk_access_mode: mmap}, so every read below goes through + * that path -- which is why these content assertions are the regression test for it, and why a child of more + * than one chunk is not enough: it has to be read to the last byte. + */ + @Test + public void alignedChildrenAreReadableEverywhere() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + // Big enough that the parent spans several 64 KiB alignment units, so the pads are real residues of + // O(i) rather than just "everything before this chunk". + insertPartitions(400, 5, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + assertEquals("this test needs the mmap read path to be the one under test", + Config.DiskAccessMode.mmap, DatabaseDescriptor.getDiskAccessMode()); + assertTrue("the parent must span several alignment units for the residues to mean anything", + parent.descriptor.fileFor(Component.DATA).length() > 4 * 64 * 1024); + + Result result; + ZeroCopySSTableSplitter.forceAlignedLayoutForTesting = true; + try + { + result = ZeroCopySSTableSplitter.split(parent, 4, null); + } + finally + { + ZeroCopySSTableSplitter.forceAlignedLayoutForTesting = false; + } + + try + { + assertEquals(4, result.children.size()); + + // Guard the guard: without a padded child this test asserts nothing new. Only the first child can + // legitimately have no pad, its first chunk being at physical 0. + assertEquals("the first child starts at physical 0 and cannot be padded", + 0, result.children.get(0).headPadBytes); + int padded = 0; + for (Child child : result.children) + { + if (child.headPadBytes > 0) + padded++; + assertTrue("head pad must be under one alignment unit", child.headPadBytes < 64 * 1024); + assertEquals("offsets[0] must be the head pad", + child.headPadBytes, child.reader.getCompressionMetadata().chunkFor(0).offset); + assertEquals("the pad is on disk and nowhere else", + child.onDiskLength(), child.descriptor.fileFor(Component.DATA).length()); + // The uncompressed dead prefix is a DIFFERENT thing and must not have moved: the head pad is + // physical, the dead prefix is where the first partition sits in uncompressed space. + RowIndexEntry first = child.reader.getPosition(child.first, SSTableReader.Operator.EQ, false); + assertNotNull(first); + assertEquals(child.deadPrefixBytes, first.position); + } + assertTrue("no child was padded; the aligned layout was not exercised", padded > 0); + assertEquals(sumHeadPad(result), result.totalHeadPadBytes); + assertTrue("the pad is accounted for in the result", result.totalHeadPadBytes > 0); + + // Reading, in every way there is to read. + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + + // The last byte of every child, which is the read the MmappedRegions bug broke and nothing else did. + for (Child child : result.children) + { + try (RandomAccessReader in = child.reader.openDataReader()) + { + in.seek(child.reader.uncompressedLength() - 1); + in.readByte(); + } + } + + // Digest.crc32 covers the pad, because Verifier CRCs the whole physical file with no reference to + // CompressionInfo.db. Extended verification also walks Data.db linearly and rebuilds the index. + for (Child child : result.children) + { + assertEquals(String.valueOf(crc32Of(child.descriptor.fileFor(Component.DATA))), + readDigest(child.descriptor)); + try (Verifier verifier = new Verifier(cfs, child.reader, true, + Verifier.options().extendedVerification(true).build())) + { + verifier.verify(); + } + } + } + finally + { + release(result); + } + + // And from a cold open, where CompressionMetadata is built from the file length rather than handed over. + List reopened = new ArrayList<>(); + try + { + for (Child child : result.children) + reopened.add(SSTableReader.open(child.descriptor, child.components, cfs.metadata)); + assertConcatenatedContentEquals(parent, reopened); + for (int i = 0; i < reopened.size(); i++) + assertEquals(result.children.get(i).onDiskLength(), + reopened.get(i).getCompressionMetadata().compressedFileLength); + } + finally + { + for (SSTableReader reader : reopened) + reader.selfRef().release(); + } + } + + /** + * Digest.crc32 is optional. It is the only component whose cost is proportional to the DATA rather than to + * the index -- one full sequential read of every child -- so with the extents shared it is the entire + * remaining cost of a split, and {@code zero_copy_split_digest_enabled: false} takes a split down to its + * Index.db pass. + * + *

What this pins is that skipping it is a supported state and not a broken one: + *

    + *
  • the file does not exist, TOC does not claim it, and the component set does not contain it -- the + * three have to agree or {@code SSTable.discoverComponentsFor} and the transaction's file bookkeeping + * disagree about what belongs to the sstable;
    • + *
  • the children still open, read and scan identically, from memory and from a cold open;
    • + *
  • {@code Verifier} still passes, and passes by the documented route: a missing digest makes it say so + * and upgrade to a full extended verification rather than fail. That upgrade is the whole cost of this + * option, so it is asserted directly rather than inferred from "verify did not throw".
    • + *
+ */ + @Test + public void digestIsOptional() throws Throwable + { + createCompressedTable(4); + disableCompaction(); + insertPartitions(60, 4, 480); + flush(); + + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + SSTableReader parent = onlySSTable(cfs); + + Result result; + DatabaseDescriptor.setZeroCopySplitDigestEnabled(false); + try + { + result = ZeroCopySSTableSplitter.split(parent, 3, null); + } + finally + { + DatabaseDescriptor.setZeroCopySplitDigestEnabled(true); + } + + try + { + assertEquals(3, result.children.size()); + for (Child child : result.children) + { + String context = "child " + child.descriptor; + assertFalse(context + ": Digest.crc32 must not have been written", + child.descriptor.fileFor(Component.DIGEST).exists()); + assertFalse(context + ": DIGEST must not be a component", child.components.contains(Component.DIGEST)); + assertFalse(context + ": TOC must not list DIGEST", + SSTable.readTOC(child.descriptor, false).contains(Component.DIGEST)); + assertFalse(context + ": nothing on disk may claim DIGEST", + SSTable.discoverComponentsFor(child.descriptor).contains(Component.DIGEST)); + } + + // Everything else is unchanged, including the components that ARE written. + assertStructure(cfs, parent, result); + assertComponents(cfs, result); + assertConcatenatedContentEquals(parent, readers(result)); + assertPointReads(parent, result); + + // The documented Verifier fallback: not quick, not extended, no digest -> says so, then does the + // full walk and succeeds. + for (Child child : result.children) + { + List output = new ArrayList<>(); + OutputHandler handler = new OutputHandler.LogOutput() + { + @Override + public void output(String msg) + { + output.add(msg); + } + }; + try (Verifier verifier = new Verifier(cfs, child.reader, handler, true, + Verifier.options().extendedVerification(false).build())) + { + verifier.verify(); + } + assertTrue("Verifier did not report the missing digest: " + output, + output.stream().anyMatch(m -> m.contains("Data digest missing"))); + assertTrue("Verifier did not fall through to the extended walk: " + output, + output.stream().anyMatch(m -> m.contains("Extended Verify requested"))); + } + + // ...and the quick path, which never looks at the digest at all. + for (Child child : result.children) + { + try (Verifier verifier = new Verifier(cfs, child.reader, true, + Verifier.options().quick(true).build())) + { + verifier.verify(); + } + } + } + finally + { + release(result); + } + + // A cold open must not miss the component either: componentsFor() rediscovers from TOC. + List reopened = new ArrayList<>(); + try + { + for (Child child : result.children) + reopened.add(SSTableReader.open(child.descriptor, child.components, cfs.metadata)); + assertConcatenatedContentEquals(parent, reopened); + } + finally + { + for (SSTableReader reader : reopened) + reader.selfRef().release(); + } + } + + /** An uncompressed parent is refused up front rather than producing a child with a misaligned CRC.db. */ + @Test + public void uncompressedParentIsRefused() throws Throwable + { + createTable("CREATE TABLE %s (pk text, ck int, val text, PRIMARY KEY (pk, ck)) " + + "WITH compression = {'enabled': 'false'}"); + disableCompaction(); + insertPartitions(10, 2, 300); + flush(); + + SSTableReader parent = onlySSTable(getCurrentColumnFamilyStore()); + assertFalse(parent.compression); + assertFalse(ZeroCopySSTableSplitter.isSupported(parent)); + + try + { + ZeroCopySSTableSplitter.split(parent, 2, null); + fail("expected an uncompressed parent to be refused"); + } + catch (UnsupportedOperationException e) + { + assertTrue(e.getMessage(), + e.getMessage().startsWith(ZeroCopySSTableSplitter.UNCOMPRESSED_UNSUPPORTED_MESSAGE)); + } + } + + // ---------------------------------------------------------------------------------------------------- + // Content equivalence + // ---------------------------------------------------------------------------------------------------- + + /** + * The single most important assertion in this file: the children, scanned in order and concatenated, + * produce exactly the parent's partition stream -- same keys in the same order, same partition level + * deletions, same rows/range tombstones, same cells, same timestamps. + */ + private static void assertConcatenatedContentEquals(SSTableReader parent, List children) + { + int compared = 0; + try (ISSTableScanner parentScanner = parent.getScanner()) + { + for (SSTableReader child : children) + { + try (ISSTableScanner childScanner = child.getScanner()) + { + while (childScanner.hasNext()) + { + assertTrue("children yielded more partitions than the parent has (at " + compared + ')', + parentScanner.hasNext()); + try (UnfilteredRowIterator expected = parentScanner.next(); + UnfilteredRowIterator actual = childScanner.next()) + { + assertSamePartition(expected, actual); + } + compared++; + } + } + } + assertFalse("the parent has partitions that no child covers (after " + compared + ')', + parentScanner.hasNext()); + } + assertTrue("nothing was compared", compared > 0); + } + + private static void assertSamePartition(UnfilteredRowIterator expected, UnfilteredRowIterator actual) + { + String context = "partition " + expected.partitionKey(); + assertEquals(context, expected.partitionKey(), actual.partitionKey()); + assertEquals(context + ": partition level deletion", + expected.partitionLevelDeletion(), actual.partitionLevelDeletion()); + assertEquals(context + ": static row", expected.staticRow(), actual.staticRow()); + assertEquals(context + ": columns", expected.columns(), actual.columns()); + assertEquals(context + ": reverse order", expected.isReverseOrder(), actual.isReverseOrder()); + + int i = 0; + while (expected.hasNext()) + { + assertTrue(context + ": child ran out of rows after " + i, actual.hasNext()); + assertEquals(context + ": unfiltered " + i, expected.next(), actual.next()); + i++; + } + assertFalse(context + ": child has extra rows after " + i, actual.hasNext()); + assertTrue(context + ": expected at least one row", i > 0); + } + + /** Every parent key is owned by exactly one child, reads back identically there, and is absent elsewhere. */ + private void assertPointReads(SSTableReader parent, Result result) throws IOException + { + ColumnFilter columns = ColumnFilter.all(parent.metadata()); + for (Rec rec : readIndex(parent.descriptor)) + { + DecoratedKey key = parent.decorateKey(rec.key); + int owners = 0; + for (Child child : result.children) + { + boolean inRange = key.compareTo(child.first) >= 0 && key.compareTo(child.last) <= 0; + RowIndexEntry entry = child.reader.getPosition(key, SSTableReader.Operator.EQ, false); + if (!inRange) + { + assertNull("child " + child.descriptor + " must not contain " + key, entry); + continue; + } + owners++; + // getPosition(EQ) consults the bloom filter first, so a null here would also be a filter + // false negative, i.e. silent data loss. + assertNotNull("child " + child.descriptor + " lost " + key, entry); + try (UnfilteredRowIterator expected = parent.rowIterator(key, Slices.ALL, columns, false, NOOP); + UnfilteredRowIterator actual = child.reader.rowIterator(key, Slices.ALL, columns, false, NOOP)) + { + assertSamePartition(expected, actual); + } + } + assertEquals("exactly one child must own " + key, 1, owners); + } + } + + /** Clustering-level slice reads inside wide partitions: this is what navigates the copied promoted index. */ + private void assertSliceReadsMatch(ColumnFamilyStore cfs, SSTableReader parent, Result result, int rowsPerPartition) + throws IOException + { + TableMetadata metadata = cfs.metadata(); + ClusteringComparator comparator = metadata.comparator; + Slices slices = Slices.with(comparator, Slice.make(comparator.make(rowsPerPartition / 3), + comparator.make(2 * rowsPerPartition / 3))); + ColumnFilter columns = ColumnFilter.all(metadata); + + int checked = 0; + for (Child child : result.children) + { + for (Rec rec : readIndex(child.descriptor)) + { + DecoratedKey key = metadata.partitioner.decorateKey(rec.key); + for (boolean reversed : new boolean[]{ false, true }) + { + try (UnfilteredRowIterator expected = parent.rowIterator(key, slices, columns, reversed, NOOP); + UnfilteredRowIterator actual = child.reader.rowIterator(key, slices, columns, reversed, NOOP)) + { + assertSamePartition(expected, actual); + } + } + checked++; + } + } + assertTrue(checked > 0); + } + + // ---------------------------------------------------------------------------------------------------- + // Structural assertions -- FACT 9 recomputed independently of the implementation + // ---------------------------------------------------------------------------------------------------- + + private void assertStructure(ColumnFamilyStore cfs, SSTableReader parent, Result result) throws IOException + { + List parentIndex = readIndex(parent.descriptor); + int n = parentIndex.size(); + + CompressionMetadata meta = parent.getCompressionMetadata(); + int chunkLength = meta.chunkLength(); + long parentUncompressed = parent.uncompressedLength(); + assertEquals(meta.dataLength, parentUncompressed); + long parentPhysical = parent.descriptor.fileFor(Component.DATA).length(); + long[] parentOffsets = readChunkOffsets(parent.descriptor); + int parentDataChunks = (int) ((parentUncompressed + chunkLength - 1) / chunkLength); + // A flushed parent's offsets table stops at the last data chunk and its metadata length is the physical + // length; a compaction-produced one has a trailing chunk beyond both. Either is legal input. + assertTrue("offsets table must address every data chunk", parentOffsets.length >= parentDataChunks); + + StatsMetadata parentStats = parent.getSSTableMetadata(); + + long physicalSum = 0; + long deadSum = 0; + long duplicatedSum = 0; + long partitionSum = 0; + int cursor = 0; + long previousLastChunk = -1; + + for (Child child : result.children) + { + String context = "child " + child.descriptor; + int from = cursor; + assertTrue(context + " starts past the end of the parent", from < n); + int to = from + (int) child.partitionCount; + assertTrue(context + " runs past the end of the parent", to <= n); + + assertEquals(context + ": first key", parentIndex.get(from).key, child.first.getKey()); + assertEquals(context + ": last key", parentIndex.get(to - 1).key, child.last.getKey()); + assertEquals(context + ": reader.first", child.first, child.reader.first); + assertEquals(context + ": reader.last", child.last, child.reader.last); + assertTrue(context + ": first > last", child.first.compareTo(child.last) <= 0); + + long lo = parentIndex.get(from).position; + long hi = to < n ? parentIndex.get(to).position : parentUncompressed; + long firstChunk = lo / chunkLength; + long lastChunk = (hi - 1) / chunkLength; + long dataLength = hi - firstChunk * chunkLength; + long physicalBytes = chunkEndOnDisk(parentOffsets, lastChunk, parentPhysical) + - parentOffsets[(int) firstChunk]; + + assertEquals(context + ": firstChunk", firstChunk, child.firstChunk); + assertEquals(context + ": lastChunk", lastChunk, child.lastChunk); + assertEquals(context + ": shift", firstChunk * chunkLength, child.shift); + assertEquals(context + ": deadPrefixBytes", lo % chunkLength, child.deadPrefixBytes); + assertEquals(context + ": dataLength", dataLength, child.dataLength); + assertEquals(context + ": physicalBytes", physicalBytes, child.physicalBytes); + // (C - 1) * L < Dp <= C * L + long chunkCount = lastChunk - firstChunk + 1; + assertTrue(context + ": (C-1)*L < Dp", (chunkCount - 1) * chunkLength < dataLength); + assertTrue(context + ": Dp <= C*L", dataLength <= chunkCount * chunkLength); + + // FACT 6: not one byte of trailing slack on disk. The head pad is the one thing that may sit in + // front of the run -- zero unless the child was aligned so its extents could be shared with the + // parent -- so every physical length here is measured from the pad, not from 0. + long pad = child.headPadBytes; + assertTrue(context + ": head pad must be under one alignment unit", pad < 64 * 1024); + assertTrue(context + ": head pad must be O(i) mod alignment, or nothing", + pad == 0 || pad == parentOffsets[(int) firstChunk] % (64 * 1024)); + assertEquals(context + ": on-disk Data.db length", pad + physicalBytes, child.onDiskLength()); + assertEquals(context + ": physical Data.db length", + pad + physicalBytes, child.descriptor.fileFor(Component.DATA).length()); + assertEquals(context + ": uncompressedLength", dataLength, child.reader.uncompressedLength()); + + CompressionMetadata childMeta = child.reader.getCompressionMetadata(); + assertEquals(context + ": offsets[0]", pad, childMeta.chunkFor(0).offset); + assertEquals(context + ": chunkLength", chunkLength, childMeta.chunkLength()); + assertEquals(context + ": maxCompressedLength", meta.maxCompressedLength(), childMeta.maxCompressedLength()); + assertEquals(context + ": CompressionInfo dataLength", dataLength, childMeta.dataLength); + assertEquals(context + ": compressedFileLength", pad + physicalBytes, childMeta.compressedFileLength); + // the last chunk plus its 4 byte inline CRC32 must end exactly at the physical end of the file + CompressionMetadata.Chunk tail = childMeta.chunkFor((chunkCount - 1) * chunkLength); + assertEquals(context + ": last chunk overruns the file", + pad + physicalBytes, tail.offset + tail.length + 4); + + // Index.db: same keys, same promoted blobs, positions rebased by exactly shift. + List childIndex = readIndex(child.descriptor); + assertEquals(context + ": partition count", child.partitionCount, childIndex.size()); + assertEquals(context + ": first index position", lo % chunkLength, childIndex.get(0).position); + assertTrue(context + ": first index position must be inside the first chunk", + childIndex.get(0).position < chunkLength); + for (int r = 0; r < childIndex.size(); r++) + { + Rec expected = parentIndex.get(from + r); + Rec actual = childIndex.get(r); + assertEquals(context + ": key " + r, expected.key, actual.key); + assertEquals(context + ": position " + r, + expected.position - firstChunk * chunkLength, actual.position); + assertArrayEquals(context + ": promoted index blob " + r + " must be copied verbatim", + expected.promoted, actual.promoted); + } + + RowIndexEntry firstEntry = child.reader.getPosition(child.first, SSTableReader.Operator.EQ, false); + assertNotNull(context + ": cannot find its own first key", firstEntry); + assertEquals(context + ": first entry position", lo % chunkLength, firstEntry.position); + + // Statistics.db: the header and the min/max encoding bases MUST be inherited verbatim or every + // relocated row silently decodes wrong; the two derived fields must be recomputed. + StatsMetadata childStats = child.reader.getSSTableMetadata(); + assertEquals(context + ": header columns", parent.header.columns(), child.reader.header.columns()); + assertEquals(context + ": header stats", parent.header.stats(), child.reader.header.stats()); + assertEquals(context + ": minTimestamp", parentStats.minTimestamp, childStats.minTimestamp); + assertEquals(context + ": maxTimestamp", parentStats.maxTimestamp, childStats.maxTimestamp); + assertEquals(context + ": minLocalDeletionTime", + parentStats.minLocalDeletionTime, childStats.minLocalDeletionTime); + assertEquals(context + ": maxLocalDeletionTime", + parentStats.maxLocalDeletionTime, childStats.maxLocalDeletionTime); + assertEquals(context + ": minTTL", parentStats.minTTL, childStats.minTTL); + assertEquals(context + ": maxTTL", parentStats.maxTTL, childStats.maxTTL); + assertEquals(context + ": sstableLevel", parentStats.sstableLevel, childStats.sstableLevel); + assertEquals(context + ": repairedAt", parentStats.repairedAt, childStats.repairedAt); + assertEquals(context + ": originatingHostId", parentStats.originatingHostId, childStats.originatingHostId); + assertEquals(context + ": compressionRatio", + (double) (pad + physicalBytes) / dataLength, childStats.compressionRatio, 1e-9); + assertEquals(context + ": estimatedPartitionSize count", + child.partitionCount, childStats.estimatedPartitionSize.count()); + + physicalSum += physicalBytes; + deadSum += lo % chunkLength; + partitionSum += child.partitionCount; + if (previousLastChunk == firstChunk) + duplicatedSum += chunkEndOnDisk(parentOffsets, firstChunk, parentPhysical) + - parentOffsets[(int) firstChunk]; + previousLastChunk = lastChunk; + cursor = to; + } + + assertEquals("children must cover every parent partition exactly once", n, cursor); + assertEquals("partition counts must sum to the parent's", n, partitionSum); + assertEquals(physicalSum, result.totalPhysicalBytesCopied); + assertEquals(deadSum, result.totalDeadPrefixBytes); + assertEquals(duplicatedSum, result.duplicatedChunkBytes); + assertEquals(parent.first, result.children.get(0).first); + assertEquals(parent.last, result.children.get(result.children.size() - 1).last); + } + + /** + * End of chunk {@code k}, inclusive of its 4-byte inline CRC32, derived from the offsets table exactly as it + * exists in CompressionInfo.db. + * + *

The physical file length is the end of chunk {@code k} only when there is no entry after {@code k}. + * This used to key off {@code ceil(dataLength / chunkLength)} instead -- the same formula production used -- + * so it agreed with the code it was supposed to be checking, and both were wrong for a compaction-produced + * parent, which carries one extra chunk offset past the end of its data. See + * {@link #splitOfCompactionProducedParentDoesNotAbsorbTheTrailingChunk}. + */ + private static long chunkEndOnDisk(long[] offsets, long k, long compressedFileLength) + { + assertTrue("chunk " + k + " is not in the offsets table", k >= 0 && k < offsets.length); + return k + 1 < offsets.length ? offsets[(int) (k + 1)] : compressedFileLength; + } + + /** The chunk offsets as stored, parsed here rather than through {@link CompressionMetadata}. */ + private static long[] readChunkOffsets(Descriptor descriptor) throws IOException + { + try (FileInputStreamPlus in = descriptor.fileFor(Component.COMPRESSION_INFO).newInputStream()) + { + in.readUTF(); // compressor class name + int optionCount = in.readInt(); + for (int i = 0; i < optionCount; i++) + { + in.readUTF(); + in.readUTF(); + } + in.readInt(); // chunkLength + if (descriptor.version.hasMaxCompressedLength()) + in.readInt(); // maxCompressedLength + in.readLong(); // dataLength + long[] offsets = new long[in.readInt()]; + for (int i = 0; i < offsets.length; i++) + offsets[i] = in.readLong(); + return offsets; + } + } + + // ---------------------------------------------------------------------------------------------------- + // Component sanity + // ---------------------------------------------------------------------------------------------------- + + private void assertComponents(ColumnFamilyStore cfs, Result result) throws IOException + { + TableMetadata metadata = cfs.metadata(); + for (Child child : result.children) + { + String context = "child " + child.descriptor; + + // TOC.txt lists exactly the components that exist on disk, and nothing else exists on disk. + assertEquals(context + ": TOC", child.components, SSTable.readTOC(child.descriptor, false)); + assertEquals(context + ": files on disk", + child.components, SSTable.discoverComponentsFor(child.descriptor)); + assertTrue(context + ": no Filter.db", child.components.contains(Component.FILTER)); + assertTrue(context + ": no CompressionInfo.db", child.components.contains(Component.COMPRESSION_INFO)); + assertFalse(context + ": a compressed sstable must not have a CRC.db", + child.components.contains(Component.CRC)); + for (Component component : child.components) + assertTrue(context + ": missing " + component, child.descriptor.fileFor(component).exists()); + + // Digest.crc32, when it was written at all, is the decimal CRC32 of every physical byte of Data.db. + // It is optional (zero_copy_split_digest_enabled), and the two states must be exactly two states: + // the component is claimed and the file is right, or it is claimed nowhere and exists nowhere. A + // file on disk that TOC does not list, or the reverse, is what the checks above would catch. + if (child.components.contains(Component.DIGEST)) + { + assertEquals(context + ": digest", + Long.toString(crc32Of(child.descriptor.fileFor(Component.DATA))), + readDigest(child.descriptor)); + } + else + { + assertFalse(context + ": Digest.crc32 must not exist when it is not a component", + child.descriptor.fileFor(Component.DIGEST).exists()); + } + + // Statistics.db: all four metadata components must deserialise standalone. This is the component + // whose loss is unrecoverable -- it carries the SerializationHeader every relocated row is decoded + // against, plus the repair state -- and it is written here through a SequentialWriter (so that it is + // fsynced) rather than through MetadataSerializer.rewriteSSTableMetadata, so assert the bytes are + // still exactly what the deserialiser expects. + Map childMetadata = + child.descriptor.getMetadataSerializer() + .deserialize(child.descriptor, EnumSet.allOf(MetadataType.class)); + for (MetadataType type : MetadataType.values()) + assertNotNull(context + ": Statistics.db is missing " + type, childMetadata.get(type)); + + // ...and it is written in place, so no tmp file may survive the split. + assertFalse(context + ": leftover Statistics.db tmp file", + new File(child.descriptor.tmpFilenameFor(Component.STATS)).exists()); + + List childIndex = readIndex(child.descriptor); + + // Bloom filter: a false negative is data loss, so every owned key must be present. + try (FileInputStreamPlus in = child.descriptor.fileFor(Component.FILTER).newInputStream(); + IFilter filter = BloomFilterSerializer.deserialize(in, child.descriptor.version.hasOldBfFormat())) + { + for (Rec rec : childIndex) + assertTrue(context + ": bloom filter false negative", + filter.isPresent(metadata.partitioner.decorateKey(rec.key))); + } + + // Summary.db deserialises standalone with the schema's index interval (otherwise the read path + // silently deletes it and rebuilds), and carries the child's own first/last keys. + try (DataInputStream in = new DataInputStream( + Files.newInputStream(child.descriptor.fileFor(Component.SUMMARY).toPath()))) + { + IndexSummary summary = IndexSummary.serializer.deserialize(in, + metadata.partitioner, + metadata.params.minIndexInterval, + metadata.params.maxIndexInterval); + try + { + assertTrue(context + ": empty summary", summary.size() > 0); + assertEquals(context + ": summary minIndexInterval", + metadata.params.minIndexInterval, summary.getMinIndexInterval()); + } + finally + { + summary.close(); + } + assertEquals(context + ": summary first key", + child.first, metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(in))); + assertEquals(context + ": summary last key", + child.last, metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(in))); + } + } + } + + // ---------------------------------------------------------------------------------------------------- + // Plumbing + // ---------------------------------------------------------------------------------------------------- + + /** One parent/child Index.db record, parsed independently of {@code ZeroCopySSTableSplitter}. */ + private static final class Rec + { + final ByteBuffer key; + final long position; + final byte[] promoted; // null when promotedSize == 0 + + Rec(ByteBuffer key, long position, byte[] promoted) + { + this.key = key; + this.position = position; + this.promoted = promoted; + } + } + + private static List readIndex(Descriptor descriptor) throws IOException + { + List records = new ArrayList<>(); + try (RandomAccessReader in = RandomAccessReader.open(descriptor.fileFor(Component.PRIMARY_INDEX))) + { + long length = in.length(); + while (in.getFilePointer() != length) + { + ByteBuffer key = ByteBufferUtil.readWithShortLength(in); + long position = RowIndexEntry.Serializer.readPosition(in); + int promotedSize = (int) in.readUnsignedVInt(); + byte[] promoted = null; + if (promotedSize > 0) + { + promoted = new byte[promotedSize]; + in.readFully(promoted); + } + records.add(new Rec(key, position, promoted)); + } + } + return records; + } + + private static long crc32Of(File file) throws IOException + { + CRC32 crc = new CRC32(); + byte[] buffer = new byte[8192]; + try (FileInputStreamPlus in = file.newInputStream()) + { + int n; + while ((n = in.read(buffer)) > 0) + crc.update(buffer, 0, n); + } + return crc.getValue(); + } + + private static String readDigest(Descriptor descriptor) throws IOException + { + byte[] bytes = Files.readAllBytes(descriptor.fileFor(Component.DIGEST).toPath()); + return new String(bytes, StandardCharsets.UTF_8).trim(); + } + + private static SSTableReader onlySSTable(ColumnFamilyStore cfs) + { + Set live = cfs.getLiveSSTables(); + assertEquals("expected exactly one sstable", 1, live.size()); + return live.iterator().next(); + } + + private static List readers(Result result) + { + List readers = new ArrayList<>(result.children.size()); + for (Child child : result.children) + readers.add(child.reader); + return readers; + } + + private static long sumHeadPad(Result result) + { + long sum = 0; + for (Child child : result.children) + sum += child.headPadBytes; + return sum; + } + + private static Child firstChildWithDeadPrefix(Result result) + { + for (Child child : result.children) + { + if (child.deadPrefixBytes > 0) + return child; + } + return null; + } + + private static void release(Result result) + { + releaseExcept(result, null); + } + + /** + * A child handed to {@link LifecycleTransaction#offline} is owned by that transaction's dummy Tracker, + * which releases it on close (LifecycleTransaction.java:143-149). Releasing it again here would throw + * "Attempted to release a reference that has already been released" and mask the real assertions. + */ + private static void releaseExcept(Result result, SSTableReader consumed) + { + for (Child child : result.children) + if (child.reader != consumed) + child.reader.selfRef().release(); + } + + private static ColumnFilter allColumns(ColumnFamilyStore cfs) + { + return ColumnFilter.all(cfs.metadata()); + } + + private String createCompressedTable(int chunkLengthInKb) throws Throwable + { + return createTable("CREATE TABLE %s (pk text, ck int, val text, PRIMARY KEY (pk, ck)) " + + "WITH compression = {'class': 'LZ4Compressor', 'chunk_length_in_kb': '" + + chunkLengthInKb + "'}"); + } + + private void insertPartitions(int partitions, int rowsPerPartition, int valueBytes) throws Throwable + { + for (int p = 0; p < partitions; p++) + for (int c = 0; c < rowsPerPartition; c++) + execute("INSERT INTO %s (pk, ck, val) VALUES (?, ?, ?)", key(p), c, randomText(valueBytes)); + } + + private static String key(int p) + { + return String.format("k%06d", p); + } + + /** Near-incompressible payload, so the sstable really does span many compression chunks. */ + private static String randomText(int length) + { + ThreadLocalRandom random = ThreadLocalRandom.current(); + char[] chars = new char[length]; + for (int i = 0; i < length; i++) + chars[i] = (char) ('!' + random.nextInt(94)); + return new String(chars); + } + + private static String fixedText(int length) + { + char[] chars = new char[length]; + Arrays.fill(chars, 'v'); + return new String(chars); + } +} diff --git a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java index 7194d3042f25..188e0276cb05 100644 --- a/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java +++ b/test/unit/org/apache/cassandra/io/util/MmappedRegionsTest.java @@ -345,6 +345,93 @@ public void testMapForCompressionMetadata() throws Exception } } + /** + * A compressed file whose first chunk does NOT start at physical 0, i.e. one carrying leading bytes that belong + * to no chunk. {@link org.apache.cassandra.io.sstable.ZeroCopySSTableSplitter} produces exactly this when it + * aligns a child's Data.db so its extents can be shared with the parent's. + *

+ * Segments are placed at a cumulative sum of {@code chunk.length + 4}, so seeding that sum at 0 rather than at + * the first chunk's offset mapped every region {@code pad} bytes too early and left the last {@code pad} bytes + * unmapped. MAX_SEGMENT_SIZE is forced down to one chunk per region so the multi-region form of the bug shows + * up, which a split of a test-sized sstable (one 2 GiB region) cannot reach. + */ + @Test + public void testMapForCompressionMetadataWithFrontPad() throws Exception + { + int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE; + MmappedRegions.MAX_SEGMENT_SIZE = 1024; + + int pad = 12345; + ByteBuffer buffer = allocateBuffer(128 * 1024); + File f = FileUtils.createTempFile("testMapForCompressionMetadataWithFrontPad", "1"); + f.deleteOnExit(); + File cf = FileUtils.createTempFile(f.name() + ".metadata", "1"); + cf.deleteOnExit(); + + // Write an ordinary compressed file, then rebuild it with `pad` junk bytes in front and shift every chunk + // offset by the same amount -- byte for byte what the splitter's aligned copy produces. + MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance)); + try (SequentialWriter writer = new CompressedSequentialWriter(f, cf.absolutePath(), + null, SequentialWriterOption.DEFAULT, + CompressionParams.snappy(), sstableMetadataCollector)) + { + writer.write(buffer); + writer.finish(); + } + + byte[] unpadded = java.nio.file.Files.readAllBytes(f.toPath()); + byte[] padded = new byte[pad + unpadded.length]; + new Random(1).nextBytes(padded); // the pad is junk, and must never be read + System.arraycopy(unpadded, 0, padded, pad, unpadded.length); + java.nio.file.Files.write(f.toPath(), padded); + + CompressionMetadata unshifted = new CompressionMetadata(cf.absolutePath(), unpadded.length, true); + int chunkCount = Ints.checkedCast((unshifted.dataLength + unshifted.chunkLength() - 1) / unshifted.chunkLength()); + Memory offsets = Memory.allocate(chunkCount * 8L); + for (int k = 0; k < chunkCount; k++) + offsets.setLong(k * 8L, unshifted.chunkFor((long) k * unshifted.chunkLength()).offset + pad); + unshifted.close(); + + CompressionMetadata metadata = new CompressionMetadata(cf.absolutePath(), CompressionParams.snappy(), + offsets, chunkCount * 8L, + 128 * 1024, padded.length); + try (ChannelProxy channel = new ChannelProxy(f); + MmappedRegions regions = MmappedRegions.map(channel, metadata)) + { + assertFalse(regions.isEmpty()); + int i = 0; + while (i < buffer.capacity()) + { + CompressionMetadata.Chunk chunk = metadata.chunkFor(i); + assertTrue("every chunk must sit past the pad", chunk.offset >= pad); + + MmappedRegions.Region region = regions.floor(chunk.offset); + assertNotNull(region); + + // one chunk per region, so the region must BE the chunk: this is the assertion that fails when the + // segment placement ignores the pad, and it fails for every region but the first + assertEquals(chunk.offset, region.offset()); + assertEquals(chunk.offset + chunk.length + 4, region.end()); + assertEquals(chunk.length + 4, region.buffer.duplicate().capacity()); + + // and the mapped bytes must be the file's bytes at that offset, not shifted by the pad + ByteBuffer mapped = region.buffer(); + assertEquals("mapped byte 0 of the chunk at " + chunk.offset, + padded[Ints.checkedCast(chunk.offset)], mapped.get(0)); + assertEquals("mapped last byte of the chunk at " + chunk.offset, + padded[Ints.checkedCast(chunk.offset + chunk.length + 3)], + mapped.get(chunk.length + 3)); + + i += metadata.chunkLength(); + } + } + finally + { + MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE; + metadata.close(); + } + } + @Test(expected = IllegalArgumentException.class) public void testIllegalArgForMap1() throws Exception { diff --git a/test/unit/org/apache/cassandra/io/util/ReflinkTest.java b/test/unit/org/apache/cassandra/io/util/ReflinkTest.java new file mode 100644 index 000000000000..a3ad95383960 --- /dev/null +++ b/test/unit/org/apache/cassandra/io/util/ReflinkTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.io.util; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Arrays; +import java.util.Random; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +/** + * {@link Reflink} can only actually share extents on xfs formatted with {@code -m reflink=1} or on btrfs, which + * is not the filesystem any of this runs on locally. So what is tested here is the CONTRACT, which holds either + * way: an attempt either shares the whole range or reports that it could not and leaves the destination alone, + * and the argument validation that stands between the caller and a silent {@code EINVAL} is unconditional. + *

+ * The assertions that need real support are guarded with {@code assumeTrue} and skip everywhere else; they are + * what proves the ioctl call itself is right, so this test is worth running on an xfs scratch mount -- + * {@code -Djava.io.tmpdir=/mnt/xfs-scratch} is enough, since that is where the files below are created. + */ +public class ReflinkTest +{ + private static final long A = Reflink.RANGE_ALIGNMENT; + private static final int SOURCE_LENGTH = (int) (4 * A); + + @BeforeClass + public static void setupDD() + { + DatabaseDescriptor.daemonInitialization(); + } + + @Before + public void forgetPreviousAnswers() + { + // The negative cache is JVM-global and keyed by directory, and every test here uses the same one. + Reflink.resetSupportCache(); + } + + /** + * The whole range is shared, or nothing is: there is no partial outcome to unwind. When it works the + * destination must be byte-identical to the source range; when it does not, the destination must still be + * untouched, because the caller's fallback assumes it is starting from an empty file. + */ + @Test + public void clonesTheWholeRangeOrLeavesTheDestinationEmpty() throws IOException + { + byte[] source = random(SOURCE_LENGTH); + File src = write("reflink-src", source); + File dst = FileUtils.createTempFile("reflink-dst", "1"); + dst.deleteOnExit(); + + boolean cloned; + try (FileChannel in = src.newReadChannel(); + FileChannel out = dst.newWriteChannel(File.WriteMode.OVERWRITE)) + { + cloned = Reflink.tryCloneRange(in, A, out, 0, 2 * A, dst.parent()); + out.force(true); + } + + if (cloned) + { + assertEquals("a clone must set the destination's length", 2 * A, dst.length()); + assertArrayEquals("shared bytes must be the source's bytes", + Arrays.copyOfRange(source, (int) A, (int) (3 * A)), readAll(dst)); + assertTrue("success must not poison the directory", Reflink.isPossibleIn(dst.parent())); + } + else + { + assertEquals("a refusal must write nothing at all", 0, dst.length()); + assertFalse("a refusal must be remembered for the directory", Reflink.isPossibleIn(dst.parent())); + } + } + + /** + * Shared extents are copy-on-write, not a shared inode: writing through one file must not change the other. + * This is the property that makes it safe to clone out of an sstable that is still being read. + */ + @Test + public void sharedExtentsAreCopyOnWrite() throws IOException + { + byte[] source = random(SOURCE_LENGTH); + File src = write("reflink-cow-src", source); + File dst = FileUtils.createTempFile("reflink-cow-dst", "1"); + dst.deleteOnExit(); + + try (FileChannel in = src.newReadChannel(); + FileChannel out = dst.newWriteChannel(File.WriteMode.OVERWRITE)) + { + assumeTrue("no filesystem support for sharing extents", + Reflink.tryCloneRange(in, 0, out, 0, 2 * A, dst.parent())); + out.position(0); + out.write(ByteBuffer.wrap(new byte[]{ (byte) ~source[0], (byte) ~source[1] })); + out.force(true); + } + + assertArrayEquals("the source must not have been modified through the clone", source, readAll(src)); + byte[] clone = readAll(dst); + assertEquals((byte) ~source[0], clone[0]); + assertEquals("and only the written bytes may differ", source[2], clone[2]); + } + + /** + * Misalignment is a caller bug -- the kernel answers it with a bare EINVAL, which would otherwise be + * indistinguishable from "this filesystem cannot do it" and get the directory written off for the lifetime + * of the process. + */ + @Test + public void unalignedArgumentsAreRejectedRatherThanRetriedAsACopy() throws IOException + { + File src = write("reflink-align-src", random(SOURCE_LENGTH)); + File dst = FileUtils.createTempFile("reflink-align-dst", "1"); + dst.deleteOnExit(); + + try (FileChannel in = src.newReadChannel(); + FileChannel out = dst.newWriteChannel(File.WriteMode.OVERWRITE)) + { + File dir = dst.parent(); + assertThatThrownBy(() -> Reflink.tryCloneRange(in, A + 1, out, 0, A, dir)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("srcOffset"); + assertThatThrownBy(() -> Reflink.tryCloneRange(in, 0, out, 4096, A, dir)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("dstOffset"); + assertThatThrownBy(() -> Reflink.tryCloneRange(in, 0, out, 0, A - 1, dir)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("length"); + assertThatThrownBy(() -> Reflink.tryCloneRange(in, 0, out, 0, 0, dir)) + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("positive"); + + assertEquals("nothing may have been written", 0, dst.length()); + } + } + + /** The alignment is what every caller has to arrange, so it must be a power of two they can mask with. */ + @Test + public void alignmentIsAPowerOfTwoAndAtLeastAPage() + { + assertEquals(0, Reflink.RANGE_ALIGNMENT & (Reflink.RANGE_ALIGNMENT - 1)); + assertTrue("must be at least the largest page size Linux uses", Reflink.RANGE_ALIGNMENT >= 64 * 1024); + } + + private static byte[] random(int length) + { + byte[] bytes = new byte[length]; + new Random(20260729L).nextBytes(bytes); + return bytes; + } + + private static File write(String name, byte[] bytes) throws IOException + { + File file = FileUtils.createTempFile(name, "1"); + file.deleteOnExit(); + try (SequentialWriter writer = new SequentialWriter(file)) + { + writer.write(bytes); + writer.finish(); + } + assertEquals(bytes.length, file.length()); + return file; + } + + private static byte[] readAll(File file) throws IOException + { + return java.nio.file.Files.readAllBytes(file.toPath()); + } +}