From 994087f26cb48ee6cc436da4541bdbe6da87552b Mon Sep 17 00:00:00 2001 From: Thomas Brady Date: Mon, 25 Nov 2024 11:55:44 -0800 Subject: [PATCH] Live to initentry conversion in lowest level live bucket merge --- src/bucket/BucketOutputIterator.cpp | 24 +++++++++ src/bucket/BucketUtils.h | 1 + src/bucket/LiveBucket.h | 3 ++ src/bucket/test/BucketListTests.cpp | 73 ++++++++++++++++++++++++++ src/bucket/test/BucketManagerTests.cpp | 4 ++ 5 files changed, 105 insertions(+) diff --git a/src/bucket/BucketOutputIterator.cpp b/src/bucket/BucketOutputIterator.cpp index dbc9e7c159..93d8c85d18 100644 --- a/src/bucket/BucketOutputIterator.cpp +++ b/src/bucket/BucketOutputIterator.cpp @@ -157,6 +157,30 @@ BucketOutputIterator::put(typename BucketT::EntryT const& e) mBuf = std::make_unique(); } + // If BucketT is a live bucket, and this is the lowest level of the + // bucketlist, we also want to convert each LIVEENTRY to an INITENTRY. + // This is because each level of the bucket list contains only one entry + // per key, and per CAP-0020, INITENTRY implies that no entry with + // the same ledger key exists in an older bucket. Therefore, all entries + // of type LIVEENTRY in the lowest level should be of type INITENTRY. + if constexpr (std::is_same_v) + { + if (!mKeepTombstoneEntries /* lowest level */ && + e.type() == LIVEENTRY && + protocolVersionStartsFrom( + mMeta.ledgerVersion, + LiveBucket:: + FIRST_PROTOCOL_CONVERTING_BOTTOM_LEVEL_LIVE_TO_INIT)) + { + ++mMergeCounters.mOutputIteratorLiveToInitRewrites; + ++mMergeCounters.mOutputIteratorBufferUpdates; + auto eCopy = e; + eCopy.type(INITENTRY); + *mBuf = eCopy; + return; + } + } + // In any case, replace *mBuf with e. ++mMergeCounters.mOutputIteratorBufferUpdates; *mBuf = e; diff --git a/src/bucket/BucketUtils.h b/src/bucket/BucketUtils.h index ebc98f79dd..7e66d6932e 100644 --- a/src/bucket/BucketUtils.h +++ b/src/bucket/BucketUtils.h @@ -64,6 +64,7 @@ struct MergeCounters uint64_t mOutputIteratorTombstoneElisions{0}; uint64_t mOutputIteratorBufferUpdates{0}; uint64_t mOutputIteratorActualWrites{0}; + uint64_t mOutputIteratorLiveToInitRewrites{0}; MergeCounters& operator+=(MergeCounters const& delta); bool operator==(MergeCounters const& other) const; }; diff --git a/src/bucket/LiveBucket.h b/src/bucket/LiveBucket.h index a34a3d8393..deb7ad5f47 100644 --- a/src/bucket/LiveBucket.h +++ b/src/bucket/LiveBucket.h @@ -73,6 +73,9 @@ class LiveBucket : public BucketBase, ProtocolVersion::V_11; static constexpr ProtocolVersion FIRST_PROTOCOL_SHADOWS_REMOVED = ProtocolVersion::V_12; + static constexpr ProtocolVersion + FIRST_PROTOCOL_CONVERTING_BOTTOM_LEVEL_LIVE_TO_INIT = + ProtocolVersion::V_23; static void checkProtocolLegality(BucketEntry const& entry, uint32_t protocolVersion); diff --git a/src/bucket/test/BucketListTests.cpp b/src/bucket/test/BucketListTests.cpp index 5bb6a71d52..f6baf4dc00 100644 --- a/src/bucket/test/BucketListTests.cpp +++ b/src/bucket/test/BucketListTests.cpp @@ -454,6 +454,79 @@ TEST_CASE_VERSIONS("hot archive bucket tombstones expire at bottom level", }); } +TEST_CASE_VERSIONS( + "live bucket entries converted to init enties at bottom level", + "[bucket][bucketlist]") +{ + VirtualClock clock; + Config const& cfg = getTestConfig(); + + for_versions_with_differing_bucket_logic(cfg, [&](Config const& cfg) { + Application::pointer app = createTestApplication(clock, cfg); + LiveBucketList bl; + BucketManager& bm = app->getBucketManager(); + auto& mergeTimer = bm.getMergeTimer(); + CLOG_INFO(Bucket, "Establishing random bucketlist"); + for (uint32_t i = 0; i < LiveBucketList::kNumLevels; ++i) + { + auto& level = bl.getLevel(i); + level.setCurr(LiveBucket::fresh( + bm, getAppLedgerVersion(app), {}, // No init entries. + LedgerTestUtils::generateValidUniqueLedgerEntries(8), + LedgerTestUtils::generateValidLedgerEntryKeysWithExclusions( + {CONFIG_SETTING}, 5), + /*countMergeEvents=*/true, clock.getIOContext(), + /*doFsync=*/true)); + level.setSnap(LiveBucket::fresh( + bm, getAppLedgerVersion(app), {}, + LedgerTestUtils::generateValidUniqueLedgerEntries(8), + LedgerTestUtils::generateValidLedgerEntryKeysWithExclusions( + {CONFIG_SETTING}, 5), + /*countMergeEvents=*/true, clock.getIOContext(), + /*doFsync=*/true)); + } + + auto countNonBottomLevelEntries = [&] { + auto size = 0; + for (uint32_t i = 0; i < LiveBucketList::kNumLevels - 1; ++i) + { + auto& level = bl.getLevel(i); + size += countEntries(level.getCurr()); + size += countEntries(level.getSnap()); + } + return size; + }; + + auto ledger = 1; + // Close ledgers until all entries have merged into the bottom level + // bucket + while (countNonBottomLevelEntries() != 0) + { + bl.addBatch(*app, ledger, getAppLedgerVersion(app), {}, {}, {}); + ++ledger; + } + + auto bottomCurr = bl.getLevel(LiveBucketList::kNumLevels - 1).getCurr(); + EntryCounts e(bottomCurr); + + if (protocolVersionStartsFrom( + cfg.LEDGER_PROTOCOL_VERSION, + LiveBucket:: + FIRST_PROTOCOL_CONVERTING_BOTTOM_LEVEL_LIVE_TO_INIT)) + { + // Assert that init entries are converted to live entries + // at the lowest level. + REQUIRE(e.nLive == 0); + REQUIRE(e.nInitOrArchived != 0); + } + else + { + REQUIRE(e.nLive != 0); + REQUIRE(e.nInitOrArchived == 0); + } + }); +} + TEST_CASE_VERSIONS("live bucket tombstones expire at bottom level", "[bucket][bucketlist][tombstones]") { diff --git a/src/bucket/test/BucketManagerTests.cpp b/src/bucket/test/BucketManagerTests.cpp index fc5390653f..bb017c06f3 100644 --- a/src/bucket/test/BucketManagerTests.cpp +++ b/src/bucket/test/BucketManagerTests.cpp @@ -760,6 +760,8 @@ class StopAndRestartBucketMergesTest mMergeCounters.mDeadEntryShadowElisions); CLOG_INFO(Bucket, "OutputIteratorTombstoneElisions: {}", mMergeCounters.mOutputIteratorTombstoneElisions); + CLOG_INFO(Bucket, "OutputIteratorLiveToInitConversions: {}", + mMergeCounters.mOutputIteratorLiveToInitRewrites); CLOG_INFO(Bucket, "OutputIteratorBufferUpdates: {}", mMergeCounters.mOutputIteratorBufferUpdates); CLOG_INFO(Bucket, "OutputIteratorActualWrites: {}", @@ -915,6 +917,8 @@ class StopAndRestartBucketMergesTest CHECK(mMergeCounters.mOutputIteratorTombstoneElisions == other.mMergeCounters.mOutputIteratorTombstoneElisions); + CHECK(mMergeCounters.mOutputIteratorLiveToInitRewrites == + other.mMergeCounters.mOutputIteratorLiveToInitRewrites); CHECK(mMergeCounters.mOutputIteratorBufferUpdates == other.mMergeCounters.mOutputIteratorBufferUpdates); CHECK(mMergeCounters.mOutputIteratorActualWrites ==