From d7fa415cad7edfb451fea1bf818c263813bb6ba7 Mon Sep 17 00:00:00 2001 From: Andreas Schildbach Date: Wed, 26 Jun 2024 22:28:27 +0200 Subject: [PATCH] StoredBlock: add serializeCompactV2(), deserializeCompactV2() The old format will soon run out of bytes for the chain work value. Also deprecates the old methods and adds tests for the V2 format. --- .../java/org/bitcoinj/core/StoredBlock.java | 70 ++++++++++++++++--- .../org/bitcoinj/core/StoredBlockTest.java | 63 +++++++++++++++-- 2 files changed, 115 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/bitcoinj/core/StoredBlock.java b/core/src/main/java/org/bitcoinj/core/StoredBlock.java index f7288b355ae..f9b412cd121 100644 --- a/core/src/main/java/org/bitcoinj/core/StoredBlock.java +++ b/core/src/main/java/org/bitcoinj/core/StoredBlock.java @@ -40,10 +40,16 @@ public class StoredBlock { // A BigInteger representing the total amount of work done so far on this chain. As of June 22, 2024, it takes 12 - // unsigned bytes to store this value, so we need to create an updated storage format soon. - private static final int CHAIN_WORK_BYTES = 12; - private static final byte[] EMPTY_BYTES = new byte[CHAIN_WORK_BYTES]; - public static final int COMPACT_SERIALIZED_SIZE = Block.HEADER_SIZE + CHAIN_WORK_BYTES + 4; // for height + // unsigned bytes to store this value, so developers should use the V2 format. + private static final int CHAIN_WORK_BYTES_V1 = 12; + // A BigInteger representing the total amount of work done so far on this chain. + private static final int CHAIN_WORK_BYTES_V2 = 32; + // Used for padding. + private static final byte[] EMPTY_BYTES = new byte[CHAIN_WORK_BYTES_V2]; // fit larger format + /** Number of bytes serialized by {@link #serializeCompact(ByteBuffer)} */ + public static final int COMPACT_SERIALIZED_SIZE = Block.HEADER_SIZE + CHAIN_WORK_BYTES_V1 + /*height*/ 4; + /** Number of bytes serialized by {@link #serializeCompactV2(ByteBuffer)} */ + public static final int COMPACT_SERIALIZED_SIZE_V2 = Block.HEADER_SIZE + CHAIN_WORK_BYTES_V2 + /*height*/ 4; private final Block header; private final BigInteger chainWork; @@ -124,12 +130,36 @@ public StoredBlock getPrev(BlockStore store) throws BlockStoreException { return store.get(getHeader().getPrevBlockHash()); } - /** Serializes the stored block to a custom packed format. Used by {@link CheckpointManager}. */ + /** + * Serializes the stored block to a custom packed format. As of June 22, 2024, it takes 12 + * unsigned bytes to store the chain work value, so developers should use the V2 format. + * + * @param buffer buffer to write to + * @deprecated use @{@link #serializeCompactV2(ByteBuffer)} + */ + @Deprecated public void serializeCompact(ByteBuffer buffer) { - byte[] chainWorkBytes = ByteUtils.bigIntegerToBytes(getChainWork(), CHAIN_WORK_BYTES); - if (chainWorkBytes.length < CHAIN_WORK_BYTES) { + byte[] chainWorkBytes = ByteUtils.bigIntegerToBytes(getChainWork(), CHAIN_WORK_BYTES_V1); + if (chainWorkBytes.length < CHAIN_WORK_BYTES_V1) { + // Pad to the right size. + buffer.put(EMPTY_BYTES, 0, CHAIN_WORK_BYTES_V1 - chainWorkBytes.length); + } + buffer.put(chainWorkBytes); + buffer.putInt(getHeight()); + byte[] bytes = getHeader().serialize(); + buffer.put(bytes, 0, Block.HEADER_SIZE); // Trim the trailing 00 byte (zero transactions). + } + + /** + * Serializes the stored block to a custom packed format. + * + * @param buffer buffer to write to + */ + public void serializeCompactV2(ByteBuffer buffer) { + byte[] chainWorkBytes = ByteUtils.bigIntegerToBytes(getChainWork(), CHAIN_WORK_BYTES_V2); + if (chainWorkBytes.length < CHAIN_WORK_BYTES_V2) { // Pad to the right size. - buffer.put(EMPTY_BYTES, 0, CHAIN_WORK_BYTES - chainWorkBytes.length); + buffer.put(EMPTY_BYTES, 0, CHAIN_WORK_BYTES_V2 - chainWorkBytes.length); } buffer.put(chainWorkBytes); buffer.putInt(getHeight()); @@ -138,14 +168,32 @@ public void serializeCompact(ByteBuffer buffer) { } /** - * Deserializes the stored block from a custom packed format. Used by {@link CheckpointManager} and - * {@link SPVBlockStore}. + * Deserializes the stored block from a custom packed format. As of June 22, 2024, it takes 12 + * unsigned bytes to store the chain work value, so developers should use the V2 format. * * @param buffer data to deserialize * @return deserialized stored block + * @deprecated use @{@link #deserializeCompactV2(ByteBuffer)} */ + @Deprecated public static StoredBlock deserializeCompact(ByteBuffer buffer) throws ProtocolException { - byte[] chainWorkBytes = new byte[StoredBlock.CHAIN_WORK_BYTES]; + byte[] chainWorkBytes = new byte[StoredBlock.CHAIN_WORK_BYTES_V1]; + buffer.get(chainWorkBytes); + BigInteger chainWork = ByteUtils.bytesToBigInteger(chainWorkBytes); + int height = buffer.getInt(); // +4 bytes + byte[] header = new byte[Block.HEADER_SIZE + 1]; // Extra byte for the 00 transactions length. + buffer.get(header, 0, Block.HEADER_SIZE); + return new StoredBlock(Block.read(ByteBuffer.wrap(header)), chainWork, height); + } + + /** + * Deserializes the stored block from a custom packed format. + * + * @param buffer data to deserialize + * @return deserialized stored block + */ + public static StoredBlock deserializeCompactV2(ByteBuffer buffer) throws ProtocolException { + byte[] chainWorkBytes = new byte[StoredBlock.CHAIN_WORK_BYTES_V2]; buffer.get(chainWorkBytes); BigInteger chainWork = ByteUtils.bytesToBigInteger(chainWorkBytes); int height = buffer.getInt(); // +4 bytes diff --git a/core/src/test/java/org/bitcoinj/core/StoredBlockTest.java b/core/src/test/java/org/bitcoinj/core/StoredBlockTest.java index d516a43d6b5..c26eaf633a5 100644 --- a/core/src/test/java/org/bitcoinj/core/StoredBlockTest.java +++ b/core/src/test/java/org/bitcoinj/core/StoredBlockTest.java @@ -33,9 +33,15 @@ public class StoredBlockTest { // Max chain work to fit in 12 bytes - private static final BigInteger MAX_WORK = new BigInteger(/* 12 bytes */ "ffffffffffffffffffffffff", 16); + private static final BigInteger MAX_WORK_V1 = new BigInteger(/* 12 bytes */ "ffffffffffffffffffffffff", 16); // Chain work too large to fit in 12 bytes - private static final BigInteger TOO_LARGE_WORK = new BigInteger(/* 13 bytes */ "ffffffffffffffffffffffffff", 16); + private static final BigInteger TOO_LARGE_WORK_V1 = new BigInteger(/* 13 bytes */ "ffffffffffffffffffffffffff", 16); + // Max chain work to fit in 32 bytes + private static final BigInteger MAX_WORK_V2 = new BigInteger(/* 32 bytes */ + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16); + // Chain work too large to fit in 32 bytes + private static final BigInteger TOO_LARGE_WORK_V2 = new BigInteger(/* 33 bytes */ + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16); // Just an arbitrary block private static final Block BLOCK = Block.createGenesis(Instant.now(), Block.EASIEST_DIFFICULTY_TARGET); @@ -44,7 +50,7 @@ private Object[] vectors_serializeCompact_pass() { new Object[] { BigInteger.ZERO }, // no work new Object[] { BigInteger.ONE }, // small work new Object[] { BigInteger.valueOf(Long.MAX_VALUE) }, // a larg-ish work - new Object[] { MAX_WORK }, + new Object[] { MAX_WORK_V1 }, }; } @@ -56,7 +62,9 @@ public void roundtripSerializeCompact_pass(BigInteger chainWork) { private Object[] vectors_serializeCompact_fail() { return new Object[] { - new Object[] { TOO_LARGE_WORK }, + new Object[] { TOO_LARGE_WORK_V1 }, + new Object[] { MAX_WORK_V2 }, + new Object[] { TOO_LARGE_WORK_V2 }, new Object[] { BigInteger.valueOf(-1) }, // negative }; } @@ -76,14 +84,55 @@ private void roundtripSerializeCompact(BigInteger chainWork) { assertEquals(StoredBlock.deserializeCompact(buf), block); } + private Object[] vectors_serializeCompactV2_pass() { + return new Object[] { + new Object[] { BigInteger.ZERO }, // no work + new Object[] { BigInteger.ONE }, // small work + new Object[] { BigInteger.valueOf(Long.MAX_VALUE) }, // a larg-ish work + new Object[] { MAX_WORK_V1 }, + new Object[] { TOO_LARGE_WORK_V1 }, + new Object[] { MAX_WORK_V2 }, + }; + } + + @Test + @Parameters(method = "vectors_serializeCompactV2_pass") + public void roundtripSerializeCompactV2_pass(BigInteger chainWork) { + roundtripSerializeCompactV2(chainWork); + } + + private Object[] vectors_serializeCompactV2_fail() { + return new Object[] { + new Object[] { TOO_LARGE_WORK_V2 }, + new Object[] { BigInteger.valueOf(-1) }, // negative + }; + } + + @Test(expected = RuntimeException.class) + @Parameters(method = "vectors_serializeCompactV2_fail") + public void roundtripSerializeCompactV2_fail(BigInteger chainWork) { + roundtripSerializeCompactV2(chainWork); + } + + private void roundtripSerializeCompactV2(BigInteger chainWork) { + StoredBlock block = new StoredBlock(BLOCK, chainWork, 0); + ByteBuffer buf = ByteBuffer.allocate(StoredBlock.COMPACT_SERIALIZED_SIZE_V2); + block.serializeCompactV2(buf); + assertEquals(StoredBlock.COMPACT_SERIALIZED_SIZE_V2, buf.position()); + ((Buffer) buf).rewind(); + assertEquals(StoredBlock.deserializeCompactV2(buf), block); + } + @Test public void moreWorkThan() { StoredBlock noWorkBlock = new StoredBlock(BLOCK, BigInteger.ZERO, 0); StoredBlock smallWorkBlock = new StoredBlock(BLOCK, BigInteger.ONE, 0); - StoredBlock maxWorkBlock = new StoredBlock(BLOCK, MAX_WORK, 0); + StoredBlock maxWorkBlockV1 = new StoredBlock(BLOCK, MAX_WORK_V1, 0); + StoredBlock maxWorkBlockV2 = new StoredBlock(BLOCK, MAX_WORK_V2, 0); assertTrue(smallWorkBlock.moreWorkThan(noWorkBlock)); - assertTrue(maxWorkBlock.moreWorkThan(noWorkBlock)); - assertTrue(maxWorkBlock.moreWorkThan(smallWorkBlock)); + assertTrue(maxWorkBlockV1.moreWorkThan(noWorkBlock)); + assertTrue(maxWorkBlockV1.moreWorkThan(smallWorkBlock)); + assertTrue(maxWorkBlockV2.moreWorkThan(maxWorkBlockV1)); } }