diff --git a/core/src/main/java/org/bitcoinj/store/SPVBlockStore.java b/core/src/main/java/org/bitcoinj/store/SPVBlockStore.java index 6dce5b84aba..ec39ff75bc5 100644 --- a/core/src/main/java/org/bitcoinj/store/SPVBlockStore.java +++ b/core/src/main/java/org/bitcoinj/store/SPVBlockStore.java @@ -60,8 +60,10 @@ public class SPVBlockStore implements BlockStore { public static final int DEFAULT_CAPACITY = 10000; @Deprecated public static final String HEADER_MAGIC = "SPVB"; - // Magic header. - private static final byte[] MAGIC_HEADER = HEADER_MAGIC.getBytes(StandardCharsets.US_ASCII); + // Magic header for the V1 format. + static final byte[] HEADER_MAGIC_V1 = HEADER_MAGIC.getBytes(StandardCharsets.US_ASCII); + // Magic header for the V2 format. + static final byte[] HEADER_MAGIC_V2 = "SPV2".getBytes(StandardCharsets.US_ASCII); protected volatile MappedByteBuffer buffer; protected final NetworkParameters params; @@ -119,8 +121,10 @@ public SPVBlockStore(NetworkParameters params, File file, int capacity, boolean Objects.requireNonNull(file); this.params = Objects.requireNonNull(params); checkArgument(capacity > 0); + try { boolean exists = file.exists(); + // Set up the backing file, empty if it doesn't exist. randomAccessFile = new RandomAccessFile(file, "rw"); FileChannel channel = randomAccessFile.getChannel(); @@ -131,44 +135,91 @@ public SPVBlockStore(NetworkParameters params, File file, int capacity, boolean throw new ChainFileLockedException("Store file is already locked by another process"); // Ensure expected file size, grow if desired. - fileLength = getFileSize(capacity); - if (!exists) { - log.info("Creating new SPV block chain file " + file); - randomAccessFile.setLength(fileLength); + this.fileLength = getFileSize(capacity); + byte[] currentHeader = new byte[4]; + if (exists) { + log.info("Using existing SPV block chain file: " + file); + // Map it into memory read/write. The kernel will take care of flushing writes to disk at the most + // efficient times, which may mean that until the map is deallocated the data on disk is randomly + // inconsistent. However the only process accessing it is us, via this mapping, so our own view will + // always be correct. Once we establish the mmap the underlying file and channel can go away. Note that + // the details of mmapping vary between platforms. + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, randomAccessFile.length()); + buffer.get(currentHeader); } else { + log.info("Creating new SPV block chain file: " + file); + randomAccessFile.setLength(fileLength); + // Map it into memory read/write. See above comment. + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, fileLength); + initNewStore(params.getGenesisBlock()); + } + + // Maybe migrate V1 to V2 format. + if (Arrays.equals(HEADER_MAGIC_V1, currentHeader)) { + long currentLength = randomAccessFile.length(); + long currentBlocksLength = currentLength - FILE_PROLOGUE_BYTES; + if (currentBlocksLength % RECORD_SIZE_V1 != 0) + throw new BlockStoreException( + "File size on disk indicates this is not a V1 block store: " + currentLength); + int currentCapacity = (int) (currentBlocksLength / RECORD_SIZE_V1); + log.info("Migrating SPV block chain file containing " + currentCapacity + " blocks from V1 to V2 " + + "format: " + file); + + randomAccessFile.setLength(fileLength); + // Map it into memory again because of the length change. + buffer.force(); + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, fileLength); + + // migrate magic header + ((Buffer) buffer).rewind(); + buffer.put(HEADER_MAGIC_V2); + + // migrate headers + final byte[] zeroPadding = new byte[20]; // 32 (V2 work) - 12 (V1 work) + for (int i = currentCapacity - 1; i >= 0; i--) { + byte[] record = new byte[RECORD_SIZE_V1]; + buffer.position(FILE_PROLOGUE_BYTES + i * RECORD_SIZE_V1); + buffer.get(record); + buffer.position(FILE_PROLOGUE_BYTES + i * RECORD_SIZE_V2); + buffer.put(record, 0, 32); // hash + buffer.put(zeroPadding); + buffer.put(record, 32, RECORD_SIZE_V1 - 32); // work, height, block header + } + + // migrate cursor + int cursorRecord = (getRingCursor() - FILE_PROLOGUE_BYTES) / RECORD_SIZE_V1; + setRingCursor(FILE_PROLOGUE_BYTES + cursorRecord * RECORD_SIZE_V2); + } + + // Maybe grow. + if (exists) { final long currentLength = randomAccessFile.length(); if (currentLength != fileLength) { - if ((currentLength - FILE_PROLOGUE_BYTES) % RECORD_SIZE != 0) + if ((currentLength - FILE_PROLOGUE_BYTES) % RECORD_SIZE_V2 != 0) { throw new BlockStoreException( - "File size on disk indicates this is not a block store: " + currentLength); - else if (!grow) + "File size on disk indicates this is not a V2 block store: " + currentLength); + } else if (!grow) { throw new BlockStoreException("File size on disk does not match expected size: " + currentLength + " vs " + fileLength); - else if (fileLength < randomAccessFile.length()) + } else if (fileLength < randomAccessFile.length()) { throw new BlockStoreException( "Shrinking is unsupported: " + currentLength + " vs " + fileLength); - else + } else { randomAccessFile.setLength(fileLength); + // Map it into memory again because of the length change. + buffer.force(); + buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, fileLength); + } } } - // Map it into memory read/write. The kernel will take care of flushing writes to disk at the most - // efficient times, which may mean that until the map is deallocated the data on disk is randomly - // inconsistent. However the only process accessing it is us, via this mapping, so our own view will - // always be correct. Once we establish the mmap the underlying file and channel can go away. Note that - // the details of mmapping vary between platforms. - buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, fileLength); - - // Check or initialize the header bytes to ensure we don't try to open some random file. - if (exists) { - byte[] currentHeader = new byte[4]; - ((Buffer) buffer).rewind(); - buffer.get(currentHeader); - if (!Arrays.equals(currentHeader, MAGIC_HEADER)) - throw new BlockStoreException("Magic header expected, got: " + ByteUtils.formatHex(currentHeader)); - } else { - initNewStore(params.getGenesisBlock()); - } + // Check the header bytes to ensure we don't try to open some random file. + byte[] header = new byte[4]; + ((Buffer) buffer).rewind(); + buffer.get(currentHeader); + if (!Arrays.equals(currentHeader, HEADER_MAGIC_V2)) + throw new BlockStoreException("Magic header V2 expected: " + new String(currentHeader, + StandardCharsets.US_ASCII)); } catch (Exception e) { try { if (randomAccessFile != null) randomAccessFile.close(); @@ -181,7 +232,7 @@ else if (fileLength < randomAccessFile.length()) private void initNewStore(Block genesisBlock) throws Exception { ((Buffer) buffer).rewind(); - buffer.put(MAGIC_HEADER); + buffer.put(HEADER_MAGIC_V2); // Insert the genesis block. lock.lock(); try { @@ -196,7 +247,7 @@ private void initNewStore(Block genesisBlock) throws Exception { /** Returns the size in bytes of the file that is used to store the chain with the current parameters. */ public static int getFileSize(int capacity) { - return RECORD_SIZE * capacity + FILE_PROLOGUE_BYTES /* extra kilobyte for stuff */; + return RECORD_SIZE_V2 * capacity + FILE_PROLOGUE_BYTES /* extra kilobyte for stuff */; } @Override @@ -215,7 +266,7 @@ public void put(StoredBlock block) throws BlockStoreException { Sha256Hash hash = block.getHeader().getHash(); notFoundCache.remove(hash); buffer.put(hash.getBytes()); - block.serializeCompact(buffer); + block.serializeCompactV2(buffer); setRingCursor(buffer.position()); blockCache.put(hash, block); } finally { lock.unlock(); } @@ -242,17 +293,17 @@ public StoredBlock get(Sha256Hash hash) throws BlockStoreException { final byte[] targetHashBytes = hash.getBytes(); byte[] scratch = new byte[32]; do { - cursor -= RECORD_SIZE; + cursor -= RECORD_SIZE_V2; if (cursor < FILE_PROLOGUE_BYTES) { // We hit the start, so wrap around. - cursor = fileLength - RECORD_SIZE; + cursor = fileLength - RECORD_SIZE_V2; } // Cursor is now at the start of the next record to check, so read the hash and compare it. ((Buffer) buffer).position(cursor); buffer.get(scratch); if (Arrays.equals(scratch, targetHashBytes)) { // Found the target. - StoredBlock storedBlock = StoredBlock.deserializeCompact(buffer); + StoredBlock storedBlock = StoredBlock.deserializeCompactV2(buffer); blockCache.put(hash, storedBlock); return storedBlock; } @@ -315,10 +366,10 @@ public void close() throws BlockStoreException { } } - protected static final int RECORD_SIZE = 32 /* hash */ + StoredBlock.COMPACT_SERIALIZED_SIZE; + static final int RECORD_SIZE_V1 = 32 /* hash */ + StoredBlock.COMPACT_SERIALIZED_SIZE; - // File format: - // 4 header bytes = "SPVB" + // V1 file format: + // 4 magic header bytes = "SPVB" // 4 cursor bytes, which indicate the offset from the first kb where the next block header should be written. // 32 bytes for the hash of the chain head // @@ -327,10 +378,24 @@ public void close() throws BlockStoreException { // 12 bytes of chain work // 4 bytes of height // 80 bytes of block header data + + static final int RECORD_SIZE_V2 = 32 /* hash */ + StoredBlock.COMPACT_SERIALIZED_SIZE_V2; + + // V2 file format: + // 4 magic header bytes = "SPV2" + // 4 cursor bytes, which indicate the offset from the first kb where the next block header should be written. + // 32 bytes for the hash of the chain head + // + // For each header (148 bytes) + // 32 bytes hash of the header + // 32 bytes of chain work + // 4 bytes of height + // 80 bytes of block header data + protected static final int FILE_PROLOGUE_BYTES = 1024; /** Returns the offset from the file start where the latest block should be written (end of prev block). */ - private int getRingCursor() { + int getRingCursor() { int c = buffer.getInt(4); checkState(c >= FILE_PROLOGUE_BYTES, () -> "integer overflow"); diff --git a/core/src/test/java/org/bitcoinj/store/SPVBlockStoreTest.java b/core/src/test/java/org/bitcoinj/store/SPVBlockStoreTest.java index c134b66d47f..9d49efb11c7 100644 --- a/core/src/test/java/org/bitcoinj/store/SPVBlockStoreTest.java +++ b/core/src/test/java/org/bitcoinj/store/SPVBlockStoreTest.java @@ -36,7 +36,11 @@ import org.junit.Test; import java.io.File; +import java.io.RandomAccessFile; import java.math.BigInteger; +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.time.Duration; import java.time.Instant; import java.util.Collections; @@ -190,4 +194,34 @@ public void oneStoreDelete() throws Exception { assertTrue(deleted); } } + + @Test + public void migrateV1toV2() throws Exception { + // create V1 format + RandomAccessFile raf = new RandomAccessFile(blockStoreFile, "rw"); + FileChannel channel = raf.getChannel(); + ByteBuffer buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, + SPVBlockStore.FILE_PROLOGUE_BYTES + SPVBlockStore.RECORD_SIZE_V1 * 3); + buffer.put(SPVBlockStore.HEADER_MAGIC_V1); // header magic + Block genesisBlock = TESTNET.getGenesisBlock(); + StoredBlock storedGenesisBlock = new StoredBlock(genesisBlock.cloneAsHeader(), genesisBlock.getWork(), 0); + Sha256Hash genesisHash = storedGenesisBlock.getHeader().getHash(); + buffer.position(SPVBlockStore.FILE_PROLOGUE_BYTES); + buffer.put(genesisHash.getBytes()); + storedGenesisBlock.serializeCompact(buffer); + buffer.putInt(4, buffer.position()); // ring cursor + ((Buffer) buffer).position(8); + buffer.put(genesisHash.getBytes()); // chain head + raf.close(); + + // migrate to V2 format + SPVBlockStore store = new SPVBlockStore(TESTNET, blockStoreFile); + + // check block is the same + assertEquals(genesisHash, store.getChainHead().getHeader().getHash()); + // check ring cursor + assertEquals(SPVBlockStore.FILE_PROLOGUE_BYTES + SPVBlockStore.RECORD_SIZE_V2 * 1, + store.getRingCursor()); + store.close(); + } }