refactor(daa): externalize block dependencies

hathor/daa.py

@@ -27,10 +27,12 @@
 
 from hathor.conf.settings import HathorSettings
 from hathor.profiler import get_cpu_profiler
+from hathor.types import VertexId
 from hathor.util import iwindows, not_none
 
 if TYPE_CHECKING:
     from hathor.transaction import Block, Transaction
+    from hathor.transaction.storage.simple_memory_storage import SimpleMemoryStorage
     from hathor.transaction.storage.vertex_storage_protocol import VertexStorageProtocol
 
 logger = get_logger()
@@ -58,15 +60,33 @@ def __init__(self, *, settings: HathorSettings, test_mode: TestMode = TestMode.D
         DifficultyAdjustmentAlgorithm.singleton = self
 
     @cpu.profiler(key=lambda _, block: 'calculate_block_difficulty!{}'.format(block.hash.hex()))
-    def calculate_block_difficulty(self, block: 'Block') -> float:
-        """ Calculate block weight according to the ascendents of `block`, using calculate_next_weight."""
+    def calculate_block_difficulty(self, block: 'Block', memory_storage: 'SimpleMemoryStorage') -> float:
+        """ Calculate block weight according to the ascendants of `block`, using calculate_next_weight."""
         if self.TEST_MODE & TestMode.TEST_BLOCK_WEIGHT:
             return 1.0
 
         if block.is_genesis:
             return self.MIN_BLOCK_WEIGHT
 
-        return self.calculate_next_weight(block.get_block_parent(), block.timestamp, not_none(block.storage))
+        parent_block = memory_storage.get_parent_block(block)
+
+        return self.calculate_next_weight(parent_block, block.timestamp, memory_storage)
+
+    def _calculate_N(self, parent_block: 'Block') -> int:
+        """Calculate the N value for the `calculate_next_weight` algorithm."""
+        return min(2 * self._settings.BLOCK_DIFFICULTY_N_BLOCKS, parent_block.get_height() - 1)
+
+    def get_block_dependencies(self, block: 'Block') -> list[VertexId]:
+        """Return the ids of the required blocks to call `calculate_block_difficulty` for the provided block."""
+        parent_block = block.get_block_parent()
+        N = self._calculate_N(parent_block)
+        ids: list[VertexId] = [not_none(parent_block.hash)]
+
+        while len(ids) <= N + 1:
+            parent_block = parent_block.get_block_parent()
+            ids.append(not_none(parent_block.hash))
+
+        return ids
 
     def calculate_next_weight(self, parent_block: 'Block', timestamp: int, storage: 'VertexStorageProtocol') -> float:
         """ Calculate the next block weight, aka DAA/difficulty adjustment algorithm.
@@ -81,7 +101,7 @@ def calculate_next_weight(self, parent_block: 'Block', timestamp: int, storage:
         from hathor.transaction import sum_weights
 
         root = parent_block
-        N = min(2 * self._settings.BLOCK_DIFFICULTY_N_BLOCKS, parent_block.get_height() - 1)
+        N = self._calculate_N(parent_block)
         K = N // 2
         T = self.AVG_TIME_BETWEEN_BLOCKS
         S = 5

hathor/transaction/base_transaction.py

@@ -842,7 +842,7 @@ def serialize_output(tx: BaseTransaction, tx_out: TxOutput) -> dict[str, Any]:
 
         return ret
 
-    def clone(self, *, include_metadata: bool = True) -> 'BaseTransaction':
+    def clone(self, *, include_metadata: bool = True, include_storage: bool = True) -> 'BaseTransaction':
         """Return exact copy without sharing memory, including metadata if loaded.
 
         :return: Transaction or Block copy
@@ -851,7 +851,8 @@ def clone(self, *, include_metadata: bool = True) -> 'BaseTransaction':
         if hasattr(self, '_metadata') and include_metadata:
             assert self._metadata is not None  # FIXME: is this actually true or do we have to check if not None
             new_tx._metadata = self._metadata.clone()
-        new_tx.storage = self.storage
+        if include_storage:
+            new_tx.storage = self.storage
         return new_tx
 
     @abstractmethod

hathor/transaction/storage/simple_memory_storage.py

@@ -0,0 +1,99 @@
+#  Copyright 2023 Hathor Labs
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from hathor.transaction import Block, Transaction
+from hathor.transaction.base_transaction import BaseTransaction
+from hathor.transaction.storage import TransactionStorage
+from hathor.transaction.storage.exceptions import TransactionDoesNotExist
+from hathor.types import VertexId
+
+
+class SimpleMemoryStorage:
+    """
+    Instances of this class simply facilitate storing some data in memory, specifically for pre-fetched verification
+    dependencies.
+    """
+    __slots__ = ('_blocks', '_transactions',)
+
+    def __init__(self) -> None:
+        self._blocks: dict[VertexId, BaseTransaction] = {}
+        self._transactions: dict[VertexId, BaseTransaction] = {}
+
+    @property
+    def _vertices(self) -> dict[VertexId, BaseTransaction]:
+        """Blocks and Transactions together."""
+        return {**self._blocks, **self._transactions}
+
+    def get_block(self, block_id: VertexId) -> Block:
+        """Return a block from the storage, throw if it's not found."""
+        block = self._get_vertex(self._blocks, block_id)
+        assert isinstance(block, Block)
+        return block
+
+    def get_transaction(self, tx_id: VertexId) -> Transaction:
+        """Return a transaction from the storage, throw if it's not found."""
+        tx = self._get_vertex(self._transactions, tx_id)
+        assert isinstance(tx, Transaction)
+        return tx
+
+    @staticmethod
+    def _get_vertex(storage: dict[VertexId, BaseTransaction], vertex_id: VertexId) -> BaseTransaction:
+        """Return a vertex from a storage, throw if it's not found."""
+        if vertex := storage.get(vertex_id):
+            return vertex
+
+        raise TransactionDoesNotExist(f'Vertex "{vertex_id.hex()}" does not exist in this SimpleMemoryStorage.')
+
+    def get_parent_block(self, block: Block) -> Block:
+        """Get the parent block of a block."""
+        parent_hash = block.get_block_parent_hash()
+
+        return self.get_block(parent_hash)
+
+    def add_vertices_from_storage(self, storage: TransactionStorage, ids: list[VertexId]) -> None:
+        """
+        Add multiple vertices to this storage. It automatically fetches data from the provided TransactionStorage
+        and a list of ids.
+        """
+        for vertex_id in ids:
+            self.add_vertex_from_storage(storage, vertex_id)
+
+    def add_vertex_from_storage(self, storage: TransactionStorage, vertex_id: VertexId) -> None:
+        """
+        Add a vertex to this storage. It automatically fetches data from the provided TransactionStorage and a list
+        of ids.
+        """
+        if vertex_id in self._vertices:
+            return
+
+        vertex = storage.get_transaction(vertex_id)
+        clone = vertex.clone(include_metadata=True, include_storage=False)
+
+        if isinstance(vertex, Block):
+            self._blocks[vertex_id] = clone
+            return
+
+        if isinstance(vertex, Transaction):
+            self._transactions[vertex_id] = clone
+            return
+
+        raise NotImplementedError
+
+    def get_vertex(self, vertex_id: VertexId) -> BaseTransaction:
+        # TODO: Currently unused, will be implemented in a next PR.
+        raise NotImplementedError
+
+    def get_best_block_tips(self) -> list[VertexId]:
+        # TODO: Currently unused, will be implemented in a next PR.
+        raise NotImplementedError

hathor/verification/block_verifier.py

@@ -25,6 +25,8 @@
     TransactionDataError,
     WeightError,
 )
+from hathor.transaction.storage.simple_memory_storage import SimpleMemoryStorage
+from hathor.util import not_none
 
 
 class BlockVerifier:
@@ -51,7 +53,11 @@ def verify_height(self, block: Block) -> None:
 
     def verify_weight(self, block: Block) -> None:
         """Validate minimum block difficulty."""
-        min_block_weight = self._daa.calculate_block_difficulty(block)
+        memory_storage = SimpleMemoryStorage()
+        dependencies = self._daa.get_block_dependencies(block)
+        memory_storage.add_vertices_from_storage(not_none(block.storage), dependencies)
+
+        min_block_weight = self._daa.calculate_block_difficulty(block, memory_storage)
         if block.weight < min_block_weight - self._settings.WEIGHT_TOL:
             raise WeightError(f'Invalid new block {block.hash_hex}: weight ({block.weight}) is '
                               f'smaller than the minimum weight ({min_block_weight})')

tests/tx/test_genesis.py

@@ -74,9 +74,9 @@ def test_genesis_weight(self):
         # Validate the block and tx weight
         # in test mode weight is always 1
         self._daa.TEST_MODE = TestMode.TEST_ALL_WEIGHT
-        self.assertEqual(self._daa.calculate_block_difficulty(genesis_block), 1)
+        self.assertEqual(self._daa.calculate_block_difficulty(genesis_block, Mock()), 1)
         self.assertEqual(self._daa.minimum_tx_weight(genesis_tx), 1)
 
         self._daa.TEST_MODE = TestMode.DISABLED
-        self.assertEqual(self._daa.calculate_block_difficulty(genesis_block), genesis_block.weight)
+        self.assertEqual(self._daa.calculate_block_difficulty(genesis_block, Mock()), genesis_block.weight)
         self.assertEqual(self._daa.minimum_tx_weight(genesis_tx), genesis_tx.weight)