From 50521b5ce7bac4d76c9f76e47d384ada8d896f0c Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Mon, 3 May 2021 14:11:00 +0200 Subject: [PATCH 1/3] bump to 0.1.2dev (#16) --- README.rst | 2 +- ScanWatch/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3fd0956..7d393d5 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ============================== -Welcome to ScanWatch 0.1.1 +Welcome to ScanWatch 0.1.2dev ============================== Note diff --git a/ScanWatch/__init__.py b/ScanWatch/__init__.py index 4a43b94..2b31360 100644 --- a/ScanWatch/__init__.py +++ b/ScanWatch/__init__.py @@ -1,4 +1,4 @@ __author__ = 'EtWnn' -__version__ = '0.1.1' +__version__ = '0.1.2dev' from ScanWatch.ScanManager import ScanManager From fe8b5ec26f96dbbf8b7955afed3da0799f7086ff Mon Sep 17 00:00:00 2001 From: EtWn <34377743+EtWnn@users.noreply.github.com> Date: Sat, 17 Jul 2021 12:14:53 +0200 Subject: [PATCH 2/3] Doc fix (#19) * fix readme * change wording --- README.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 7d393d5..e963978 100644 --- a/README.rst +++ b/README.rst @@ -5,11 +5,9 @@ Welcome to ScanWatch 0.1.2dev Note ---- -This library is under development by EtWnn, feel free to drop your suggestions or remarks in +This library is developed and maintained by EtWnn, feel free to drop your suggestions or remarks in the discussion tab of the git repo. You are also welcome to contribute by submitting PRs. -This library is made to retrieve price or candle history of crypto assets using multiple sources. - **Source Code:** https://github.com/EtWnn/ScanWatch **Documentation:** @@ -28,7 +26,7 @@ Go on `etherscan `__ for the Ethereum chain and o `bscscan `__ for the BSC chain. (If you want to use both chains, you will need an API token for each). -``ScanWatch`` is not yet available on `PYPI `_, install with ``pip``: +``ScanWatch`` is available on `PYPI `_, install with ``pip``: .. code:: bash From cc847eb5b25e34d8b9d77acfc2a0ee8dadbd7053 Mon Sep 17 00:00:00 2001 From: EtWnn Date: Mon, 27 Dec 2021 12:58:23 +0100 Subject: [PATCH 3/3] Feature testnet support (#21) * fix 403 error for testnet API * add test nets options for the client * add net differentiation in the table name * add net differentiation in the database model * add net name support for the manager * docstring fix * add main / test nets description * bump to version 0.1.2 --- README.rst | 16 ++++++++++++- ScanWatch/Client.py | 40 +++++++++++++++++++++---------- ScanWatch/ScanManager.py | 13 ++++++---- ScanWatch/__init__.py | 2 +- ScanWatch/storage/ScanDataBase.py | 18 +++++++++----- ScanWatch/storage/tables.py | 9 +++++-- 6 files changed, 70 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index e963978..bb839a5 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ============================== -Welcome to ScanWatch 0.1.2dev +Welcome to ScanWatch 0.1.2 ============================== Note @@ -88,6 +88,20 @@ Otherwise you can just fetch the transactions that have been previously saved, a manager.get_transactions(TRANSACTION.INTERNAL) # internal transactions +Main / test nets +---------------- + +If you want to switch from main to test nets, you can specify the net name at the manager creation: + +.. code:: python + + manager = ScanManager(address, , api_token, ) + +Supported nets are: + - For Ethereum: "main", "goerli", "kovan", "rinkeby", "ropsten" + - For BSC: "main", "test" + + Donation -------- diff --git a/ScanWatch/Client.py b/ScanWatch/Client.py index 6bd76a6..3991b89 100644 --- a/ScanWatch/Client.py +++ b/ScanWatch/Client.py @@ -12,10 +12,21 @@ class Client: https://etherscan.io/apis https://bscscan.com/apis """ - BASE_ETH_URL = "https://api.etherscan.io/api" - BASE_BSC_URL = "https://api.bscscan.com/api" + BASE_URLS = { + NETWORK.BSC: { + "main": "https://api.bscscan.com/api", + "test": "https://api-testnet.bscscan.com/api" + }, + NETWORK.ETHER: { + "main": "https://api.etherscan.io/api", + "goerli": "https://api-goerli.etherscan.io/api", + "kovan": "https://api-kovan.etherscan.io/api", + "rinkeby": "https://api-rinkeby.etherscan.io/api", + "ropsten": "https://api-ropsten.etherscan.io/api" + } + } - def __init__(self, api_token: str, nt_type: NETWORK): + def __init__(self, api_token: str, nt_type: NETWORK, net: str = "main"): """ @@ -23,15 +34,19 @@ def __init__(self, api_token: str, nt_type: NETWORK): :type api_token: str :param nt_type: type of the network :type nt_type: NETWORK + :param net: name of the network, used to differentiate main and test nets + :type net: str, default 'main' """ self.api_token = api_token self.nt_type = nt_type + self.net = net + self.get_url_request() # test if network parameters are valid def get_mined_blocks(self, address: str, start_block: Optional[int] = None, end_block: Optional[int] = None): """ - fetch mined blocks by an eth address + fetch mined blocks by an address - :param address: ETH address + :param address: network address :type address: str :param start_block: fetch mined blocks starting with this block :type start_block: Optional[int] @@ -209,7 +224,7 @@ def get_balance(self, address: str) -> float: def get_url_request(self, **kwargs) -> str: """ - Construct the url to make a request to the etherscan.io API + Construct the url to make a request to the etherscan.io / bscscan.com API :param kwargs: keywords args for the endpoint :type kwargs: Any @@ -218,12 +233,10 @@ def get_url_request(self, **kwargs) -> str: """ _keywords = {**kwargs, "apikey": self.api_token} string_kws = "&".join((f"{key}={value}" for key, value in _keywords.items())) - if self.nt_type == NETWORK.ETHER: - base_url = Client.BASE_ETH_URL - elif self.nt_type == NETWORK.BSC: - base_url = Client.BASE_BSC_URL - else: - raise ValueError(f"unknown network type: {self.nt_type}") + try: + base_url = self.BASE_URLS[self.nt_type][self.net] + except KeyError as err: + raise ValueError(f"unknown network with type {self.nt_type} and name {self.net}") from err return f"{base_url}?{string_kws}" @staticmethod @@ -236,7 +249,8 @@ def get_result(url: str): :return: API result :rtype: depend of the endpoint """ - response = requests.get(url) + response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}) + response.raise_for_status() r_json = response.json() if int(r_json['status']) > 0 or r_json['message'] == 'No transactions found': return r_json['result'] diff --git a/ScanWatch/ScanManager.py b/ScanWatch/ScanManager.py index f0a6c3d..3cc3600 100644 --- a/ScanWatch/ScanManager.py +++ b/ScanWatch/ScanManager.py @@ -10,7 +10,7 @@ class ScanManager: This class is the interface between the user, the API and the Database """ - def __init__(self, address: str, nt_type: NETWORK, api_token: str): + def __init__(self, address: str, nt_type: NETWORK, api_token: str, net: str = "main"): """ Initiate the manager @@ -20,10 +20,13 @@ def __init__(self, address: str, nt_type: NETWORK, api_token: str): :type nt_type: NETWORK :param api_token: token to communicate with the API :type api_token: str + :param net: name of the network, used to differentiate main and test nets + :type net: str, default 'main' """ self.address = address self.nt_type = nt_type - self.client = Client(api_token, self.nt_type) + self.net = net + self.client = Client(api_token, self.nt_type, self.net) self.db = ScanDataBase() def update_transactions(self, tr_type: TRANSACTION): @@ -35,7 +38,7 @@ def update_transactions(self, tr_type: TRANSACTION): :return: None :rtype: None """ - last_block = self.db.get_last_block_number(self.address, self.nt_type, tr_type) + last_block = self.db.get_last_block_number(self.address, self.nt_type, self.net, tr_type) if tr_type == TRANSACTION.NORMAL: new_transactions = self.client.get_normal_transactions(self.address, start_block=last_block + 1) elif tr_type == TRANSACTION.INTERNAL: @@ -46,7 +49,7 @@ def update_transactions(self, tr_type: TRANSACTION): new_transactions = self.client.get_erc721_transactions(self.address, start_block=last_block + 1) else: raise ValueError(f"unknown transaction type: {tr_type}") - self.db.add_transactions(self.address, self.nt_type, tr_type, new_transactions) + self.db.add_transactions(self.address, self.nt_type, self.net, tr_type, new_transactions) def update_all_transactions(self): """ @@ -74,4 +77,4 @@ def get_transactions(self, tr_type: TRANSACTION): :return: list of transactions :rtype: List[Dict] """ - return self.db.get_transactions(self.address, self.nt_type, tr_type) + return self.db.get_transactions(self.address, self.nt_type, self.net, tr_type) diff --git a/ScanWatch/__init__.py b/ScanWatch/__init__.py index 2b31360..b828470 100644 --- a/ScanWatch/__init__.py +++ b/ScanWatch/__init__.py @@ -1,4 +1,4 @@ __author__ = 'EtWnn' -__version__ = '0.1.2dev' +__version__ = '0.1.2' from ScanWatch.ScanManager import ScanManager diff --git a/ScanWatch/storage/ScanDataBase.py b/ScanWatch/storage/ScanDataBase.py index 2e91241..b3f22d5 100644 --- a/ScanWatch/storage/ScanDataBase.py +++ b/ScanWatch/storage/ScanDataBase.py @@ -19,7 +19,7 @@ def __init__(self, name: str = 'scan_db'): """ super().__init__(name) - def add_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION, transactions: List[Dict]): + def add_transactions(self, address: str, nt_type: NETWORK, net: str, tr_type: TRANSACTION, transactions: List[Dict]): """ Add a list of transactions to the database @@ -27,6 +27,8 @@ def add_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION, :type address: str :param nt_type: type of network :type nt_type: NETWORK + :param net: name of the network, used to differentiate main and test nets + :type net: str :param tr_type: type of the transaction to record :type tr_type: TRANSACTION :param transactions: list of the transaction to record @@ -34,13 +36,13 @@ def add_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION, :return: None :rtype: None """ - table = get_transaction_table(address, nt_type, tr_type) + table = get_transaction_table(address, nt_type, net, tr_type) for transaction in transactions: row = table.dict_to_tuple(transaction) self.add_row(table, row, auto_commit=False) self.commit() - def get_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION) -> List[Dict]: + def get_transactions(self, address: str, nt_type: NETWORK, net: str, tr_type: TRANSACTION) -> List[Dict]: """ Return the List of the transactions recorded in the database @@ -48,16 +50,18 @@ def get_transactions(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION) :type address: str :param nt_type: type of network :type nt_type: NETWORK + :param net: name of the network, used to differentiate main and test nets + :type net: str :param tr_type: type of the transaction to fetch :type tr_type: TRANSACTION :return: list of the transaction recorded :rtype: List[Dict] """ - table = get_transaction_table(address, nt_type, tr_type) + table = get_transaction_table(address, nt_type, net, tr_type) rows = self.get_all_rows(table) return [table.tuple_to_dict(row) for row in rows] - def get_last_block_number(self, address: str, nt_type: NETWORK, tr_type: TRANSACTION) -> int: + def get_last_block_number(self, address: str, nt_type: NETWORK, net: str, tr_type: TRANSACTION) -> int: """ Return the last block number seen in recorded transactions (per address, type of transaction and network) If None are found, return 0 @@ -66,12 +70,14 @@ def get_last_block_number(self, address: str, nt_type: NETWORK, tr_type: TRANSAC :type address: str :param nt_type: type of network :type nt_type: NETWORK + :param net: name of the network, used to differentiate main and test nets + :type net: str :param tr_type: type of the transaction to fetch :type tr_type: TRANSACTION :return: last block number :rtype: int """ - table = get_transaction_table(address, nt_type, tr_type) + table = get_transaction_table(address, nt_type, net, tr_type) selection = f"MAX({table.blockNumber})" result = self.get_conditions_rows(table, selection=selection) default = 0 diff --git a/ScanWatch/storage/tables.py b/ScanWatch/storage/tables.py index 7ebaca4..baf9101 100644 --- a/ScanWatch/storage/tables.py +++ b/ScanWatch/storage/tables.py @@ -113,7 +113,7 @@ def get_normal_transaction_table(address: str, scan_type: NETWORK): return Table(f"{scan_type}_{address}_normal_transaction", rows, row_types) -def get_transaction_table(address: str, nt_type: NETWORK, tr_type: TRANSACTION): +def get_transaction_table(address: str, nt_type: NETWORK, net: str, tr_type: TRANSACTION): """ Return the table used to store the transactions depending on the address, network type and transaction type @@ -121,6 +121,8 @@ def get_transaction_table(address: str, nt_type: NETWORK, tr_type: TRANSACTION): :type address: str :param nt_type: type of network :type nt_type: NETWORK + :param net: name of the network, used to differentiate main and test nets + :type net: str :param tr_type: type of the transaction to record :type tr_type: TRANSACTION :return: corresponding table @@ -212,4 +214,7 @@ def get_transaction_table(address: str, nt_type: NETWORK, tr_type: TRANSACTION): raise ValueError(f"unknown transaction type: {tr_type}") row_types = len(rows) * ['TEXT'] - return Table(f"{nt_type.name.lower()}_{tr_type.name.lower()}_{address}_transaction", rows, row_types) + pre_name = f"{nt_type.name.lower()}_{tr_type.name.lower()}" + if net != "main": # backward compatibility + pre_name += f"_{net}" + return Table(pre_name + f"_{address}_transaction", rows, row_types)