From ca48931147efb985c9bf31d4d050489c7a3562d5 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Thu, 15 Nov 2018 09:46:44 +0100 Subject: [PATCH 1/6] Initial Django inspired component loading --- CHANGELOG.rst | 2 +- neo/Settings.py | 3 +++ neo/bin/api_server.py | 31 +++++++++---------------------- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 41b10b62a..4dd581763 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,7 @@ All notable changes to this project are documented in this file. - Fix RPC's ``getaccountstate`` response schema to match ``neo-cli`` `#714 ` - Add fix to ensure tx is saved to wallet when sent using RPC - Add bad peers to the ``getpeers`` RPC method `#715 ` - +- Introduce Django inspired component loading for REST and RPC server [0.8.2] 2018-10-31 ------------------- diff --git a/neo/Settings.py b/neo/Settings.py index 607b8ebe8..87475dd4a 100644 --- a/neo/Settings.py +++ b/neo/Settings.py @@ -382,3 +382,6 @@ def check_privatenet(self): if not os.getenv("SKIP_PY_CHECK"): if sys.version_info < (3, 6): raise SystemCheckError("Needs Python 3.6+. Currently used: %s" % sys.version) + +RPC_SERVER = 'neo.api.JSONRPC.JsonRpcApi.JsonRpcApi' +REST_SERVER = 'neo.api.REST.RestApi.RestApi' diff --git a/neo/bin/api_server.py b/neo/bin/api_server.py index ee2b50386..999806491 100755 --- a/neo/bin/api_server.py +++ b/neo/bin/api_server.py @@ -2,7 +2,7 @@ """ API server to run the JSON-RPC and REST API. -Uses neo.api.JSONRPC.JsonRpcApi or neo.api.JSONRPC.ExtendedJsonRpcApi and neo.api.REST.RestApi +Uses servers specified in neo.Settings.RPC_SERVER and neo.Settings.REST_SERVER Print the help and all possible arguments: @@ -52,16 +52,14 @@ # neo methods and modules from neo.Core.Blockchain import Blockchain from neo.Implementations.Blockchains.LevelDB.LevelDBBlockchain import LevelDBBlockchain -from neo.api.JSONRPC.JsonRpcApi import JsonRpcApi -from neo.api.JSONRPC.ExtendedJsonRpcApi import ExtendedJsonRpcApi from neo.Implementations.Notifications.LevelDB.NotificationDB import NotificationDB -from neo.api.REST.RestApi import RestApi from neo.Wallets.utils import to_aes_key from neo.Implementations.Wallets.peewee.UserWallet import UserWallet from neo.Network.NodeLeader import NodeLeader from neo.Settings import settings - +from neo.Utils.plugin import load_class_from_path +import neo.Settings # Logfile default settings (only used if --logfile arg is used) LOGFILE_MAX_BYTES = 5e7 # 50 MB @@ -128,9 +126,6 @@ def main(): # host parser.add_argument("--host", action="store", type=str, help="Hostname ( for example 127.0.0.1)", default="0.0.0.0") - # extended json-rpc api - parser.add_argument("--extended-rpc", action="store_true", default=False, help="Use extended json-rpc api") - # Now parse args = parser.parse_args() # print(args) @@ -249,28 +244,20 @@ def loopingCallErrorHandler(error): d.setDaemon(True) # daemonizing the thread will kill it when the main thread is quit d.start() - if args.port_rpc and args.extended_rpc: - logger.info("Starting extended json-rpc api server on http://%s:%s" % (args.host, args.port_rpc)) - api_server_rpc = ExtendedJsonRpcApi(args.port_rpc, wallet=wallet) - endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, args.host) - endpoints.serverFromString(reactor, endpoint_rpc).listen(Site(api_server_rpc.app.resource())) -# reactor.listenTCP(int(args.port_rpc), server.Site(api_server_rpc)) -# api_server_rpc.app.run(args.host, args.port_rpc) - - elif args.port_rpc: + if args.port_rpc: logger.info("Starting json-rpc api server on http://%s:%s" % (args.host, args.port_rpc)) - api_server_rpc = JsonRpcApi(args.port_rpc, wallet=wallet) + rpc_class = load_class_from_path(neo.Settings.RPC_SERVER) + api_server_rpc = rpc_class(args.port_rpc, wallet=wallet) + endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, args.host) endpoints.serverFromString(reactor, endpoint_rpc).listen(Site(api_server_rpc.app.resource())) -# reactor.listenTCP(int(args.port_rpc), server.Site(api_server_rpc)) -# api_server_rpc.app.run(args.host, args.port_rpc) if args.port_rest: logger.info("Starting REST api server on http://%s:%s" % (args.host, args.port_rest)) - api_server_rest = RestApi() + rest_api = load_class_from_path(neo.Settings.REST_SERVER) + api_server_rest = rest_api() endpoint_rest = "tcp:port={0}:interface={1}".format(args.port_rest, args.host) endpoints.serverFromString(reactor, endpoint_rest).listen(Site(api_server_rest.app.resource())) -# api_server_rest.app.run(args.host, args.port_rest) reactor.run() From 668bcc52baca359ec8367222a92389ed0b9b8614 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Thu, 15 Nov 2018 09:52:29 +0100 Subject: [PATCH 2/6] add graceful exception handling --- neo/bin/api_server.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/neo/bin/api_server.py b/neo/bin/api_server.py index 999806491..4c941bf80 100755 --- a/neo/bin/api_server.py +++ b/neo/bin/api_server.py @@ -33,6 +33,7 @@ * https://twistedmatrix.com/documents/17.9.0/api/twisted.logger.STDLibLogObserver.html """ import os +import sys import argparse import threading from time import sleep @@ -246,7 +247,11 @@ def loopingCallErrorHandler(error): if args.port_rpc: logger.info("Starting json-rpc api server on http://%s:%s" % (args.host, args.port_rpc)) - rpc_class = load_class_from_path(neo.Settings.RPC_SERVER) + try: + rpc_class = load_class_from_path(neo.Settings.RPC_SERVER) + except ValueError as err: + logger.error(err) + sys.exit() api_server_rpc = rpc_class(args.port_rpc, wallet=wallet) endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, args.host) @@ -254,7 +259,11 @@ def loopingCallErrorHandler(error): if args.port_rest: logger.info("Starting REST api server on http://%s:%s" % (args.host, args.port_rest)) - rest_api = load_class_from_path(neo.Settings.REST_SERVER) + try: + rest_api = load_class_from_path(neo.Settings.REST_SERVER) + except ValueError as err: + logger.error(err) + sys.exit() api_server_rest = rest_api() endpoint_rest = "tcp:port={0}:interface={1}".format(args.port_rest, args.host) endpoints.serverFromString(reactor, endpoint_rest).listen(Site(api_server_rest.app.resource())) From 1c5d535cdcc350afd16b6240be3076190c589e59 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Thu, 15 Nov 2018 10:13:43 +0100 Subject: [PATCH 3/6] Add missing class --- neo/Utils/plugin.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 neo/Utils/plugin.py diff --git a/neo/Utils/plugin.py b/neo/Utils/plugin.py new file mode 100644 index 000000000..4bca6dd20 --- /dev/null +++ b/neo/Utils/plugin.py @@ -0,0 +1,26 @@ +import importlib + + +def load_class_from_path(path_and_class: str): + """ + Dynamically load a class from a module at the specified path + + Args: + path_and_class: relative path where to find the module and its class name + i.e. 'neo....' + + Returns: + class object + """ + try: + module_path = '.'.join(path_and_class.split('.')[:-1]) + module = importlib.import_module(module_path) + except ImportError as err: + raise ValueError(f"Failed to import module {module_path} with error: {err}") + + try: + class_name = path_and_class.split('.')[-1] + class_obj = getattr(module, class_name) + return class_obj + except AttributeError as err: + raise ValueError(f"Failed to get class {class_name} with error: {err}") From 7be2291d3bd2343e0fb49ee32c15ec589494338e Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Sat, 17 Nov 2018 09:43:36 +0100 Subject: [PATCH 4/6] cleanup docs --- neo/Utils/plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/neo/Utils/plugin.py b/neo/Utils/plugin.py index 4bca6dd20..a566e8778 100644 --- a/neo/Utils/plugin.py +++ b/neo/Utils/plugin.py @@ -9,6 +9,9 @@ def load_class_from_path(path_and_class: str): path_and_class: relative path where to find the module and its class name i.e. 'neo....' + Raises: + ValueError: if the Module or Class is not found. + Returns: class object """ From 34642cf04f16bc6221716e1424a0a9c7aaa64d16 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Sat, 17 Nov 2018 14:57:00 +0100 Subject: [PATCH 5/6] move settings to protocol file --- neo/Settings.py | 10 +++++++--- neo/bin/api_server.py | 4 ++-- neo/data/protocol.mainnet.json | 4 +++- neo/data/protocol.privnet.json | 6 ++++-- neo/data/protocol.testnet.json | 4 +++- neo/data/protocol.unittest-net.json | 4 +++- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/neo/Settings.py b/neo/Settings.py index 87475dd4a..1e07103a2 100644 --- a/neo/Settings.py +++ b/neo/Settings.py @@ -107,6 +107,11 @@ class SettingsHolder: VERSION_NAME = "/NEO-PYTHON:%s/" % __version__ + RPC_SERVER = None + REST_SERVER = None + DEFAULT_RPC_SERVER = 'neo.api.JSONRPC.JsonRpcApi.JsonRpcApi' + DEFAULT_REST_SERVER = 'neo.api.REST.RestApi.RestApi' + # Logging settings log_level = None log_smart_contract_events = False @@ -210,6 +215,8 @@ def get_config_and_warn(key, default, abort=False): self.NOTIFICATION_DB_PATH = config.get('NotificationDataPath', 'Chains/notification_data') self.SERVICE_ENABLED = config.get('ServiceEnabled', False) self.COMPILER_NEP_8 = config.get('CompilerNep8', False) + self.REST_SERVER = config.get('RestServer', self.DEFAULT_REST_SERVER) + self.RPC_SERVER = config.get('RPCServer', self.DEFAULT_RPC_SERVER) def setup_mainnet(self): """ Load settings from the mainnet JSON config file """ @@ -382,6 +389,3 @@ def check_privatenet(self): if not os.getenv("SKIP_PY_CHECK"): if sys.version_info < (3, 6): raise SystemCheckError("Needs Python 3.6+. Currently used: %s" % sys.version) - -RPC_SERVER = 'neo.api.JSONRPC.JsonRpcApi.JsonRpcApi' -REST_SERVER = 'neo.api.REST.RestApi.RestApi' diff --git a/neo/bin/api_server.py b/neo/bin/api_server.py index 4c941bf80..27f9a8ce7 100755 --- a/neo/bin/api_server.py +++ b/neo/bin/api_server.py @@ -248,7 +248,7 @@ def loopingCallErrorHandler(error): if args.port_rpc: logger.info("Starting json-rpc api server on http://%s:%s" % (args.host, args.port_rpc)) try: - rpc_class = load_class_from_path(neo.Settings.RPC_SERVER) + rpc_class = load_class_from_path(settings.RPC_SERVER) except ValueError as err: logger.error(err) sys.exit() @@ -260,7 +260,7 @@ def loopingCallErrorHandler(error): if args.port_rest: logger.info("Starting REST api server on http://%s:%s" % (args.host, args.port_rest)) try: - rest_api = load_class_from_path(neo.Settings.REST_SERVER) + rest_api = load_class_from_path(settings.REST_SERVER) except ValueError as err: logger.error(err) sys.exit() diff --git a/neo/data/protocol.mainnet.json b/neo/data/protocol.mainnet.json index 3a47526e6..d5e769b60 100644 --- a/neo/data/protocol.mainnet.json +++ b/neo/data/protocol.mainnet.json @@ -57,6 +57,8 @@ "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest", "DebugStorage": 1, "AcceptIncomingPeers": false, - "CompilerNep8": false + "CompilerNep8": false, + "RestServer": "neo.api.REST.RestApi.RestApi", + "RPCServer": "neo.api.JSONRPC.JsonRpcApi.JsonRpcApi" } } diff --git a/neo/data/protocol.privnet.json b/neo/data/protocol.privnet.json index 9c5d3668a..82fb358b7 100644 --- a/neo/data/protocol.privnet.json +++ b/neo/data/protocol.privnet.json @@ -35,10 +35,12 @@ ], "SslCert": "", "SslCertPassword": "", - "BootstrapName": "mainnet", + "BootstrapName": "privnet", "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest", "DebugStorage": 1, "AcceptIncomingPeers": false, - "CompilerNep8":false + "CompilerNep8": false, + "RestServer": "neo.api.REST.RestApi.RestApi", + "RPCServer": "neo.api.JSONRPC.JsonRpcApi.JsonRpcApi" } } diff --git a/neo/data/protocol.testnet.json b/neo/data/protocol.testnet.json index f4db921f2..f7fff6b33 100644 --- a/neo/data/protocol.testnet.json +++ b/neo/data/protocol.testnet.json @@ -14,7 +14,9 @@ "BootstrapFiles": "https://s3.us-east-2.amazonaws.com/cityofzion/bootstrap_latest", "DebugStorage": 1, "AcceptIncomingPeers": false, - "CompilerNep8": false + "CompilerNep8": false, + "RestServer": "neo.api.REST.RestApi.RestApi", + "RPCServer": "neo.api.JSONRPC.JsonRpcApi.JsonRpcApi" }, "ProtocolConfiguration": { "AddressVersion": 23, diff --git a/neo/data/protocol.unittest-net.json b/neo/data/protocol.unittest-net.json index b3d2a8beb..83671ee38 100644 --- a/neo/data/protocol.unittest-net.json +++ b/neo/data/protocol.unittest-net.json @@ -41,6 +41,8 @@ "AcceptIncomingPeers": false, "CompilerNep8": true, "BootstrapName": "fauxnet", - "BootstrapFiles": "this_does_not_exist_for_this_network" + "BootstrapFiles": "this_does_not_exist_for_this_network", + "RestServer": "neo.api.REST.RestApi.RestApi", + "RPCServer": "neo.api.JSONRPC.JsonRpcApi.JsonRpcApi" } } From 52d60b04b16c23254f90bc944f60eb1aa3fc5ae1 Mon Sep 17 00:00:00 2001 From: Erik van den Brink Date: Sat, 17 Nov 2018 15:01:01 +0100 Subject: [PATCH 6/6] update api_server server location comment --- neo/bin/api_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/bin/api_server.py b/neo/bin/api_server.py index 27f9a8ce7..29b03b01f 100755 --- a/neo/bin/api_server.py +++ b/neo/bin/api_server.py @@ -2,7 +2,7 @@ """ API server to run the JSON-RPC and REST API. -Uses servers specified in neo.Settings.RPC_SERVER and neo.Settings.REST_SERVER +Uses servers specified in protocol.xxx.json files Print the help and all possible arguments: