diff --git a/.gitignore b/.gitignore index 50bd70a8..55c54299 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ config.ini storage.ini api_config.ini +web_config.ini #Default report location report.json diff --git a/libs/common.py b/libs/common.py index ad11a1c4..2b916109 100644 --- a/libs/common.py +++ b/libs/common.py @@ -16,6 +16,7 @@ except: SSH = False + def load_module(name, path): """ Loads a module by filename and path. Returns module object @@ -31,14 +32,16 @@ def load_module(name, path): print(e) return loaded_mod + def list2cmdline(list): """ This is used to overwrite the default subprocess list2cmdline function. - + The default subprocess list2cmdline function on windows messes with quotes arguments. This will not """ return ' '.join(list) + def convert_encoding(data, encoding='UTF-8', errors='replace'): """ Converts dicts, lists, and strs to the encoding. It uses data.decode to do this. @@ -65,6 +68,7 @@ def convert_encoding(data, encoding='UTF-8', errors='replace'): else: return data + def parse_config(config_object): """Take a config object and returns it as a dictionary""" return_var = {} @@ -78,24 +82,26 @@ def parse_config(config_object): return_var[section] = section_dict return return_var -def get_storage_config_path(config_file): - """Gets the location of the storage config file from the multiscanner config file""" + +def get_config_path(config_file, component): + """Gets the location of the config file for the given multiscanner component + from the multiscanner config file + + Components: + storage + api + web""" conf = configparser.SafeConfigParser() conf.read(config_file) conf = parse_config(conf) try: - return conf['main']['storage-config'] + return conf['main']['%s-config' % component] except KeyError: - print("ERROR: Couldn't find 'storage-config' value in 'main' section "\ - "of config file. Have you run 'python multiscanner.py init'?") + print("ERROR: Couldn't find '%s-config' value in 'main' section " + "of config file. Have you run 'python multiscanner.py init'?" + % component) sys.exit() -def get_api_config_path(config_file): - """Gets the location of the API config file from the multiscanner config file""" - conf = configparser.SafeConfigParser() - conf.read(config_file) - conf = parse_config(conf) - return conf['main']['api-config'] def dirname(path): """OS independent version of os.path.dirname""" @@ -106,6 +112,7 @@ def dirname(path): split = path.split('\\') return '\\'.join(split[:-1]) + def basename(path): """OS independent version of os.path.basename""" if path.endswith('/') or path.endswith('\\'): @@ -116,11 +123,12 @@ def basename(path): else: split = path.split('\\') return split[-1] - + + def parseDir(directory, recursive=False): """ Returns a list of files in a directory. - + dir - The directory to search recursive - If true it will recursively find files. """ @@ -138,11 +146,12 @@ def parseDir(directory, recursive=False): else: filelist.append(item) return filelist - + + def parseFileList(FileList, recursive=False): """ Takes a list of files and directories and returns a list of files. - + FileList - A list of files and directories. Files in each directory will be returned recursive - If true it will recursively find files in directories. """ @@ -159,20 +168,21 @@ def parseFileList(FileList, recursive=False): pass return filelist + def chunk_file_list(filelist, cmdlength=7191): """ Takes the file list and splits it into chunks so windows won't break. Returns a list of lists of strings. - + filelist - The list to be chunked cmdlength - Max length of all filenames appended to each other """ - #This fixes if the cmd line would be far too long - #8191 is the windows limit + # This fixes if the cmd line would be far too long + # 8191 is the windows limit filechunks = [] if len(list2cmdline(filelist)) >= cmdlength: filechunks.append(filelist[:len(filelist)/2]) filechunks.append(filelist[len(filelist)/2:]) - #Keeps splitting chunks until all are correct size + # Keeps splitting chunks until all are correct size splitter = True while splitter: splitter = False @@ -186,17 +196,19 @@ def chunk_file_list(filelist, cmdlength=7191): filechunks = [filelist] return filechunks + def queue2list(queue): """Takes a queue a returns a list of the elements in the queue.""" list = [] while not queue.empty(): list.append(queue.get()) return list - + + def hashfile(fname, hasher, blocksize=65536): """ Hashes a file in chunks and returns the hash algorithms digest. - + fname - The file to be hashed hasher - The hasher from hashlib. E.g. hashlib.md5() blocksize - The size of each block to read in from the file @@ -209,22 +221,24 @@ def hashfile(fname, hasher, blocksize=65536): afile.close() return hasher.hexdigest() + def sshconnect(hostname, port=22, username=None, password=None, pkey=None, key_filename=None, timeout=None, allow_agent=True, look_for_keys=True, compress=False, sock=None): """A wrapper for paramiko, returns a SSHClient after it connects.""" client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostname, port=port, username=username, password=password, pkey=pkey, key_filename=key_filename, timeout=timeout, allow_agent=allow_agent, look_for_keys=look_for_keys, compress=compress, sock=sock) return client - + + def sessionexec(client, cmd): """Creates a session object and executes a command. Returns the session object""" session = client.get_transport().open_session() session.exec_command(cmd) return session - + + def sshexec(hostname, cmd, port=22, username=None, password=None, key_filename=None): """Connects and runs a command. Returns the contents of stdin.""" client = sshconnect(hostname, port=port, username=username, password=password, key_filename=key_filename) stdin, stdout, stderr = client.exec_command(cmd) return stdout.read() - diff --git a/modules/Antivirus/Metadefender.py b/modules/Antivirus/Metadefender.py index 9f2d15aa..5d3fddea 100644 --- a/modules/Antivirus/Metadefender.py +++ b/modules/Antivirus/Metadefender.py @@ -118,7 +118,7 @@ def _parse_scan_result(response): overall_results = response_json.get("scan_results", {}) scan_details = overall_results.get("scan_details", {}) engine_results = [] - for engine_name, engine_output in scan_details.iteritems(): + for engine_name, engine_output in scan_details.items(): scan_code = engine_output.get("scan_result_i", MD_UNKNOWN_SCAN_RES) scan_result_string = MD_SCAN_RES_CODES[scan_code] engine_result = {'engine_name': engine_name, diff --git a/modules/Signature/YaraScan.py b/modules/Signature/YaraScan.py index ac13acff..9370d632 100644 --- a/modules/Signature/YaraScan.py +++ b/modules/Signature/YaraScan.py @@ -76,7 +76,7 @@ def scan(filelist, conf=DEFAULTCONF): finally: f.close() if hit: - hlist = [] + hdict = {} for h in hit: if not set(h.tags).intersection(set(conf["ignore-tags"])): hit_dict = { @@ -85,9 +85,13 @@ def scan(filelist, conf=DEFAULTCONF): 'rule' : h.rule, 'tags' : h.tags, } - hlist.append(hit_dict) - hlist = sorted(hlist, key=itemgetter('rule')) - matches.append((m, hlist)) + try: + h_key = '{}:{}'.format(hit_dict['namespace'].split('/')[-1], hit_dict['rule']) + except IndexError: + h_key = '{}'.format(hit_dict['rule']) + hdict[h_key] = hit_dict + matches.append((m, hdict)) + metadata = {} rulelist = list(ruleset) diff --git a/multiscanner.py b/multiscanner.py index 90961ae1..1cb3a032 100755 --- a/multiscanner.py +++ b/multiscanner.py @@ -51,6 +51,7 @@ "group-types": ["Antivirus"], "storage-config": os.path.join(MS_WD, 'storage.ini'), "api-config": os.path.join(MS_WD, 'api_config.ini'), + "web-config": os.path.join(MS_WD, 'web_config.ini'), } VERBOSE = False diff --git a/storage/elasticsearch_storage.py b/storage/elasticsearch_storage.py index 51f1f4e9..50ad156a 100644 --- a/storage/elasticsearch_storage.py +++ b/storage/elasticsearch_storage.py @@ -97,8 +97,21 @@ def setup(self): name=CUCKOO_TEMPLATE_NAME, body=json.dumps(template) ) - if not es_indices.exists(self.index): + + # Try to create the index, pass if it exists + try: es_indices.create(self.index) + except TransportError: + pass + + # Set the total fields limit + try: + es_indices.put_settings( + index=self.index, + body={'index.mapping.total_fields.limit': ES_MAX}, + ) + except TransportError: + pass # Create parent-child mappings if don't exist yet mappings = es_indices.get_mapping(index=self.index)[self.index]['mappings'].keys() @@ -327,13 +340,11 @@ def search(self, query_string, search_type='default'): '''Run a Query String query and return a list of sample_ids associated with the matches. Run the query against all document types. ''' - print(search_type) if search_type == 'advanced': query = self.build_query(query_string) else: es_reserved_chars_re = '([\+\-=\>\<\!\(\)\{\}\[\]\^\"\~\*\?\:\\/ ])' query_string = re.sub(es_reserved_chars_re, r'\\\g<1>', query_string) - print(query_string) if search_type == 'default': query = self.build_query("*" + query_string + "*") elif search_type == 'exact': diff --git a/storage/sql_driver.py b/storage/sql_driver.py index bf1595d4..dcbf4217 100644 --- a/storage/sql_driver.py +++ b/storage/sql_driver.py @@ -153,12 +153,13 @@ def db_session_scope(self): finally: ses.close() - def add_task(self, task_id=None, task_status='Pending', sample_id=None): + def add_task(self, task_id=None, task_status='Pending', sample_id=None, timestamp=None): with self.db_session_scope() as ses: task = Task( task_id=task_id, task_status=task_status, sample_id=sample_id, + timestamp=timestamp, ) try: ses.add(task) diff --git a/utils/api.py b/utils/api.py index 03073846..bfa04d07 100755 --- a/utils/api.py +++ b/utils/api.py @@ -7,10 +7,12 @@ Proposed supported operations: GET / ---> Test functionality. {'Message': 'True'} +GET /api/v1/files/get/?raw={t|f} ----> download sample, defaults to passwd protected zip GET /api/v1/tasks/list ---> Receive list of tasks in multiscanner GET /api/v1/tasks/list/ ---> receive task in JSON format GET /api/v1/tasks/report/ ---> receive report in JSON GET /api/v1/tasks/delete/ ----> delete task_id +GET /api/v1/tasks/file/?raw={t|f} ----> download sample, defaults to passwd protected zip POST /api/v1/tasks/create ---> POST file and receive report id Sample POST usage: curl -i -X POST http://localhost:8080/api/v1/tasks/create/ -F file=@/bin/ls @@ -29,7 +31,9 @@ import hashlib import codecs import configparser +import json import multiprocessing +import subprocess import queue import shutil from datetime import datetime @@ -87,7 +91,7 @@ def default(self, obj): app.json_encoder = CustomJSONEncoder api_config_object = configparser.SafeConfigParser() api_config_object.optionxform = str -api_config_file = multiscanner.common.get_api_config_path(multiscanner.CONFIG) +api_config_file = multiscanner.common.get_config_path(multiscanner.CONFIG, 'api') api_config_object.read(api_config_file) if not api_config_object.has_section('api') or not os.path.isfile(api_config_file): # Write default config @@ -103,7 +107,7 @@ def default(self, obj): # To run under Apache, we need to set up the DB outside of __main__ db.init_db() -storage_conf = multiscanner.common.get_storage_config_path(multiscanner.CONFIG) +storage_conf = multiscanner.common.get_config_path(multiscanner.CONFIG, 'storage') storage_handler = multiscanner.storage.StorageHandler(configfile=storage_conf) for handler in storage_handler.loaded_storage: if isinstance(handler, elasticsearch_storage.ElasticSearchStorage): @@ -152,8 +156,10 @@ def multiscanner_process(work_queue, exit_signal): continue filelist = [item[0] for item in metadata_list] + #modulelist = [item[5] for item in metadata_list] resultlist = multiscanner.multiscan( filelist, configfile=multiscanner.CONFIG + #module_list ) results = multiscanner.parse_reports(resultlist, python=True) @@ -205,6 +211,28 @@ def index(): return jsonify({'Message': 'True'}) +@app.route('/api/v1/modules', methods=['GET']) +def modules(): + ''' + Return a list of module names available for Multiscanner to use, + and whether or not they are enabled in the config. + ''' + files = multiscanner.parseDir(multiscanner.MODULEDIR, True) + filenames = [os.path.splitext(os.path.basename(f)) for f in files] + module_names = [m[0] for m in filenames if m[1] == '.py'] + + ms_config = configparser.SafeConfigParser() + ms_config.optionxform = str + ms_config.read(multiscanner.CONFIG) + modules = {} + for module in module_names: + try: + modules[module] = ms_config.get(module, 'ENABLED') + except (configparser.NoSectionError, configparser.NoOptionError): + pass + return jsonify({'Modules': modules}) + + @app.route('/api/v1/tasks/list/', methods=['GET']) def task_list(): ''' @@ -218,7 +246,7 @@ def task_list(): def search(params, get_all=False): # Pass search term to Elasticsearch, get back list of sample_ids search_term = params['search[value]'] - search_type = params.pop('search_type', 'Default') + search_type = params.pop('search_type', 'default') if search_term == '': es_result = None else: @@ -296,6 +324,30 @@ def save_hashed_filename(f, zipped=False): return (f_name, full_path) +class InvalidScanTimeFormatError(ValueError): + pass + + +def import_task(file_): + ''' + Import a JSON report that was downloaded from MultiScanner. + ''' + report = json.loads(file_.read().decode('utf-8')) + try: + report['Scan Time'] = datetime.strptime(report['Scan Time'], '%Y-%m-%dT%H:%M:%S.%f') + except ValueError: + raise InvalidScanTimeFormatError() + + task_id = db.add_task( + sample_id=report['SHA256'], + task_status='Complete', + timestamp=report['Scan Time'], + ) + storage_handler.store({report['filename']: report}, wait=False) + + return task_id + + def queue_task(original_filename, f_name, full_path, metadata): ''' Queue up a single new task, for a single non-archive file. @@ -324,15 +376,51 @@ def create_task(): UPLOAD_FOLDER, optionally unzipping it. Return task id and 201 status. ''' file_ = request.files['file'] + if request.form.get('upload_type', None) == 'import': + try: + task_id = import_task(file_) + except KeyError: + return make_response( + jsonify({'Message': 'Cannot import report missing \'Scan Time\' field!'}), + HTTP_BAD_REQUEST) + except InvalidScanTimeFormatError: + return make_response( + jsonify({'Message': 'Cannot import report with \'Scan Time\' of invalid format!'}), + HTTP_BAD_REQUEST) + except (UnicodeDecodeError, ValueError): + return make_response( + jsonify({'Message': 'Cannot import non-JSON files!'}), + HTTP_BAD_REQUEST) + + return make_response( + jsonify({'Message': {'task_ids': [task_id]}}), + HTTP_CREATED + ) + original_filename = file_.filename metadata = {} task_id_list = [] + extract_dir = None for key in request.form.keys(): - if key in ['file_id', 'archive-password'] or request.form[key] == '': + if key in ['file_id', 'archive-password', 'upload_type'] or request.form[key] == '': continue + elif key == 'modules': + module_names = request.form[key] + files = multiscanner.parseDir(multiscanner.MODULEDIR, True) + modules = [] + for f in files: + split = os.path.splitext(os.path.basename(f)) + if split[0] in module_names and split[1] == '.py': + modules.append(f) elif key == 'archive-analyze' and request.form[key] == 'true': extract_dir = api_config['api']['upload_folder'] + if not os.path.isdir(extract_dir): + return make_response( + jsonify({'Message': "'upload_folder' in API config is not " + "a valid folder!"}), + HTTP_BAD_REQUEST) + # Get password if present if 'archive-password' in request.form: password = request.form['archive-password'] @@ -340,45 +428,44 @@ def create_task(): password = bytes(password, 'utf-8') else: password = '' - # Extract a zip - if zipfile.is_zipfile(file_): - z = zipfile.ZipFile(file_) - try: - # NOTE: zipfile module prior to Py 2.7.4 is insecure! - # https://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.extract - z.extractall(path=extract_dir, pwd=password) - for uzfile in z.namelist(): - unzipped_file = open(os.path.join(extract_dir, uzfile)) - f_name, full_path = save_hashed_filename(unzipped_file, True) - tid = queue_task(uzfile, f_name, full_path, metadata) - task_id_list.append(tid) - except RuntimeError as e: - msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) - return make_response( - jsonify({'Message': msg}), - HTTP_BAD_REQUEST - ) - # Extract a rar - elif rarfile.is_rarfile(file_): - r = rarfile.RarFile(file_) - try: - r.extractall(path=extract_dir, pwd=password) - for urfile in r.namelist(): - unrarred_file = open(os.path.join(extract_dir, urfile)) - f_name, full_path = save_hashed_filename(unrarred_file, True) - tid = queue_task(urfile, f_name, full_path, metadata) - task_id_list.append(tid) - except RuntimeError as e: - msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) - return make_response( - jsonify({'Message': msg}), - HTTP_BAD_REQUEST - ) else: metadata[key] = request.form[key] - if not task_id_list: - # File was not zipped + if extract_dir: + # Extract a zip + if zipfile.is_zipfile(file_): + z = zipfile.ZipFile(file_) + try: + # NOTE: zipfile module prior to Py 2.7.4 is insecure! + # https://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.extract + z.extractall(path=extract_dir, pwd=password) + for uzfile in z.namelist(): + unzipped_file = open(os.path.join(extract_dir, uzfile)) + f_name, full_path = save_hashed_filename(unzipped_file, True) + tid = queue_task(uzfile, f_name, full_path, metadata) + task_id_list.append(tid) + except RuntimeError as e: + msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) + return make_response( + jsonify({'Message': msg}), + HTTP_BAD_REQUEST) + # Extract a rar + elif rarfile.is_rarfile(file_): + r = rarfile.RarFile(file_) + try: + r.extractall(path=extract_dir, pwd=password) + for urfile in r.namelist(): + unrarred_file = open(os.path.join(extract_dir, urfile)) + f_name, full_path = save_hashed_filename(unrarred_file, True) + tid = queue_task(urfile, f_name, full_path, metadata) + task_id_list.append(tid) + except RuntimeError as e: + msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) + return make_response( + jsonify({'Message': msg}), + HTTP_BAD_REQUEST) + else: + # File was not an archive to extract f_name, full_path = save_hashed_filename(file_) tid = queue_task(original_filename, f_name, full_path, metadata) task_id_list = [tid] @@ -389,26 +476,51 @@ def create_task(): HTTP_CREATED ) - @app.route('/api/v1/tasks/report/', methods=['GET']) def get_report(task_id): ''' Return a JSON dictionary corresponding to the given task ID. ''' + + download = request.args.get('d', default='False', type=str)[0].lower() + + report_dict, success = get_report_dict(task_id) + if success and (download == 't' or download == 'y' or download == '1'): + response = make_response(jsonify(report_dict)) + response.headers['Content-Type'] = 'application/json' + response.headers['Content-Disposition'] = 'attachment; filename=%s.json' % task_id + return response + else: + return jsonify(report_dict) + +@app.route('/api/v1/tasks/file/', methods=['GET']) +def files_get_task(task_id): + # try to get report dict + report_dict, success = get_report_dict(task_id) + if not success: + return jsonify(report_dict) + + # okay, we have report dict; get sha256 + sha256 = report_dict.get('Report', {}).get('SHA256') + if sha256: + return files_get_sha256_helper( + sha256, + request.args.get('raw', default=None)) + else: + return jsonify({'Error': 'sha256 not in report!'}) + +def get_report_dict(task_id): task = db.get_task(task_id) if not task: abort(HTTP_NOT_FOUND) if task.task_status == 'Complete': - report = handler.get_report(task.sample_id, task.timestamp) + return {'Report': handler.get_report(task.sample_id, task.timestamp)}, True elif task.task_status == 'Pending': - report = {'Report': 'Task still pending'} + return {'Report': 'Task still pending'}, False else: - report = {'Report': 'Task failed'} - - return jsonify({'Report': report}) - + return {'Report': 'Task failed'}, False @app.route('/api/v1/tasks/delete/', methods=['GET']) def delete_report(task_id): @@ -514,7 +626,7 @@ def edit_note(task_id, note_id): abort(HTTP_NOT_FOUND) response = handler.edit_note(task.sample_id, note_id, - Markup(request.form['text']).striptags()) + Markup(request.form.get('text', '')).striptags()) if not response: abort(HTTP_BAD_REQUEST) return jsonify(response) @@ -534,6 +646,65 @@ def del_note(task_id, note_id): abort(HTTP_BAD_REQUEST) return jsonify(response) +@app.route('/api/v1/files/get/', methods=['GET']) +# get raw file - /api/v1/files/get/?raw=true +def files_get_sha256(sha256): + ''' + Returns binary from storage. Defaults to password protected zipfile. + ''' + # is there a robust way to just get this as a bool? + raw = request.args.get('raw', default='False', type=str) + + return files_get_sha256_helper(sha256, raw) + +def files_get_sha256_helper(sha256, raw=None): + ''' + Returns binary from storage. Defaults to password protected zipfile. + ''' + file_path = os.path.join(api_config['api']['upload_folder'], sha256) + if not os.path.exists(file_path): + abort(HTTP_NOT_FOUND) + + with open(file_path, "rb") as fh: + fh_content = fh.read() + + raw = raw[0].lower() + if raw == 't' or raw == 'y' or raw == '1': + response = make_response(fh_content) + response.headers['Content-Type'] = 'application/octet-stream; charset=UTF-8' + response.headers['Content-Disposition'] = 'inline; filename={}.bin'.format(sha256) # better way to include fname? + else: + # ref: https://github.com/crits/crits/crits/core/data_tools.py#L122 + rawname = sha256 + '.bin' + with open(os.path.join('/tmp/', rawname), 'wb') as raw_fh: + raw_fh.write(fh_content) + + zipname = sha256 + '.zip' + args = ['/usr/bin/zip', '-j', + os.path.join('/tmp', zipname), + os.path.join('/tmp', rawname), + '-P', 'infected'] + proc = subprocess.Popen(args) + wait_seconds = 30 + while proc.poll() is None and wait_seconds: + time.sleep(1) + wait_seconds -= 1 + + if proc.returncode: + return make_response(jsonify({'Error': 'Failed to create zip ()'.format(proc.returncode)})) + elif not wait_seconds: + proc.terminate() + return make_response(jsonify({'Error': 'Process timed out'})) + else: + with open(os.path.join('/tmp', zipname), 'rb') as zip_fh: + zip_data = zip_fh.read() + if len(zip_data) == 0: + return make_response(jsonify({'Error': 'Zip file empty'})) + response = make_response(zip_data) + response.headers['Content-Type'] = 'application/zip; charset=UTF-8' + response.headers['Content-Disposition'] = 'inline; filename={}.zip'.format(sha256) + return response + if __name__ == '__main__': diff --git a/utils/celery_worker.py b/utils/celery_worker.py index 1d41295a..dfea08e3 100644 --- a/utils/celery_worker.py +++ b/utils/celery_worker.py @@ -9,6 +9,7 @@ import codecs import configparser from datetime import datetime +from socket import gethostname MS_WD = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Append .. to sys path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -37,7 +38,7 @@ config_object = configparser.SafeConfigParser() config_object.optionxform = str -configfile = common.get_api_config_path(multiscanner.CONFIG) +configfile = common.get_config_path(multiscanner.CONFIG, 'api') config_object.read(configfile) if not config_object.has_section('celery') or not os.path.isfile(configfile): @@ -69,7 +70,7 @@ def celery_task(files, config=multiscanner.CONFIG): handler(s) specified in the storage configuration file. ''' # Get the storage config - storage_conf = multiscanner.common.get_storage_config_path(config) + storage_conf = multiscanner.common.get_config_path(config, 'storage') storage_handler = multiscanner.storage.StorageHandler(configfile=storage_conf) resultlist = multiscanner.multiscan(list(files), configfile=config) @@ -84,6 +85,21 @@ def celery_task(files, config=multiscanner.CONFIG): task_id = files[file_]['task_id'] file_hash = files[file_]['file_hash'] metadata = files[file_]['metadata'] + # Get the Scan Config that the task was run with and + # add it to the task metadata + scan_config_object = configparser.SafeConfigParser() + scan_config_object.optionxform = str + scan_config_object.read(config) + full_conf = common.parse_config(scan_config_object) + sub_conf = {} + for key in full_conf: + if key == 'main': + continue + sub_conf[key] = {} + sub_conf[key]['ENABLED'] = full_conf[key]['ENABLED'] + results[file_]['Scan Metadata'] = {} + results[file_]['Scan Metadata']['Worker Node'] = gethostname() + results[file_]['Scan Metadata']['Scan Config'] = sub_conf # Use the original filename as the value for the filename # in the report (instead of the tmp path assigned to the file diff --git a/utils/dir_monitor.py b/utils/dir_monitor.py index db1c68f2..7ea4c5f8 100755 --- a/utils/dir_monitor.py +++ b/utils/dir_monitor.py @@ -72,7 +72,7 @@ def start_observer(directory, work_queue, recursive=False): def multiscanner_process(work_queue, config, batch_size, wait_seconds, delete, exit_signal): filelist = [] time_stamp = None - storage_conf = multiscanner.common.get_storage_config_path(config) + storage_conf = multiscanner.common.get_config_path(config, 'storage') storage_handler = multiscanner.storage.StorageHandler(configfile=storage_conf) while not exit_signal.value: time.sleep(1) diff --git a/utils/distributed_worker.py b/utils/distributed_worker.py index 945d84db..7f35acf5 100755 --- a/utils/distributed_worker.py +++ b/utils/distributed_worker.py @@ -35,7 +35,7 @@ def multiscanner_process(work_queue, config, batch_size, wait_seconds, delete, exit_signal): filelist = [] time_stamp = None - storage_conf = multiscanner.common.get_storage_config_path(config) + storage_conf = multiscanner.common.get_config_path(config, 'storage') storage_handler = multiscanner.storage.StorageHandler(configfile=storage_conf) while not exit_signal.value: time.sleep(1) diff --git a/web/app.py b/web/app.py index 8004ac53..1b3a64b6 100644 --- a/web/app.py +++ b/web/app.py @@ -1,7 +1,49 @@ -from flask import (Flask, render_template, request, redirect, url_for, make_response, flash) +import codecs +from collections import namedtuple +import configparser +from flask import Flask, render_template, request +import os +import re +import sys + +MS_WD = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if MS_WD not in sys.path: + sys.path.insert(0, os.path.join(MS_WD)) +import multiscanner + +DEFAULTCONF = { + 'HOST': "localhost", + 'PORT': 8000, + 'API_LOC': "http://localhost:8080", + 'DEBUG': False, + 'METADATA_FIELDS': [ + "Submitter Name", + "Submission Description", + "Submitter Email", + "Submitter Organization", + "Submitter Phone", + ] +} app = Flask(__name__) -app.config.from_object('config') + +# Finagle Flask to read config from .ini file instead of .py file +web_config_object = configparser.SafeConfigParser() +web_config_object.optionxform = str +web_config_file = multiscanner.common.get_config_path(multiscanner.CONFIG, 'web') +web_config_object.read(web_config_file) +if not web_config_object.has_section('web') or not os.path.isfile(web_config_file): + # Write default config + web_config_object.add_section('web') + for key in DEFAULTCONF: + web_config_object.set('web', key, str(DEFAULTCONF[key])) + conffile = codecs.open(web_config_file, 'w', 'utf-8') + web_config_object.write(conffile) + conffile.close() +web_config = multiscanner.common.parse_config(web_config_object)['web'] +conf_tuple = namedtuple('WebConfig', web_config.keys())(*web_config.values()) +app.config.from_object(conf_tuple) + @app.route('/', methods=['GET']) def index(): @@ -9,20 +51,32 @@ def index(): metadata_fields=app.config['METADATA_FIELDS']) -@app.route('/analyses', methods=['GET']) +@app.route('/analyses', methods=['GET', 'POST']) def tasks(): - return render_template('analyses.html', api_loc=app.config['API_LOC']) + if request.method == 'POST': + return render_template('analyses.html', api_loc=app.config['API_LOC'], + search_term=request.form['search_term'], + search_type=request.form['search_type_buttons']) + else: + return render_template('analyses.html', api_loc=app.config['API_LOC']) @app.route('/report/', methods=['GET']) def reports(task_id=1): + term = re.escape(request.args.get('st', '')) + return render_template('report.html', task_id=task_id, - api_loc=app.config['API_LOC']) + api_loc=app.config['API_LOC'], search_term=term) -@app.route('/history', methods=['GET']) +@app.route('/history', methods=['GET', 'POST']) def history(): - return render_template('history.html', api_loc=app.config['API_LOC']) + if request.method == 'POST': + return render_template('history.html', api_loc=app.config['API_LOC'], + search_term=request.form['search_term'], + search_type=request.form['search_type_buttons']) + else: + return render_template('history.html', api_loc=app.config['API_LOC']) if __name__ == "__main__": diff --git a/web/config.py b/web/config.py deleted file mode 100644 index 1e7e7075..00000000 --- a/web/config.py +++ /dev/null @@ -1,16 +0,0 @@ -# Host to bind this server on -HOST = "localhost" -# Port to bind this server on -PORT = 8000 -# URI for the API server to which we will connect. Make sure to include the http:// or https:// -API_LOC = "http://localhost:8080" -# Set to True to enable debug mode, but DO NOT USE in production -DEBUG = False -# List of metadata fields that can be set when submitting a file for scanning -METADATA_FIELDS = [ - "Submitter Name", - "Submission Description", - "Submitter Email", - "Submitter Organization", - "Submitter Phone", -] \ No newline at end of file diff --git a/web/static/css/styles.css b/web/static/css/styles.css index b5061724..79e18004 100644 --- a/web/static/css/styles.css +++ b/web/static/css/styles.css @@ -11,6 +11,32 @@ } } +#nav-search-opts { + position: absolute; + left: -10px; + width: 325px; + margin-top: 5px; + background-color: #3e648d; + padding: 0px 18px; + border: 1px solid #345578; + border-top: 0px; + border-bottom-left-radius: 10px; + border-bottom-right-radius: 10px; + z-index: 1; +} + +#nav-search-opts.collapse.in { + box-shadow: 3px 3px 3px #0008; +} + +#nav-search-opts > div { + padding: 5px 0px; +} + +#nav-search-opts > div > label { + color: #ddd; +} + .alert-row { position: absolute; right: 20px; @@ -42,6 +68,29 @@ padding-top: 0; } +#adv-options .opt-section:not(:last-child) { + padding-bottom: 12px; +} + +#file-group { + width: 100%; +} +@media (max-width: 992px) { + .input-group { + width: 90%; + margin: auto; + } +} + + +#module-opts { + display: inline-block; +} + +#module-opts .checkbox { + text-align: left; +} + .file-preview { border: 0; } @@ -171,6 +220,7 @@ padding: 8px; text-align: left; vertical-align: top; + max-width: 350px; } #report td { @@ -186,6 +236,13 @@ border-top: 0px; } +#report .table > thead > tr > th, .table > tbody > tr > th, +#report .table > tfoot > tr > th, .table > thead > tr > td, +#report .table > tbody > tr > td, .table > tfoot > tr > td { + word-wrap: break-word; + overflow-wrap: break-word; +} + #report td.cell-table { padding-top: 0px; } @@ -245,6 +302,30 @@ background-color: #428bca; } +#dl-json { + position: absolute; + padding-left: 25px; + padding-right: 25px; + right: 5px; +} +@media (min-width: 992px) { + #dl-json { + right: 16.66666667%; + } +} + +#dl-dropdown { + position: absolute; + padding-left: 25px; + padding-right: 25px; + right: 5px; +} +@media (min-width: 992px) { + #dl-dropdown { + right: 16.66666667%; + } +} + #report-notes { position: fixed; z-index: 1; @@ -267,11 +348,16 @@ #notes-close { padding-bottom: 10px; position: fixed; - top: 55px; + top: 93px; left: -50px; border-radius: 0 20px 20px 0; padding-right: 20px; } +@media (min-width: 992px) { + #notes-close { + top: 55px; + } +} #notes-close.open { transition: .5s ease-out; @@ -280,11 +366,16 @@ #notes-open { position: fixed; - top: 55px; + top: 93px; left: 0; border-radius: 0 20px 20px 0; padding-right: 20px; } +@media (min-width: 992px) { + #notes-open { + top: 55px; + } +} #report-notes .panel { margin-bottom: 5px; @@ -392,6 +483,11 @@ animation: spin-rev 1s infinite linear; } +.highlight { + background-color: #d47500; + color: #fbeed5; +} + .dataTables_wrapper .dataTables_length .dt-buttons { float: right; } @@ -409,3 +505,35 @@ overflow: hidden; word-wrap: break-word; } + +table.dataTable thead .sorting:after { + content: None; +} +table.dataTable thead .sorting:after, +table.dataTable thead .sorting_asc:after, +table.dataTable thead .sorting_desc:after, +table.dataTable thead .sorting_asc_disabled:after, +table.dataTable thead .sorting_desc_disabled:after { + content: None; +} +table.dataTable thead .sorting:before { + opacity: 0.2; + content: "\e150"; +} +table.dataTable thead .sorting_desc:before { + opacity: 0.5; + content: "\e156"; +} +table.dataTable thead .sorting_asc:before { + opacity: 0.5; + content: "\e155"; +} +table.dataTable thead .sorting:before, +table.dataTable thead .sorting_asc:before, +table.dataTable thead .sorting_desc:before, +table.dataTable thead .sorting_asc_disabled:before, +table.dataTable thead .sorting_desc_disabled:before { + position: relative; + display: block; + font-family: 'Glyphicons Halflings'; +} diff --git a/web/templates/analyses.html b/web/templates/analyses.html index c7cf76cc..32b61a8a 100644 --- a/web/templates/analyses.html +++ b/web/templates/analyses.html @@ -5,20 +5,31 @@ + {% block head %}{% endblock %} - @@ -54,7 +102,7 @@ -