From 8e655ed8082461ff80b2ab8951186fe3e0840598 Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 19 Jul 2017 16:26:51 -0400 Subject: [PATCH 01/50] Add module options to scan page, populated from MultiScanner config --- utils/api.py | 22 +++++++++++++++ web/config.py | 2 +- web/static/css/styles.css | 12 +++++++++ web/templates/index.html | 56 +++++++++++++++++++++++++-------------- 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/utils/api.py b/utils/api.py index 03073846..cb4d05b4 100755 --- a/utils/api.py +++ b/utils/api.py @@ -205,6 +205,28 @@ def index(): return jsonify({'Message': 'True'}) +@app.route('/api/v1/modules', methods=['GET']) +def modules(): + ''' + Return a list of module names available for Multiscanner to use, + and whether or not they are enabled in the config. + ''' + files = multiscanner.parseDir(multiscanner.MODULEDIR, True) + filenames = [os.path.splitext(os.path.basename(f)) for f in files] + module_names = [m[0] for m in filenames if m[1] == '.py'] + + ms_config = configparser.SafeConfigParser() + ms_config.optionxform = str + ms_config.read(multiscanner.CONFIG) + modules = {} + for module in module_names: + try: + modules[module] = ms_config.get(module, 'ENABLED') + except (configparser.NoSectionError, configparser.NoOptionError): + pass + return jsonify({'Modules': modules}) + + @app.route('/api/v1/tasks/list/', methods=['GET']) def task_list(): ''' diff --git a/web/config.py b/web/config.py index c1d46acc..9b89a4bb 100644 --- a/web/config.py +++ b/web/config.py @@ -13,4 +13,4 @@ "Submitter Email", "Submitter Organization", "Submitter Phone", -] \ No newline at end of file +] diff --git a/web/static/css/styles.css b/web/static/css/styles.css index b5061724..e5ce5527 100644 --- a/web/static/css/styles.css +++ b/web/static/css/styles.css @@ -42,6 +42,18 @@ padding-top: 0; } +#adv-options .opt-section:not(:last-child) { + padding-bottom: 12px; +} + +#module-opts { + display: inline-block; +} + +#module-opts .checkbox { + text-align: left; +} + .file-preview { border: 0; } diff --git a/web/templates/index.html b/web/templates/index.html index 5a440864..6f530da7 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -22,7 +22,7 @@ ' ' + ' {footer}\n' + ' \n' + - '\n' + + '\n' + '
\n' + {% for metadata in metadata_fields %} '
\n' + @@ -52,7 +52,7 @@ ' {upload} {delete} {other}' + ' \n' + + ' \n' + '
\n' + ' {drag}\n' + '
{indicator}
\n' + @@ -105,7 +105,7 @@ $('#filesUpload').on('fileloaded', function(event, file, previewId, index, reader) { // Scroll to file - window.scrollTo(0, $("#" + previewId).offset().top); + window.scrollTo(0, $("#" + previewId).offset().top); // Singularize/pluralize submit button text if ($("#filesUpload").fileinput('getFilesCount') > 1) { @@ -127,6 +127,17 @@ }); }); + // Add options for selecting which modules to run + $.get("http://{{ api_loc }}/api/v1/modules", function(data) { + var modules = '

Modules

Select which modules to use:

'; + for (mod in data.Modules) { + checked = (data.Modules[mod] == "True" ? "checked" : ""); + modules += '
'; + } + modules += '
'; + $('#adv-options').prepend(modules); + }); + // Set metadata field values for all files $('#adv-options .metadata-group input').change(function() { var field = $(this).attr('id').substring(9); @@ -171,26 +182,31 @@

MultiScanner

-
- -
-
- -
- +
+

Archives

+
+ +
+
+ +
+ +
-

Metadata Fields

-

(These apply to all files in this submission.)

- {% for metadata in metadata_fields %} -
- {% set field = metadata|replace(" ", "-")|replace(".", "-")|replace("#", "-")%} - -
- -
+
+

Metadata Fields

+

(These apply to all files in this submission.)

+ {% for metadata in metadata_fields %} +
+ {% set field = metadata|replace(" ", "-")|replace(".", "-")|replace("#", "-")%} + +
+ +
+
+ {% endfor %}
- {% endfor %}
From efa49004ded109e1dbc97c3bda79f5e952a06fcb Mon Sep 17 00:00:00 2001 From: clenk Date: Thu, 20 Jul 2017 15:16:14 -0400 Subject: [PATCH 02/50] Parse modules list in the API --- utils/api.py | 87 +++++++++++++++++++++++----------------- web/templates/index.html | 7 +++- 2 files changed, 57 insertions(+), 37 deletions(-) diff --git a/utils/api.py b/utils/api.py index cb4d05b4..cd8ba090 100755 --- a/utils/api.py +++ b/utils/api.py @@ -152,8 +152,10 @@ def multiscanner_process(work_queue, exit_signal): continue filelist = [item[0] for item in metadata_list] + #modulelist = [item[5] for item in metadata_list] resultlist = multiscanner.multiscan( filelist, configfile=multiscanner.CONFIG + #module_list ) results = multiscanner.parse_reports(resultlist, python=True) @@ -353,8 +355,22 @@ def create_task(): for key in request.form.keys(): if key in ['file_id', 'archive-password'] or request.form[key] == '': continue + elif key == 'modules': + module_names = request.form[key] + files = multiscanner.parseDir(multiscanner.MODULEDIR, True) + modules = [] + for f in files: + split = os.path.splitext(os.path.basename(f)) + if split[0] in module_names and split[1] == '.py': + modules.append(f) elif key == 'archive-analyze' and request.form[key] == 'true': extract_dir = api_config['api']['upload_folder'] + if not os.path.isdir(extract_dir): + return make_response( + jsonify({'Message': "'upload_folder' in API config is not " + "a valid folder!"}), + HTTP_BAD_REQUEST) + # Get password if present if 'archive-password' in request.form: password = request.form['archive-password'] @@ -362,45 +378,44 @@ def create_task(): password = bytes(password, 'utf-8') else: password = '' - # Extract a zip - if zipfile.is_zipfile(file_): - z = zipfile.ZipFile(file_) - try: - # NOTE: zipfile module prior to Py 2.7.4 is insecure! - # https://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.extract - z.extractall(path=extract_dir, pwd=password) - for uzfile in z.namelist(): - unzipped_file = open(os.path.join(extract_dir, uzfile)) - f_name, full_path = save_hashed_filename(unzipped_file, True) - tid = queue_task(uzfile, f_name, full_path, metadata) - task_id_list.append(tid) - except RuntimeError as e: - msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) - return make_response( - jsonify({'Message': msg}), - HTTP_BAD_REQUEST - ) - # Extract a rar - elif rarfile.is_rarfile(file_): - r = rarfile.RarFile(file_) - try: - r.extractall(path=extract_dir, pwd=password) - for urfile in r.namelist(): - unrarred_file = open(os.path.join(extract_dir, urfile)) - f_name, full_path = save_hashed_filename(unrarred_file, True) - tid = queue_task(urfile, f_name, full_path, metadata) - task_id_list.append(tid) - except RuntimeError as e: - msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) - return make_response( - jsonify({'Message': msg}), - HTTP_BAD_REQUEST - ) else: metadata[key] = request.form[key] - if not task_id_list: - # File was not zipped + if extract_dir: + # Extract a zip + if zipfile.is_zipfile(file_): + z = zipfile.ZipFile(file_) + try: + # NOTE: zipfile module prior to Py 2.7.4 is insecure! + # https://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.extract + z.extractall(path=extract_dir, pwd=password) + for uzfile in z.namelist(): + unzipped_file = open(os.path.join(extract_dir, uzfile)) + f_name, full_path = save_hashed_filename(unzipped_file, True) + tid = queue_task(uzfile, f_name, full_path, metadata) + task_id_list.append(tid) + except RuntimeError as e: + msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) + return make_response( + jsonify({'Message': msg}), + HTTP_BAD_REQUEST) + # Extract a rar + elif rarfile.is_rarfile(file_): + r = rarfile.RarFile(file_) + try: + r.extractall(path=extract_dir, pwd=password) + for urfile in r.namelist(): + unrarred_file = open(os.path.join(extract_dir, urfile)) + f_name, full_path = save_hashed_filename(unrarred_file, True) + tid = queue_task(urfile, f_name, full_path, metadata) + task_id_list.append(tid) + except RuntimeError as e: + msg = "ERROR: Failed to extract " + str(file_) + ' - ' + str(e) + return make_response( + jsonify({'Message': msg}), + HTTP_BAD_REQUEST) + else: + # File was not an archive to extract f_name, full_path = save_hashed_filename(file_) tid = queue_task(original_filename, f_name, full_path, metadata) task_id_list = [tid] diff --git a/web/templates/index.html b/web/templates/index.html index 6f530da7..2afc0ef1 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -69,14 +69,19 @@ uploadClass: 'btn btn-xs btn-primary', }, uploadExtraData: function (previewId, index) { + // Send options along with the file var obj = {}; + // Modules options + var moduleList = $("#module-opts input:checked").map(function(){return $(this).attr("name");}); + obj['modules'] = moduleList; + // Archive options if ($('#archive-analyze').is(':checked')) { obj['archive-analyze'] = 'true'; if ($('#archive-password').val() !== '') { obj['archive-password'] = $('#archive-password').val(); } } - // Send metadata along with the file + // Metadata options $('.metadata-group[data-target=' + previewId + '] .form-group-sm').each(function() { var key = $(this).children('label').first().html(); var val = $(this).find('input').first().val(); From 96375a300ab20c991635bc9f79c5c40b1b22bb1c Mon Sep 17 00:00:00 2001 From: clenk Date: Thu, 20 Jul 2017 15:19:07 -0400 Subject: [PATCH 03/50] Hide modules options until we decide how pass them to multiscanner --- utils/api.py | 1 + web/templates/index.html | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/utils/api.py b/utils/api.py index cd8ba090..e06c4688 100755 --- a/utils/api.py +++ b/utils/api.py @@ -352,6 +352,7 @@ def create_task(): metadata = {} task_id_list = [] + extract_dir = None for key in request.form.keys(): if key in ['file_id', 'archive-password'] or request.form[key] == '': continue diff --git a/web/templates/index.html b/web/templates/index.html index 2afc0ef1..4c515a9d 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -133,15 +133,15 @@ }); // Add options for selecting which modules to run - $.get("http://{{ api_loc }}/api/v1/modules", function(data) { - var modules = '

Modules

Select which modules to use:

'; - for (mod in data.Modules) { - checked = (data.Modules[mod] == "True" ? "checked" : ""); - modules += '
'; - } - modules += '
'; - $('#adv-options').prepend(modules); - }); + //$.get("http://{{ api_loc }}/api/v1/modules", function(data) { + // var modules = '

Modules

Select which modules to use:

'; + // for (mod in data.Modules) { + // checked = (data.Modules[mod] == "True" ? "checked" : ""); + // modules += '
'; + // } + // modules += '
'; + // $('#adv-options').prepend(modules); + //}); // Set metadata field values for all files $('#adv-options .metadata-group input').change(function() { From c81f890ddf1c3e63705f59986bb9770e7dc65061 Mon Sep 17 00:00:00 2001 From: clenk Date: Thu, 20 Jul 2017 16:56:25 -0400 Subject: [PATCH 04/50] Store but don't index imports, exports, or sections in pefile output to prevent "Limit of total fields ... exceeded" error --- storage/elasticsearch_storage.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/storage/elasticsearch_storage.py b/storage/elasticsearch_storage.py index 8724e8bf..383a727a 100644 --- a/storage/elasticsearch_storage.py +++ b/storage/elasticsearch_storage.py @@ -54,6 +54,24 @@ def setup(self): es_indices.put_mapping(index=self.index, doc_type=self.doc_type, body={ '_parent': { 'type': 'sample' + }, + 'properties': { + 'pefile': { + 'properties': { + 'imports': { + 'type': 'object', + 'dynamic': 'false', + }, + 'exports': { + 'type': 'object', + 'dynamic': 'false', + }, + 'sections': { + 'type': 'object', + 'dynamic': 'false', + }, + } + } } }) if 'note' not in mappings: @@ -232,13 +250,11 @@ def search(self, query_string, search_type='default'): '''Run a Query String query and return a list of sample_ids associated with the matches. Run the query against all document types. ''' - print(search_type) if search_type == 'advanced': query = self.build_query(query_string) else: es_reserved_chars_re = '([\+\-=\>\<\!\(\)\{\}\[\]\^\"\~\*\?\:\\/ ])' query_string = re.sub(es_reserved_chars_re, r'\\\g<1>', query_string) - print(query_string) if search_type == 'default': query = self.build_query("*" + query_string + "*") elif search_type == 'exact': From 7feb941a918845ea806b87849af1cd7f9791b52e Mon Sep 17 00:00:00 2001 From: clenk Date: Mon, 24 Jul 2017 09:27:07 -0400 Subject: [PATCH 05/50] Add button to download report JSON --- utils/api.py | 10 +++++++++- web/static/css/styles.css | 8 ++++++++ web/templates/report.html | 8 +++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/utils/api.py b/utils/api.py index e06c4688..84bec462 100755 --- a/utils/api.py +++ b/utils/api.py @@ -438,6 +438,8 @@ def get_report(task_id): if not task: abort(HTTP_NOT_FOUND) + download = request.args.get('d', None) + if task.task_status == 'Complete': report = handler.get_report(task.sample_id, task.timestamp) elif task.task_status == 'Pending': @@ -445,7 +447,13 @@ def get_report(task_id): else: report = {'Report': 'Task failed'} - return jsonify({'Report': report}) + if download == 't': + response = make_response(jsonify(report)) + response.headers['Content-Type'] = 'application/json' + response.headers['Content-Disposition'] = 'attachemnt' + return response + else: + return jsonify({'Report': report}) @app.route('/api/v1/tasks/delete/', methods=['GET']) diff --git a/web/static/css/styles.css b/web/static/css/styles.css index e5ce5527..735a6822 100644 --- a/web/static/css/styles.css +++ b/web/static/css/styles.css @@ -257,6 +257,14 @@ background-color: #428bca; } +#dl-json { + position: fixed; + top: 55px; + right: 0; + border-radius: 20px 0 0 20px; + padding-left: 20px; +} + #report-notes { position: fixed; z-index: 1; diff --git a/web/templates/report.html b/web/templates/report.html index 55c0f0cf..efbb953c 100644 --- a/web/templates/report.html +++ b/web/templates/report.html @@ -272,6 +272,11 @@ var noteId = $(this).parents('.panel').attr('id'); var response = deleteNote(noteId); })); + + // Handle downloading of JSON report + $('#dl-json').on('click', (function(event) { + window.location = "http://{{ api_loc }}/api/v1/tasks/report/{{ task_id }}?d=t"; + })); }); {% endblock %} @@ -279,7 +284,8 @@ {% block title %}Report{% endblock %} {% block content %} - + +

Task {{ task_id }} Report

From 01ea558e68a6abaf5eeb98fa3892e90695d5dd3b Mon Sep 17 00:00:00 2001 From: clenk Date: Mon, 24 Jul 2017 18:51:38 -0400 Subject: [PATCH 06/50] Add ability to import JSON reports --- storage/sql_driver.py | 3 ++- utils/api.py | 26 ++++++++++++++++++++++++++ web/templates/index.html | 21 +++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/storage/sql_driver.py b/storage/sql_driver.py index bf1595d4..dcbf4217 100644 --- a/storage/sql_driver.py +++ b/storage/sql_driver.py @@ -153,12 +153,13 @@ def db_session_scope(self): finally: ses.close() - def add_task(self, task_id=None, task_status='Pending', sample_id=None): + def add_task(self, task_id=None, task_status='Pending', sample_id=None, timestamp=None): with self.db_session_scope() as ses: task = Task( task_id=task_id, task_status=task_status, sample_id=sample_id, + timestamp=timestamp, ) try: ses.add(task) diff --git a/utils/api.py b/utils/api.py index 84bec462..e27f6860 100755 --- a/utils/api.py +++ b/utils/api.py @@ -29,6 +29,7 @@ import hashlib import codecs import configparser +import json import multiprocessing import queue import shutil @@ -320,6 +321,23 @@ def save_hashed_filename(f, zipped=False): return (f_name, full_path) +def import_task(file_): + ''' + Import a JSON report that was downloaded from MultiScanner. + ''' + report = json.loads(file_.read().decode('utf-8')) + report['Scan Time'] = datetime.strptime(report['Scan Time'], '%Y-%m-%dT%H:%M:%S.%f') + + task_id = db.add_task( + sample_id=report['SHA256'], + task_status='Complete', + timestamp=report['Scan Time'], + ) + storage_handler.store({report['filename']: report}, wait=False) + + return task_id + + def queue_task(original_filename, f_name, full_path, metadata): ''' Queue up a single new task, for a single non-archive file. @@ -348,6 +366,14 @@ def create_task(): UPLOAD_FOLDER, optionally unzipping it. Return task id and 201 status. ''' file_ = request.files['file'] + if request.form['upload_type'] == 'import': + task_id = import_task(file_) + + return make_response( + jsonify({'Message': {'task_ids': [task_id]}}), + HTTP_CREATED + ) + original_filename = file_.filename metadata = {} diff --git a/web/templates/index.html b/web/templates/index.html index 4c515a9d..6d0b922b 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -71,6 +71,9 @@ uploadExtraData: function (previewId, index) { // Send options along with the file var obj = {}; + //Scan or Import? + var upload_type = $('#upload-opts .btn.active input').val(); + obj['upload_type'] = upload_type; // Modules options var moduleList = $("#module-opts input:checked").map(function(){return $(this).attr("name");}); obj['modules'] = moduleList; @@ -132,6 +135,14 @@ }); }); + // Bootstrap Toggle Switch (https://www.bootply.com/92189) + $('.btn-toggle').click(function() { + $(this).find('.btn').toggleClass('active'); + $(this).find('.btn').toggleClass('btn-primary'); + $(this).find('.btn').toggleClass('btn-default'); + $(this).find('.btn').prop('checked', !$(this).find('.btn').prop('checked')); + }); + // Add options for selecting which modules to run //$.get("http://{{ api_loc }}/api/v1/modules", function(data) { // var modules = '

Modules

Select which modules to use:

'; @@ -187,6 +198,16 @@

MultiScanner

+
+
+ + +
+

Archives

From f0d3a526e497ee67d9ee03a48254ef712894d1a3 Mon Sep 17 00:00:00 2001 From: awest1339 Date: Wed, 9 Aug 2017 10:47:28 -0600 Subject: [PATCH 07/50] Change iteritems() to items() for python 2+3 --- modules/Antivirus/Metadefender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Antivirus/Metadefender.py b/modules/Antivirus/Metadefender.py index 9f2d15aa..5d3fddea 100644 --- a/modules/Antivirus/Metadefender.py +++ b/modules/Antivirus/Metadefender.py @@ -118,7 +118,7 @@ def _parse_scan_result(response): overall_results = response_json.get("scan_results", {}) scan_details = overall_results.get("scan_details", {}) engine_results = [] - for engine_name, engine_output in scan_details.iteritems(): + for engine_name, engine_output in scan_details.items(): scan_code = engine_output.get("scan_result_i", MD_UNKNOWN_SCAN_RES) scan_result_string = MD_SCAN_RES_CODES[scan_code] engine_result = {'engine_name': engine_name, From 80537dab1faa5987938dd6ba011b24a2d1be16c0 Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 9 Aug 2017 15:31:14 -0400 Subject: [PATCH 08/50] Add icon to signify collapsible report content --- web/templates/report.html | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/web/templates/report.html b/web/templates/report.html index efbb953c..52723045 100644 --- a/web/templates/report.html +++ b/web/templates/report.html @@ -16,21 +16,19 @@ } if (typeof content === 'string' && content.startsWith('' + - ''; + th = '' + headText + + ' '; + td = 'Expand' + + ' ' + + '' + + content + ''; } else { - th = ''; - td = ''; + th = '' + headText + ''; + td = '' + content + ''; } - return '' + - th + headText + '' + - td + content + '' + - ''; + return '' + th + td + ''; } function createSubtable(data, isArray) { @@ -146,7 +144,7 @@ return; } else if (data.Report.hasOwnProperty('Report') && data.Report.Report == 'Task failed') { - $('#pending').html('


Scan Failed!

'); + $('#pending').html('


Scan Failed!

'); return; } $('#pending').hide(); @@ -162,9 +160,13 @@ $('.expander').each(function(idx, elem) { $(elem).click(function() { $(this).siblings('td').toggle(); + $(this).siblings('th').children('span').toggle(); if ($(this).is('td')) { $(this).hide(); } + if ($(this).is('th')) { + $(this).children('span').toggle(); + } }); }); From 1ca8bbd1d5bb71b0c52f22a5afffe04fdb757bb7 Mon Sep 17 00:00:00 2001 From: clenk Date: Wed, 9 Aug 2017 16:44:33 -0400 Subject: [PATCH 09/50] If collapsed data is 5 lines or less expand by default --- web/templates/report.html | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/web/templates/report.html b/web/templates/report.html index 52723045..4620d28f 100644 --- a/web/templates/report.html +++ b/web/templates/report.html @@ -16,11 +16,19 @@ } if (typeof content === 'string' && content.startsWith('/g) || []).length > 5) { + expanderStyle = '' + contentStyle = 'style="display: none;"' + } + else { + expanderStyle = 'style="display: none;"' + contentStyle = '' + } th = '' + headText + - ' '; - td = 'Expand' + + ' '; + td = 'Expand' + ' ' + - '' + + '' + content + ''; } else { From 5d35aa7e759d77528ddee1d897e66c2d306c036d Mon Sep 17 00:00:00 2001 From: clenk Date: Thu, 10 Aug 2017 10:32:59 -0400 Subject: [PATCH 10/50] Improve JSON report importing --- utils/api.py | 7 ++++++- web/templates/index.html | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/utils/api.py b/utils/api.py index e27f6860..8a25c1a8 100755 --- a/utils/api.py +++ b/utils/api.py @@ -367,7 +367,12 @@ def create_task(): ''' file_ = request.files['file'] if request.form['upload_type'] == 'import': - task_id = import_task(file_) + try: + task_id = import_task(file_) + except (UnicodeDecodeError, ValueError): + return make_response( + jsonify({'Message': 'Cannot import non-JSON files!'}), + HTTP_BAD_REQUEST) return make_response( jsonify({'Message': {'task_ids': [task_id]}}), diff --git a/web/templates/index.html b/web/templates/index.html index 6d0b922b..4c1fccdc 100644 --- a/web/templates/index.html +++ b/web/templates/index.html @@ -4,6 +4,23 @@ From c158f82930d86886d293336ab609cf6bfd486e77 Mon Sep 17 00:00:00 2001 From: Austin West Date: Thu, 10 Aug 2017 17:27:40 -0400 Subject: [PATCH 21/50] Reduce limit for rows to autoexpand --- web/templates/report.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/templates/report.html b/web/templates/report.html index d8e89b0b..c36e98b5 100644 --- a/web/templates/report.html +++ b/web/templates/report.html @@ -16,7 +16,7 @@ } if (typeof content === 'string' && content.startsWith('/g) || []).length > 10) { + if ((content.match(//g) || []).length > 3) { expanderStyle = '' contentStyle = 'style="display: none;"' } From 7161bf379b4ff5e24247602112bed43165e2ab9a Mon Sep 17 00:00:00 2001 From: Austin West Date: Thu, 10 Aug 2017 18:25:12 -0400 Subject: [PATCH 22/50] Add worker node to metadata --- utils/celery_worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/celery_worker.py b/utils/celery_worker.py index 1d41295a..8c80b532 100644 --- a/utils/celery_worker.py +++ b/utils/celery_worker.py @@ -9,6 +9,7 @@ import codecs import configparser from datetime import datetime +from socket import gethostname MS_WD = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Append .. to sys path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -84,6 +85,7 @@ def celery_task(files, config=multiscanner.CONFIG): task_id = files[file_]['task_id'] file_hash = files[file_]['file_hash'] metadata = files[file_]['metadata'] + metadata['Worker Node'] = gethostname() # Use the original filename as the value for the filename # in the report (instead of the tmp path assigned to the file From fe32b31a6dcd18950ec03ed445ed8b3ae710bb1e Mon Sep 17 00:00:00 2001 From: clenk Date: Fri, 11 Aug 2017 14:56:09 -0400 Subject: [PATCH 23/50] Don't include upload type (scan/import) in report --- utils/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/api.py b/utils/api.py index a0a7e321..74b18074 100755 --- a/utils/api.py +++ b/utils/api.py @@ -385,7 +385,7 @@ def create_task(): task_id_list = [] extract_dir = None for key in request.form.keys(): - if key in ['file_id', 'archive-password'] or request.form[key] == '': + if key in ['file_id', 'archive-password', 'upload_type'] or request.form[key] == '': continue elif key == 'modules': module_names = request.form[key] From 185276682ead8eb582330d2281c76ccf7df7516d Mon Sep 17 00:00:00 2001 From: awest1339 Date: Fri, 11 Aug 2017 17:06:31 -0600 Subject: [PATCH 24/50] Add scan config to metadata --- utils/celery_worker.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/utils/celery_worker.py b/utils/celery_worker.py index 8c80b532..6e113b4a 100644 --- a/utils/celery_worker.py +++ b/utils/celery_worker.py @@ -86,6 +86,19 @@ def celery_task(files, config=multiscanner.CONFIG): file_hash = files[file_]['file_hash'] metadata = files[file_]['metadata'] metadata['Worker Node'] = gethostname() + # Get the Scan Config that the task was run with and + # add it to the task metadata + scan_config_object = configparser.SafeConfigParser() + scan_config_object.optionxform = str + scan_config_object.read(config) + full_conf = common.parse_config(scan_config_object) + sub_conf = {} + for key in full_conf: + if key == 'main': + continue + sub_conf[key] = {} + sub_conf[key]['ENABLED'] = full_conf[key]['ENABLED'] + metadata['Scan Config'] = sub_conf # Use the original filename as the value for the filename # in the report (instead of the tmp path assigned to the file From d50bc36449986e742f094f318c499654c68b6e16 Mon Sep 17 00:00:00 2001 From: clenk Date: Mon, 14 Aug 2017 09:59:08 -0400 Subject: [PATCH 25/50] Only expand items if they have <10 siblings --- web/templates/report.html | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/web/templates/report.html b/web/templates/report.html index c36e98b5..4c93c891 100644 --- a/web/templates/report.html +++ b/web/templates/report.html @@ -5,26 +5,25 @@ + {% block head %}{% endblock %} - @@ -54,7 +90,7 @@
-