From b88dda88e10ab4975737361bd1f4e3ca9257a387 Mon Sep 17 00:00:00 2001 From: Andy Boughton Date: Fri, 17 Dec 2021 13:58:07 -0500 Subject: [PATCH 1/5] Use newer parser for ingest pipeline --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 346768b..eec292f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -25,5 +25,5 @@ pyyaml # Required for OpenAPI rendering boltons~=20.2.1 scipy~=1.5.3 python-magic==0.4.18 -zorp[perf,lookups]==0.3.3 +zorp[perf,lookups]==0.3.4 genelocator==1.1.2 From 8f225af6db666c1fa6c725b9d9e43660736e575a Mon Sep 17 00:00:00 2001 From: Andy Boughton Date: Mon, 10 Jan 2022 15:58:18 -0500 Subject: [PATCH 2/5] Improve error messages: when files are unsorted, point to the position of the first unsorted chrom or pos Also, remove SHA message on upload failure page, due to limits of how the data is stored at present. (if ingest fails, metadata.files is unset, so there was no SHA to show) --- locuszoom_plotting_service/templates/gwas/gwas_summary.html | 5 +++-- util/ingest/validators.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/locuszoom_plotting_service/templates/gwas/gwas_summary.html b/locuszoom_plotting_service/templates/gwas/gwas_summary.html index 0622538..7f1450f 100644 --- a/locuszoom_plotting_service/templates/gwas/gwas_summary.html +++ b/locuszoom_plotting_service/templates/gwas/gwas_summary.html @@ -142,8 +142,9 @@

QQ Plot:

{% else %}

We were unable to process your uploaded GWAS file. Please review the logs for the cause, and contact us if you believe the problem is an error.

-

You can verify that you uploaded the correct file by comparing the checksum (SHA256):
- {{ gwas.files.file_sha256.hex }}

+ {# FIXME: At present, metadata.files is only set at conclusion of a successful upload (tasks.py). This makes it difficult to show the SHA, which lives in the associated "this is the file for a study" record. #} + {#

You can verify that you uploaded the correct file by comparing the checksum (SHA256):
#} + {# {{ gwas.files.file_sha256.hex }}

#} {% if request.user == gwas.owner %}
{% csrf_token %} diff --git a/util/ingest/validators.py b/util/ingest/validators.py index 0d40637..30c6362 100644 --- a/util/ingest/validators.py +++ b/util/ingest/validators.py @@ -57,11 +57,11 @@ def _validate_data_rows(self, reader) -> bool: if cur_chrom == prev_chrom and cp[1] < prev_pos: # Positions not in correct order for Pheweb to use - raise v_exc.ValidationException('Positions must be sorted prior to uploading') + raise v_exc.ValidationException(f'Positions must be sorted prior to uploading. Position chr{cur_chrom}:{cp[1]} should not follow chr{prev_chrom}:{prev_pos}') if cur_chrom != prev_chrom: if cur_chrom in chrom_seen: - raise v_exc.ValidationException('Chromosomes must be sorted (so that all variants for the same chromosome are contiguous)') # noqa + raise v_exc.ValidationException(f'Chromosomes must be sorted (so that all variants for the same chromosome are contiguous). Error at position: chr{cur_chrom}:{cp[1]}') # noqa else: chrom_seen.add(cur_chrom) From c5ea76b859e218cdaebfc81c725ad6545f57906d Mon Sep 17 00:00:00 2001 From: Andy Boughton Date: Mon, 10 Jan 2022 16:37:30 -0500 Subject: [PATCH 3/5] Correctly track metrics for "add track" button --- assets/vue/gwas_region.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/assets/vue/gwas_region.vue b/assets/vue/gwas_region.vue index 7bae110..6e9dcbd 100644 --- a/assets/vue/gwas_region.vue +++ b/assets/vue/gwas_region.vue @@ -95,6 +95,8 @@ } } + count_add_track(data_type); + this.known_tracks.push({data_type, filename, display_name}); }, }, components: { BatchSpec, BatchScroller, GwasToolbar, PlotPanes, RegionPicker } From c96d6689eb67236424fcec6756b068f742886cf5 Mon Sep 17 00:00:00 2001 From: Andy Boughton Date: Mon, 10 Jan 2022 16:42:56 -0500 Subject: [PATCH 4/5] Increase max upload size from 1.0 -> 1.25 GB for testing on the staging server --- assets/js/pages/gwas_upload.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/js/pages/gwas_upload.js b/assets/js/pages/gwas_upload.js index 5769c35..c825a62 100644 --- a/assets/js/pages/gwas_upload.js +++ b/assets/js/pages/gwas_upload.js @@ -13,7 +13,7 @@ import { _isHeader } from 'locuszoom/esm/ext/lz-parsers/gwas/sniffers'; import { makeGWASParser } from 'locuszoom/esm/ext/lz-parsers'; const PREVIEW_BYTES = 5000; // enough for 50-100 lines -const MAX_UPLOAD_SIZE = 1048576 * 1000; // 1000 MiB # FIXME: Currently upload limit is only implemented in the frontend +const MAX_UPLOAD_SIZE = 1048576 * 1250; // 1250 MiB # FIXME: Currently upload limit is only implemented in the frontend class BaseReader { From edf33ae827de7d783c33cea9844fa7892d652705 Mon Sep 17 00:00:00 2001 From: Andy Boughton Date: Mon, 10 Jan 2022 18:34:59 -0500 Subject: [PATCH 5/5] Fix build command to respect lockfile + add correct import --- assets/vue/gwas_region.vue | 2 +- compose/production/django/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assets/vue/gwas_region.vue b/assets/vue/gwas_region.vue index 6e9dcbd..c1fdbb6 100644 --- a/assets/vue/gwas_region.vue +++ b/assets/vue/gwas_region.vue @@ -8,7 +8,7 @@ import RegionPicker from 'localzoom/src/components/RegionPicker.vue'; import { DATA_TYPES } from 'localzoom/src/util/constants'; import { activateUserLD } from 'localzoom/src/util/lz-helpers'; - import {setup_feature_metrics} from 'localzoom/src/util/metrics'; + import {count_add_track, setup_feature_metrics} from 'localzoom/src/util/metrics'; const MAX_REGION_SIZE = 1000000; diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index 89a2882..8e261f0 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -6,7 +6,7 @@ FROM node:fermium as jsbuilder COPY ./package.json ./package-lock.json /build/ COPY . /build/ WORKDIR /build/ -RUN npm install ci && npm run prod +RUN npm ci && npm run prod # Step 2 (main): Django app. Install python dependencies, add volume mounts, and run.