diff --git a/.envs/.local/.django b/.envs/.local/.django index 8a8458d..72c3a7d 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -21,10 +21,11 @@ LZ_OFFICIAL_DOMAIN=localhost:8000 # ------------------------------------------------------------------------------ GOOGLE_ANALYTICS_ID= -# Sentry uses two buckets: one for python errors (backend) and one for JS (frontend) +# Our Sentry configuration uses two buckets: one for python errors (backend) and one for JS (frontend) # ------------------------------------------------------------------------------ SENTRY_DSN= SENTRY_DSN_FRONTEND= # Set the location of download large lookup files (which are downloaded separately, after the build step) +# This path is relative to the docker container ZORP_ASSETS_DIR=/app/.lookups diff --git a/.envs/.production/.django-sample b/.envs/.production/.django-sample new file mode 100644 index 0000000..2e0b5d3 --- /dev/null +++ b/.envs/.production/.django-sample @@ -0,0 +1,60 @@ +##### Example file. Rename to .django to use in production + +# General +# ------------------------------------------------------------------------------ +# DJANGO_READ_DOT_ENV_FILE=True +DJANGO_SETTINGS_MODULE=config.settings.production +DJANGO_SECRET_KEY= +# The admin site has restricted functionality, but make it hard for bots to find. In the future this could be served separately behind a VPN +DJANGO_ADMIN_URL=admin-changeme-something-very-hard-to-guess/ +# The localhost entries work if we are using a reverse proxy +DJANGO_ALLOWED_HOSTS=.my.locuszoom.org,localhost,localhost:5000 +LZ_OFFICIAL_DOMAIN=my.locuszoom.org + +# Security +# ------------------------------------------------------------------------------ +# TIP: It is better to handle the redirect via the server. However, django can handle the redirect if needed. +DJANGO_SECURE_SSL_REDIRECT=False + +# Email +# ------------------------------------------------------------------------------ +DJANGO_SERVER_EMAIL=locuszoom-noreply@host.example + +DJANGO_EMAIL_HOST=smtp.host.example +DJANGO_EMAIL_HOST_USER=locuszoom-noreply +DJANGO_EMAIL_HOST_PASSWORD= + +# django-allauth +# ------------------------------------------------------------------------------ +# This can be disabling if, eg, you are running a private instance and don't want random people to upload things +DJANGO_ACCOUNT_ALLOW_REGISTRATION=True + +# Gunicorn +# ------------------------------------------------------------------------------ +# Recommend 2-4x # cores +WEB_CONCURRENCY=8 + +# Our Sentry configuration uses two buckets: one for python errors (backend) and one for JS (frontend) +# ------------------------------------------------------------------------------ +SENTRY_DSN= +SENTRY_DSN_FRONTEND= + +# Google Analytics +# ------------------------------------------------------------------------------ +GOOGLE_ANALYTICS_ID= + +# Redis +# ------------------------------------------------------------------------------ +REDIS_URL=redis://redis:6379/0 + +# Celery +# ------------------------------------------------------------------------------ + +# Flower +## Set these to very hard to guess values +CELERY_FLOWER_USER= +CELERY_FLOWER_PASSWORD= +CELERY_WORKERS=12 + +# For now, re-use the local "uploads" mount point as a place to store the large lookup files required by annotations +ZORP_ASSETS_DIR=/lz-uploads/.lookups diff --git a/.envs/.production/.postgres-sample b/.envs/.production/.postgres-sample new file mode 100644 index 0000000..34f9447 --- /dev/null +++ b/.envs/.production/.postgres-sample @@ -0,0 +1,8 @@ +# PostgreSQL +# ------------------------------------------------------------------------------ +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=locuszoom_plotting_service +# Please make sure to set these. +POSTGRES_USER= +POSTGRES_PASSWORD= diff --git a/.gitignore b/.gitignore index 92fcfd2..306415e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,7 @@ celerybeat.pid staticfiles/ # Webpack built assets (and files that reference them) -assets/webpack_bundles/ -webpack-stats.json +locuszoom_plotting_service/static/webpack_bundles/ # Sphinx documentation docs/_build/ @@ -43,3 +42,7 @@ scripts/data_loaders/sources/* # Configuration files that should not be checked into Git .envs/* !.envs/.local/ +# Exclude production envs, EXCEPT samples +!.envs/.production/ +.envs/.production/* +!.envs/.production/*sample diff --git a/.nvmrc b/.nvmrc index e1fcd1e..518633e 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -lts/erbium +lts/fermium diff --git a/README.md b/README.md index 7b25243..b5b3ae1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # LocusZoom: Hosted Upload Service -Upload and share GWAS results with LocusZoom.js +Upload, analyze, and share GWAS results with LocusZoom.js. Try it at [my.locuszoom.org](https://my.locuszoom.org). ## Settings @@ -11,10 +11,11 @@ For a basic guide to most settings, see the [cookiecutter-django docs](https://c ### Quickstart -The following commands will start a development environment. Some IDEs (such as Pycharm) are able to run the app via - run configurations, which may be more convenient than starting things through the terminal. +The following commands will start a development environment. Some IDEs (such as Pycharm) are able to run the app via run configurations, which may be more convenient than starting things through the terminal. + +For production deployment instructions, see [docs/deploy/index.md](docs/deploy/index.md). -- In one tab, build assets: +- In one tab, build assets (in local development, this is currently done on the host system, outside of Docker): `$ yarn run prod` @@ -25,12 +26,15 @@ or with live rebuilding, if you intend to be changing JS code as you work: - In a second open terminal:: ``` -$ docker-compose -f local.yml build +$ docker system prune && docker-compose -f local.yml build --pull $ docker-compose -f local.yml up ``` -On the first installation, you will also need to download some large asset files required for annotations. (see -deployment docs for the correct command to use with production assets) +(`docker system prune` is optional, but it can save your hard drive from filling up as you experiment with different build options) + +On the first installation, you will also need to download some large asset files required for annotations. For local development, a "test" version is available that will only annotate a limited subset of biologically interesting genes; this subset is much smaller than the full database, and easier to use on a laptop. + +(see deployment docs for the correct command to use with production assets) ```bash $ docker-compose -f local.yml run --rm django zorp-assets download --type snp_to_rsid_test --tag genome_build GRCh37 --no-update @@ -63,8 +67,7 @@ into your browser. Now the user's email should be verified and ready to go. `$ docker-compose -f local.yml run --rm django python manage.py makemigrations` -Then verify the migration file is correct, and restart docker to apply the migrations automatically. -(in production, you must apply the migrations manually; see deployment guide for details) +Then verify the migration file is correct, and restart Docker to apply the migrations automatically. (in production, you must apply the migrations manually; see deployment guide for details) ## Development and testing helpers @@ -78,7 +81,7 @@ This script generates fake studies for search results, but it notably does not r It may be improved in the future to generate more realistic and complete fake data. ### Opening a terminal for debugging -Because all development happens inside a docker container, it is sometimes useful to open a terminal for debugging +Because all development happens inside a Docker container, it is sometimes useful to open a terminal for debugging purposes. This can be done as follows. On a running container:: @@ -114,55 +117,18 @@ A suite of unit tests is available:: `$ docker-compose -f local.yml run --rm django pytest` -## Celery - -This app comes with Celery. The docker configuration will automatically launch celery workers when the app starts, but -the commands below may be useful when running the app in other environments. - -To run a celery worker: - -```bash -$ cd locuszoom_plotting_service -$ celery -A locuszoom_plotting_service.taskapp worker -l info -``` - -Please note: For Celery's import magic to work, it is important *where* the celery commands are run. If you are in the -same folder with *manage.py*, you should be ok. - - ## Sentry Sentry is an error logging aggregator service. If a key (DSN) is provided in your .env file, errors will be tracked - automatically. You will need one DSN each for your frontend (JS) and backend (python) code. + automatically. ## Deployment ### Docker -This app uses docker to manage dependencies and create a working environment. See +This app uses Docker to manage dependencies and create a working environment. See [deployment documentation](docs/deploy/index.md) for instructions on how to create a working, production server -environment. A local, debug-friendly docker configuration is also provided in this repo, and many of the instructions +environment. A local, debug-friendly Docker configuration is also provided in this repo, and many of the instructions in this document assume this is what you will use. - The original docker configuration has been modified from cookiecutter-django; see their docs for more information + The original Docker configuration has been modified from cookiecutter-django; see their docs for more information about default options and design choices. - - -### (future) Initializing the app with default data - -Certain app features, such as "tagging datasets", will require loading initial data into the database. - -This feature is not yet used in production, but the notes below demonstrate loader scripts in progress. - -These datasets may be large or restricted by licensing rules; as such, they are not distributed with the code and must -be downloaded/reprocessed separately for loading. - -- [SNOMED CT (Core) / May 2019](https://www.nlm.nih.gov/research/umls/Snomed/core_subset.html) - -These files must be downloaded separately due to license issues (they cannot be distributed with this repo). -Run the appropriate scripts in `scripts/data_loaders/` to transform them into a format suitable for django usage. - -After creating the app, run the following command (once) to load them in (using the appropriate docker-compose file):: - -`$ docker-compose -f local.yml run --rm django python3 manage.py loaddata scripts/data_loaders/sources/snomed.json` - -[![Built with Cookiecutter Django](https://img.shields.io/badge/built%20with-Cookiecutter%20Django-ff69b4.svg)](https://github.com/pydanny/cookiecutter-django/) diff --git a/assets/js/library.js b/assets/js/library.js deleted file mode 100644 index e69de29..0000000 diff --git a/assets/js/pages/gwas_summary.js b/assets/js/pages/gwas_summary.js index b534ee8..c797b56 100644 --- a/assets/js/pages/gwas_summary.js +++ b/assets/js/pages/gwas_summary.js @@ -6,7 +6,7 @@ import {create_qq_plot, create_gwas_plot} from '../util/pheweb_plots'; import Tabulator from 'tabulator-tables'; import 'tabulator-tables/dist/css/bootstrap/tabulator_bootstrap4.css'; -import { pairs, sortBy } from 'underscore'; +import { toPairs, sortBy } from 'lodash'; function createTopHitsTable(selector, data, region_url) { // Filter the manhattan json to a subset of just peaks, largest -log10p first @@ -95,7 +95,7 @@ if (window.template_args.ingest_status === 2) { return resp.json(); }) .then(data => { - sortBy(pairs(data.overall.gc_lambda), function (d) { + sortBy(toPairs(data.overall.gc_lambda), function (d) { return -d[0]; }).forEach(function (d, i) { // FIXME: Manually constructed HTML; change @@ -111,7 +111,8 @@ if (window.template_args.ingest_status === 2) { } else { create_qq_plot([{ maf_range: [0, 0.5], qq: data.overall.qq, count: data.overall.count }], data.ci); } - }).catch(() => { + }).catch((e) => { + console.error(e); document.getElementById('qq_plot_container').textContent = 'Could not fetch QQ plot data.'; }); }); diff --git a/assets/js/util/pheweb_plots.js b/assets/js/util/pheweb_plots.js index 20ec8b2..4a3944f 100644 --- a/assets/js/util/pheweb_plots.js +++ b/assets/js/util/pheweb_plots.js @@ -4,9 +4,11 @@ * TODO: Replace this with a manhattan plot implementation in LZjs Core */ -/* global $, d3 */ -// The specified d3.tip version does not work well with modules; revisit -import _ from 'underscore'; +/* global $ */ + +import * as d3 from 'd3'; +import d3Tip from 'd3-tip'; +import {memoize, property, range, some, sortBy, template} from 'lodash'; // NOTE: `qval` means `-log10(pvalue)`. function fmt(format) { @@ -21,9 +23,9 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t // FIXME: Replace global variables with options object // Order from weakest to strongest pvalue, so that the strongest variant will be on top (z-order) and easily hoverable // In the DOM, later siblings are displayed over top of (and occluding) earlier siblings. - unbinned_variants = _.sortBy(unbinned_variants, function(d) { return d.neg_log_pvalue; }); + unbinned_variants = sortBy(unbinned_variants, function(d) { return d.neg_log_pvalue; }); - const get_chrom_offsets = _.memoize(function() { + const get_chrom_offsets = memoize(function() { const chrom_padding = 2e7; const chrom_extents = {}; @@ -39,7 +41,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t variant_bins.forEach(update_chrom_extents); unbinned_variants.forEach(update_chrom_extents); - const chroms = _.sortBy(Object.keys(chrom_extents), parseInt); + const chroms = sortBy(Object.keys(chrom_extents), parseInt); const chrom_genomic_start_positions = {}; chrom_genomic_start_positions[chroms[0]] = 0; @@ -70,11 +72,11 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t function get_y_axis_config(max_data_qval, plot_height, includes_pval0) { let possible_ticks = []; - if (max_data_qval <= 14) { possible_ticks = _.range(0, 14.1, 2); } - else if (max_data_qval <= 28) { possible_ticks = _.range(0, 28.1, 4); } - else if (max_data_qval <= 40) { possible_ticks = _.range(0, 40.1, 8); } + if (max_data_qval <= 14) { possible_ticks = range(0, 14.1, 2); } + else if (max_data_qval <= 28) { possible_ticks = range(0, 28.1, 4); } + else if (max_data_qval <= 40) { possible_ticks = range(0, 40.1, 8); } else { - possible_ticks = _.range(0, 20.1, 4); + possible_ticks = range(0, 20.1, 4); if (max_data_qval <= 70) { possible_ticks = possible_ticks.concat([30,40,50,60,70]); } else if (max_data_qval <= 120) { possible_ticks = possible_ticks.concat([40,60,80,100,120]); } else if (max_data_qval <= 220) { possible_ticks = possible_ticks.concat([60,100,140,180,220]); } @@ -97,7 +99,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t let max_plot_qval = ticks[ticks.length - 1]; // If we have any qval=inf (pval=0) variants, leave space for them. if (includes_pval0) { max_plot_qval *= 1.1; } - let scale = d3.scale.linear().clamp(true); + let scale = d3.scaleLinear().clamp(true); if (max_plot_qval <= 40) { scale = scale .domain([max_plot_qval, 0]) @@ -141,7 +143,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t // Significance Threshold line const significance_threshold = 5e-8; - const significance_threshold_tooltip = d3.tip() + const significance_threshold_tooltip = d3Tip() .attr('class', 'd3-tip') .html('Significance Threshold: 5E-8') .offset([-8,0]); @@ -153,11 +155,11 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t return d3.extent(extent1.concat(extent2)); })(); - const x_scale = d3.scale.linear() + const x_scale = d3.scaleLinear() .domain(genomic_position_extent) .range([0, plot_width]); - const includes_pval0 = _.any(unbinned_variants, function(variant) { return variant.pvalue === 0; }); + const includes_pval0 = some(unbinned_variants, function(variant) { return variant.pvalue === 0; }); const highest_plot_qval = Math.max( -Math.log10(significance_threshold) + 0.5, @@ -167,7 +169,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t })); if (best_unbinned_qval !== undefined) {return best_unbinned_qval;} return d3.max(variant_bins, function(bin) { - return d3.max(bin, _.property('qval')); + return d3.max(bin, property('qval')); }); })()); @@ -175,9 +177,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t const y_scale = y_axis_config.scale; // TODO: draw a small y-axis-break at 20 if `y_axis_config.draw_break_at_20` - const y_axis = d3.svg.axis() - .scale(y_scale) - .orient('left') + const y_axis = d3.axisLeft(y_scale) .tickFormat(d3.format('d')) .tickValues(y_axis_config.ticks); gwas_plot.append('g') @@ -217,7 +217,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t }); })(); - const color_by_chrom = d3.scale.ordinal() + const color_by_chrom = d3.scaleOrdinal() .domain(get_chrom_offsets().chroms) .range(['#AFAFAF', '#007bff']); @@ -250,8 +250,8 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t .on('mouseout', significance_threshold_tooltip.hide); // Points & labels - const tooltip_template = _.template(window.model.tooltip_underscoretemplate); - const point_tooltip = d3.tip() + const tooltip_template = template(window.model.tooltip_underscoretemplate); + const point_tooltip = d3Tip() .attr('class', 'd3-tip') .html(function(d) { return tooltip_template({d: d}); @@ -260,7 +260,7 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t gwas_svg.call(point_tooltip); function get_link_to_LZ(variant) { - var base = new URL(window.model.urlprefix, window.location.origin); + let base = new URL(window.model.urlprefix, window.location.origin); base.searchParams.set('chrom', variant.chrom); base.searchParams.set('start', Math.max(0, variant.pos - 200 * 1000)); base.searchParams.set('end', variant.pos + 200 * 1000); @@ -363,38 +363,47 @@ function create_gwas_plot(variant_bins, unbinned_variants, {url_prefix = null, t .enter() .append('g') .attr('class', 'bin') + .attr('data-index', function(d, i) { return i; }) // make parent index available from DOM .each(function(d) { //todo: do this in a forEach d.x = x_scale(get_genomic_position(d)); d.color = color_by_chrom(d.chrom); }); bins.selectAll('circle.binned_variant_point') - .data(_.property('qvals')) + .data(property('qvals')) .enter() .append('circle') .attr('class', 'binned_variant_point') - .attr('cx', function(d, i, parent_i) { + .attr('cx', function(d, i) { + const parent_i = +this.parentNode.getAttribute('data-index'); return variant_bins[parent_i].x; }) .attr('cy', function(qval) { return y_scale(qval); }) .attr('r', 2.3) - .style('fill', function(d, i, parent_i) { - // return color_by_chrom(d3.select(this.parentNode).datum().chrom); //slow - // return color_by_chrom(this.parentNode.__data__.chrom); //slow? - // return this.parentNode.__data__.color; + .style('fill', function(d, i) { + const parent_i = +this.parentNode.getAttribute('data-index'); return variant_bins[parent_i].color; }); bins.selectAll('circle.binned_variant_line') - .data(_.property('qval_extents')) + .data(property('qval_extents')) .enter() .append('line') .attr('class', 'binned_variant_line') - .attr('x1', function(d, i, parent_i) { return variant_bins[parent_i].x; }) - .attr('x2', function(d, i, parent_i) { return variant_bins[parent_i].x; }) + .attr('x1', function(d, i) { + const parent_i = +this.parentNode.getAttribute('data-index'); + return variant_bins[parent_i].x; + }) + .attr('x2', function(d, i) { + const parent_i = +this.parentNode.getAttribute('data-index'); + return variant_bins[parent_i].x; + }) .attr('y1', function(d) { return y_scale(d[0]); }) .attr('y2', function(d) { return y_scale(d[1]); }) - .style('stroke', function(d, i, parent_i) { return variant_bins[parent_i].color; }) + .style('stroke', function(d, i) { + const parent_i = +this.parentNode.getAttribute('data-index'); + return variant_bins[parent_i].color; + }) .style('stroke-width', 4.6) .style('stroke-linecap', 'round'); } @@ -456,10 +465,10 @@ function create_qq_plot(maf_ranges, qq_ci) { .attr('id', 'qq_plot') .attr('transform', fmt('translate({0},{1})', plot_margin.left, plot_margin.top)); - const x_scale = d3.scale.linear() + const x_scale = d3.scaleLinear() .domain([0, exp_max]) .range([0, plot_width]); - const y_scale = d3.scale.linear() + const y_scale = d3.scaleLinear() .domain([0, obs_max]) .range([plot_height, 0]); @@ -467,7 +476,7 @@ function create_qq_plot(maf_ranges, qq_ci) { qq_plot.append('path') .attr('class', 'trumpet_ci') .datum(qq_ci) - .attr('d', d3.svg.area() + .attr('d', d3.area() .x( function(d) { return x_scale(d.x); }).y0( function(d) { @@ -483,6 +492,7 @@ function create_qq_plot(maf_ranges, qq_ci) { .data(maf_ranges) .enter() .append('g') + .attr('data-index', function(d, i) { return i; }) // make parent index available from DOM .attr('class', 'qq_points') .selectAll('circle.qq_point') .data(function(maf_range) { return maf_range.qq.bins; }) @@ -491,7 +501,9 @@ function create_qq_plot(maf_ranges, qq_ci) { .attr('cx', function(d) { return x_scale(d[0]); }) .attr('cy', function(d) { return y_scale(d[1]); }) .attr('r', 1.5) - .attr('fill', function (d, i, parent_index) { + .attr('fill', function (d, i) { + // Nested selections, d3 v4 workaround + const parent_index = +this.parentNode.getAttribute('data-index'); return maf_ranges[parent_index].color; }); @@ -526,27 +538,23 @@ function create_qq_plot(maf_ranges, qq_ci) { }); // Axes - const xAxis = d3.svg.axis() - .scale(x_scale) - .orient('bottom') - .innerTickSize(-plot_height) // this approach to a grid is taken from - .outerTickSize(0) + const xAxis = d3.axisBottom(x_scale) + .tickSizeInner(-plot_height) // this approach to a grid is taken from + .tickSizeOuter(0) .tickPadding(7) .tickFormat(d3.format('d')) //integers - .tickValues(_.range(exp_max)); //prevent unlabeled, non-integer ticks. + .tickValues(range(exp_max)); //prevent unlabeled, non-integer ticks. qq_plot.append('g') .attr('class', 'x axis') .attr('transform', fmt('translate(0,{0})', plot_height)) .call(xAxis); - const y_axis = d3.svg.axis() - .scale(y_scale) - .orient('left') - .innerTickSize(-plot_width) - .outerTickSize(0) + const y_axis = d3.axisLeft(y_scale) + .tickSizeInner(-plot_width) + .tickSizeOuter(0) .tickPadding(7) .tickFormat(d3.format('d')) //integers - .tickValues(_.range(obs_max)); //prevent unlabeled, non-integer ticks. + .tickValues(range(obs_max)); //prevent unlabeled, non-integer ticks. qq_plot.append('g') .attr('class', 'y axis') .call(y_axis); diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 2fd6b6c..ac4ce0d 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.8 # NOTE: Some dependencies require specialized libraries that slimmer images (like alpine) do not yet support, eg # musl doesn't seem to provide GLOB_TILDE @@ -6,11 +6,7 @@ ENV PYTHONUNBUFFERED 1 # libmagic: file format detection RUN apt-get update && apt-get install -y libmagic-dev - -# Requirements are installed here to ensure they will be cached. -COPY ./requirements /requirements RUN pip install --upgrade pip wheel -RUN pip install -r /requirements/local.txt COPY ./compose/production/django/entrypoint /entrypoint RUN sed -i 's/\r//' /entrypoint @@ -32,6 +28,10 @@ COPY ./compose/local/django/celery/flower/start /start-flower RUN sed -i 's/\r//' /start-flower RUN chmod +x /start-flower +# Requirements are installed here to ensure they will be cached. +COPY ./requirements /requirements +RUN pip install -r /requirements/local.txt + WORKDIR /app ENTRYPOINT ["/entrypoint"] diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index ca4e74b..5ded81e 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -1,4 +1,16 @@ -FROM python:3.6 +### Production dockerfile for the web app. JS and python are implemented as two steps of a multistage build. +# Multistage build + +# Step 1: JS assets. Ensure that a change to package.json or yarn.lock invalidates the cache +FROM node:fermium as jsbuilder +COPY ./package.json ./yarn.lock /build/ +COPY . /build/ +WORKDIR /build/ +RUN yarn install --from-lockfile && yarn run prod + + +# Step 2 (main): Django app. Install python dependencies, add volume mounts, and run. +FROM python:3.8 ARG UID ARG GID @@ -51,7 +63,10 @@ RUN sed -i 's/\r//' /start-flower RUN chmod +x /start-flower RUN chown lzupload /start-flower + +# Copy code and built JS assets FIXME: A little clunky as it depends on some very specific known pathnames. COPY . /app +COPY --from=jsbuilder /build/locuszoom_plotting_service/static/webpack_bundles /app/locuszoom_plotting_service/static/webpack_bundles RUN chown -R lzupload /app @@ -61,7 +76,4 @@ USER lzupload WORKDIR /app -# TODO: add a JS build step. Eg: -# https://medium.com/@shakyShane/lets-talk-about-docker-artifacts-27454560384f - ENTRYPOINT ["/entrypoint"] diff --git a/compose/production/django/celery/flower/start b/compose/production/django/celery/flower/start index 339d0a4..a9253b0 100644 --- a/compose/production/django/celery/flower/start +++ b/compose/production/django/celery/flower/start @@ -4,7 +4,7 @@ set -o errexit set -o nounset -# The URL prefix should match what is used in the Apache configuration. +# The URL prefix should match what is used in the Apache configuration. (it is used to guide fetching of static assets, like css) # In the future we can make this more tolerant of other deployment scenarios. celery flower \ --app=locuszoom_plotting_service.taskapp \ diff --git a/config/auth_urls.py b/config/auth_urls.py index 67a7bd1..781aba7 100644 --- a/config/auth_urls.py +++ b/config/auth_urls.py @@ -17,7 +17,7 @@ from allauth.socialaccount import providers from allauth.socialaccount import views as social_views -providers_urlpatterns = [] # type: ignore +providers_urlpatterns = [] for provider in providers.registry.get_list(): prov_mod = importlib.import_module(provider.get_package() + '.urls') diff --git a/config/settings/base.py b/config/settings/base.py index d7e4b6c..876e1db 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -70,7 +70,6 @@ 'allauth.socialaccount.providers.google', 'rest_framework', 'django_filters', - 'webpack_loader' ] LOCAL_APPS = [ 'locuszoom_plotting_service.users.apps.UsersAppConfig', # Auth, login, and user detail pages @@ -255,12 +254,19 @@ CELERY_TASK_SERIALIZER = 'json' # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_serializer CELERY_RESULT_SERIALIZER = 'json' +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_acks_late +# This causes messages from worker tasks to be received only after the task has completed, not right +# before it executes. This means your tasks MUST be idempotent (i.e. no side effects or issues from +# running the task multiple times). +CELERY_TASK_ACKS_LATE = True +# This prevents the task from being lost simply because the worker was killed or terminated +CELERY_TASK_REJECT_ON_WORKER_LOST = True # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-time-limit -# TODO: set to whatever value is adequate in your circumstances CELERY_TASK_TIME_LIMIT = None # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-soft-time-limit -# TODO: set to whatever value is adequate in your circumstances CELERY_TASK_SOFT_TIME_LIMIT = None +# https://docs.celeryproject.org/en/v4.4.6/userguide/configuration.html#std:setting-worker_prefetch_multiplier +CELERY_WORKER_PREFETCH_MULTIPLIER = 1 # Disabled prefetching so that workers don't reserve tasks, best for long tasks ##### @@ -324,15 +330,3 @@ # ------------------------------------------------------------------------------ SENTRY_DSN = env('SENTRY_DSN', default=None) SENTRY_DSN_FRONTEND = env('SENTRY_DSN_FRONTEND', default=None) - -# This is used to find the interactive parts of pages, which are written and built using Vue.js + Webpack -WEBPACK_LOADER = { - 'DEFAULT': { - 'CACHE': not DEBUG, - 'BUNDLE_DIR_NAME': 'webpack_bundles/', # must end with slash - 'STATS_FILE': str(ROOT_DIR.path('webpack-stats.json')), - 'POLL_INTERVAL': 0.1, - 'TIMEOUT': None, - 'IGNORE': [r'.+\.hot-update.js'] - } -} diff --git a/config/settings/local.py b/config/settings/local.py index 9ae6aaa..51e1767 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -67,6 +67,7 @@ # Celery # ------------------------------------------------------------------------------ # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-always-eager +# Enabling this can facilitate debugging, but for ingest tasks, running synchronously is very slow CELERY_TASK_ALWAYS_EAGER = False # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-eager-propagates CELERY_TASK_EAGER_PROPAGATES = True @@ -74,4 +75,3 @@ # Misc other configuration # ------------------------------------------------------------------------------ LZ_OFFICIAL_DOMAIN = 'localhost:8000' -WEBPACK_LOADER['DEFAULT']['CACHE'] = not DEBUG # noqa F405 diff --git a/config/settings/production.py b/config/settings/production.py index 22eb0d4..0a25c04 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -53,7 +53,7 @@ # https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff SECURE_CONTENT_TYPE_NOSNIFF = env.bool('DJANGO_SECURE_CONTENT_TYPE_NOSNIFF', default=True) -# Storages +# Storages. This will automatically add compression and filename hashing/cachebusting for static assets. STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" MEDIA_ROOT = '/lz-uploads' diff --git a/config/settings/test.py b/config/settings/test.py index ca8e0bd..49b0658 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -10,6 +10,7 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#debug DEBUG = False # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key +# Signoff: This key is not used in production; randomly generated and test-only SECRET_KEY = env("DJANGO_SECRET_KEY", default="ph39zwvHUWA8J9iC4KtNed1hYuX6gTciHCUvSUjbFXaA0Cg8pdASyCPLAjUdirQY") # https://docs.djangoproject.com/en/dev/ref/settings/#test-runner TEST_RUNNER = "django.test.runner.DiscoverRunner" diff --git a/docs/deploy/index.md b/docs/deploy/index.md index cde5b77..03d1564 100644 --- a/docs/deploy/index.md +++ b/docs/deploy/index.md @@ -2,20 +2,18 @@ Deployment has been tested on Ubuntu 16.04 LTS. Other systems may work, but have not been tested. ## System requirements -The hosted LocusZoom app is run inside a docker container, but the web server (eg apache) is left to the host machine. -Sample configuration files are provided where appropriate. +The hosted LocusZoom app is run inside a Docker container, but the web server (eg apache) is left to the host machine. This configuration was chosen because our production app runs on shared hardware, but there is no technical reason why the server could not be containerized as well in principle. Sample configuration files are provided where appropriate. - Apache 2 - Several modules must be enabled (`sudo a2enmod ssl proxy_http headers rewrite`) - - All examples assume that you have enabled HTTPs in production. We use - [Certbot + LetsEncrypt](https://certbot.eff.org/lets-encrypt/ubuntuxenial-apache). -- [Docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/) + - All examples assume that you have enabled HTTPS in production. We use [Certbot + LetsEncrypt](https://certbot.eff.org/lets-encrypt/). +- [Docker](https://docs.docker.com/engine/install/ubuntu/) - [Docker Compose](https://docs.docker.com/compose/install/) ## Required server configuration ### Apache virtualhosts -The app is run within a docker container, and reached from the outside world via a reverse proxy. Sample config is +The app is run within a Docker container, and reached from the outside world via a reverse proxy. Sample config is provided for Apache2. Install the provided `sample-apache.conf` file (editing the `ServerName` as appropriate), and reload Apache. @@ -31,15 +29,14 @@ files will be stored. ## Required app configuration Create two config files describing the production environment and populate secrets, config variables, etc according to -the pre-populated templates: `.envs/.production/.django` and `.envs/.production/.postgres` +the pre-populated templates: `.envs/.production/.django` and `.envs/.production/.postgres` + +(sample files are provided; copy them from `*-sample` to match the names above) Be sure to keep your production settings private! ## Build and deployment -Make sure to build the UI code (`yarn install && yarn run prod`) before creating the docker container. (in the future -this step should be automated!!) - -Build the docker container in production (or download an appropriate pre-made image): +Build the Docker container in production (in the future we may create a pre-built image): `sudo docker-compose -f production.yml build --pull` Start the container: @@ -48,31 +45,31 @@ Start the container: This app uses internal django features to serve static assets, so those do not require a separate deploy step. ## Once the app is running... -### Run migrations for the first time +### Run migrations for the first time to set up the database `$ sudo docker-compose -f production.yml run --rm django python manage.py migrate` ### Load sample data required for base functionality + +The "find rsID for SNP" feature requires downloading premade lookup tables from our server. + ```bash $ docker-compose -f production.yml run --rm django zorp-assets download --type snp_to_rsid --tag genome_build GRCh37 --no-update $ docker-compose -f production.yml run --rm django zorp-assets download --type snp_to_rsid --tag genome_build GRCh38 --no-update ``` + ### Create an admin user You will need to enter some configuration into the admin panel before using the app for the first time. -In production, your admin site must be hidden behind an obfuscated URL. See ___ for details. +In production, your admin site must be hidden behind an obfuscated URL. This is determined by the `DJANGO_ADMIN_URL` + setting in `.envs/.production/.django`. -Use the following command to create an admin user. +Use the following command to create an admin user. Be sure to protect these credentials! `$ sudo docker-compose -f production.yml run --rm django python manage.py createsuperuser` - ### OAuth settings -This site uses social OAuth login via Django-allauth. In order to log in, you will need to do -Follow the [auth setup instructions](https://django-allauth.readthedocs.io/en/latest/installation.html) to register -OAuth credentials (client ID and secret) for your local app. The site URL must match the callback registered -with the OAuth provider. +This site uses social OAuth login via Django-allauth. In order to log in, you will need to follow the [auth setup instructions](https://django-allauth.readthedocs.io/en/latest/installation.html) to register OAuth credentials (client ID and secret) for your local app. The site URL must match the callback registered with the OAuth provider. -You do not need to create a `Site` entry in the Django admin, as the app will do this automatically for you on -first startup (based on the `LZ_OFFICIAL_URL` registered in your .env file) +You do not need to create a `Site` entry in the Django admin, as the app will do this automatically for you on first startup (based on the `LZ_OFFICIAL_URL` registered in your .env file) A sample callback URL for OAuth registration (in local development) would be: http://localhost:8000/accounts/google/login/callback/ @@ -81,16 +78,10 @@ A sample callback URL for OAuth registration (in local development) would be: ## Releasing a new version (checklist) -- (future) Verify backup status on DB backups -- `yarn install --from-lockfile && yarn run prod` - `docker-compose -f production.yml build --pull` - `docker-compose -f production.yml up -d` - (optional) `docker-compose -f production.yml run --rm django python manage.py migrate` -Note: if you are using experimental versions of JS libraries (such as pinning to a git commit), yarn may ignore its -lockfile and install an older version instead. In that case, run `yarn cache clean []` before -you begin. As the project matures, we will shift to using official version releases in place of git commits. - ### Monitoring the new release - Monitor new error reports from Sentry - Watch application logs during manual QA via: `docker-compose -f production.yml logs --follow --tail 20` diff --git a/docs/pycharm/configuration.rst b/docs/pycharm/configuration.rst index f3a58e7..3d9f97f 100644 --- a/docs/pycharm/configuration.rst +++ b/docs/pycharm/configuration.rst @@ -14,7 +14,7 @@ This repository comes with already prepared "Run/Debug Configurations" for docke .. image:: images/2.png -But as you can see, at the beggining there is something wrong with them. They have red X on django icon, and they cannot be used, without configuring remote python interpteter. To do that, you have to go to *Settings > Build, Execution, Deployment* first. +But as you can see, at the beginning there is something wrong with them. They have red X on django icon, and they cannot be used, without configuring remote python interpreter. To do that, you have to go to *Settings > Build, Execution, Deployment* first. Next, you have to add new remote python interpreter, based on already tested deployment settings. Go to *Settings > Project > Project Interpreter*. Click on the cog icon, and click *Add Remote*. diff --git a/local.yml b/local.yml index 513a9af..4d0fc1d 100644 --- a/local.yml +++ b/local.yml @@ -35,7 +35,7 @@ services: - ./.envs/.local/.postgres redis: - image: redis:3.2 + image: redis:5.0 celeryworker: <<: *django diff --git a/locuszoom_plotting_service/gwas/admin.py b/locuszoom_plotting_service/gwas/admin.py index 58a7360..7da4fec 100644 --- a/locuszoom_plotting_service/gwas/admin.py +++ b/locuszoom_plotting_service/gwas/admin.py @@ -12,7 +12,7 @@ # social auth user tokens. We'll manually de-register them in an file that is loaded after allauth admin.site.unregister([SocialToken]) -# We may want to re-enable this in the future, but for now this is a powerful action and we will delete it +# We may want to re-enable this in the future, but for now this is a powerful action and we will disable it admin.site.disable_action('delete_selected') diff --git a/locuszoom_plotting_service/gwas/models.py b/locuszoom_plotting_service/gwas/models.py index db5a10c..b95c705 100644 --- a/locuszoom_plotting_service/gwas/models.py +++ b/locuszoom_plotting_service/gwas/models.py @@ -163,7 +163,7 @@ class AnalysisFileset(TimeStampedModel): # Options related to the ingestion pipeline parser_options = JSONField(null=False, blank=False, - default={}, # Uploads must tell us how to they are parsed + default=lambda: {}, # Uploads must tell us how they are parsed help_text='Parser options (zorp-compatible parser kwarg names)') ingest_status = models.IntegerField(choices=constants.INGEST_STATES, default=0, diff --git a/locuszoom_plotting_service/gwas/tests/factories.py b/locuszoom_plotting_service/gwas/tests/factories.py index af5f7a0..11ad8fe 100644 --- a/locuszoom_plotting_service/gwas/tests/factories.py +++ b/locuszoom_plotting_service/gwas/tests/factories.py @@ -4,6 +4,7 @@ from django.db.models import signals from django.utils import timezone import factory +from factory.django import DjangoModelFactory from locuszoom_plotting_service.users.tests.factories import UserFactory from .. import constants as lz_constants @@ -19,7 +20,7 @@ def choose_consortium() -> str: @factory.django.mute_signals(signals.post_save) -class AnalysisFilesetFactory(factory.DjangoModelFactory): +class AnalysisFilesetFactory(DjangoModelFactory): raw_gwas_file = None # Only create temp files if has_data trait is True ingest_status = 0 # pending (most tests don't run celery tasks, and therefore are "pending" processing) @@ -50,7 +51,7 @@ class Params: ) -class AnalysisInfoFactory(factory.DjangoModelFactory): +class AnalysisInfoFactory(DjangoModelFactory): owner = factory.SubFactory(UserFactory) label = factory.Faker('sentence', nb_words=2) study_name = factory.LazyFunction(choose_consortium) @@ -65,7 +66,7 @@ class Meta: model = lz_models.AnalysisInfo -class ViewLinkFactory(factory.DjangoModelFactory): +class ViewLinkFactory(DjangoModelFactory): label = factory.Faker('sentence', nb_words=2) gwas = factory.SubFactory(AnalysisInfoFactory) diff --git a/locuszoom_plotting_service/static/sass/custom_bootstrap_vars.scss b/locuszoom_plotting_service/static/sass/custom_bootstrap_vars.scss deleted file mode 100644 index a7fc64d..0000000 --- a/locuszoom_plotting_service/static/sass/custom_bootstrap_vars.scss +++ /dev/null @@ -1 +0,0 @@ -// FIXME: This file may be deprecated diff --git a/locuszoom_plotting_service/static/sass/project.scss b/locuszoom_plotting_service/static/sass/project.scss deleted file mode 100644 index 3ac991d..0000000 --- a/locuszoom_plotting_service/static/sass/project.scss +++ /dev/null @@ -1,37 +0,0 @@ -// FIXME: This file may be deprecated - - - -// project specific CSS goes here - -//////////////////////////////// - //Variables// -//////////////////////////////// - -// Alert colors - -$white: #fff; -$mint-green: #d6e9c6; -$black: #000; -$pink: #f2dede; -$dark-pink: #eed3d7; -$red: #b94a48; - -//////////////////////////////// - //Alerts// -//////////////////////////////// - -// bootstrap alert CSS, translated to the django-standard levels of -// debug, info, success, warning, error - -.alert-debug { - background-color: $white; - border-color: $mint-green; - color: $black; -} - -.alert-error { - background-color: $pink; - border-color: $dark-pink; - color: $red; -} diff --git a/locuszoom_plotting_service/templates/base.html b/locuszoom_plotting_service/templates/base.html index 4e94261..0d5a9a6 100644 --- a/locuszoom_plotting_service/templates/base.html +++ b/locuszoom_plotting_service/templates/base.html @@ -5,7 +5,7 @@ - {% block title %}My.LocusZoom.org{% endblock title %} + {% block title %}Plot Your Own Data{% endblock title %} | LocusZoom @@ -114,7 +114,7 @@ {% block content %}
-

Use this document as a way to quick start any new project.

+

Override this section for new content in page.

{% endblock content %} @@ -137,6 +137,7 @@ integrity="sha384-Q6E9RHvbIyZFJoft+2mJbHaEWldlvI9IOYy5n3zV9zzTtmI3UksdQRVvoxMfooAo"> + {% block javascript %}{% endblock javascript %} diff --git a/locuszoom_plotting_service/templates/gwas/gwas_region.html b/locuszoom_plotting_service/templates/gwas/gwas_region.html index e3a19e7..aa4b527 100644 --- a/locuszoom_plotting_service/templates/gwas/gwas_region.html +++ b/locuszoom_plotting_service/templates/gwas/gwas_region.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% load render_bundle from webpack_loader %} +{% load static %} {% block title %}Region plot- {{ gwas.label }}{% endblock %} @@ -37,5 +37,5 @@ {# {{ js_vars | json_script: 'js-vars' }}#} {# Define the LocusZoom plot #} - {% render_bundle 'gwas_region' %} + {% endblock %} diff --git a/locuszoom_plotting_service/templates/gwas/gwas_summary.html b/locuszoom_plotting_service/templates/gwas/gwas_summary.html index 7da3d3e..6ee79f2 100644 --- a/locuszoom_plotting_service/templates/gwas/gwas_summary.html +++ b/locuszoom_plotting_service/templates/gwas/gwas_summary.html @@ -1,6 +1,5 @@ {% extends 'base.html' %} -{% load render_bundle from webpack_loader %} - +{% load static %} {% block title %}Study- {{ gwas.label }}{% endblock %} @@ -98,7 +97,7 @@

{{ gwas.label }}

Manhattan Plot

Click on any peak in the manhattan plot below to jump to a specific LocusZoom region view, or visit the - region page to make plots for any region in your dataset + region page to make plots for any region in your dataset (with search features for gene, position, or rsid).

@@ -138,10 +137,6 @@

QQ Plot:

{% endblock %} {% block javascript %} - - - - {% endif %} - {% render_bundle 'gwas_summary' %} + {% endblock %} diff --git a/locuszoom_plotting_service/templates/gwas/upload.html b/locuszoom_plotting_service/templates/gwas/upload.html index cb01e7b..3654038 100644 --- a/locuszoom_plotting_service/templates/gwas/upload.html +++ b/locuszoom_plotting_service/templates/gwas/upload.html @@ -1,6 +1,8 @@ {% extends 'base.html' %} +{% load static %} {% load crispy_forms_tags %} -{% load render_bundle from webpack_loader %} + +{% block title %}Upload your data{% endblock %} {% block css %}