Merge pull request #75 from cancervariants/staging

Staging

.flake8

@@ -1,5 +1,6 @@
 [flake8]
 extend-ignore = D205, D400
+max-line-length = 88
 exclude =
   .git
   venv

Pipfile

@@ -29,4 +29,4 @@ rdflib = "*"
 boto3 = "*"
 requests = "*"
 typing-extensions = "*"
-"ga4gh.vrsatile.pydantic" = ">=0.0.5"
+"ga4gh.vrsatile.pydantic" = ">=0.1.dev1"

disease/cli.py

@@ -2,7 +2,7 @@
 import click
 from disease import SOURCES_CLASS_LOOKUP, SOURCES_LOWER_LOOKUP, logger
 from disease.schemas import SourceName
-from disease.database import Database
+from disease.database import Database, confirm_aws_db_use
 from disease.etl.mondo import Mondo
 from disease.etl.merge import Merge
 from botocore.exceptions import ClientError
@@ -43,6 +43,11 @@ class CLI:
     def update_normalizer_db(normalizer, prod, db_url, update_all,
                              update_merged):
         """Update selected source(s) in the Disease Normalizer database."""
+        # Sometimes DISEASE_NORM_EB_PROD is accidentally set. We should verify that
+        # it should actually be used in CLI
+        if "DISEASE_NORM_EB_PROD" in environ:
+            confirm_aws_db_use("PROD")
+
         if prod:
             environ['DISEASE_NORM_PROD'] = "TRUE"
             db: Database = Database()

disease/database.py

@@ -9,6 +9,16 @@
 from typing import Optional, Dict, List, Any
 
 
+def confirm_aws_db_use(env_name: str) -> None:
+    """Check to ensure that AWS instance should actually be used."""
+    if click.confirm(f"Are you sure you want to use the AWS {env_name} database?",
+                     default=False):
+        click.echo(f"***DISEASE {env_name.upper()} DATABASE IN USE***")
+    else:
+        click.echo("Exiting.")
+        sys.exit()
+
+
 class Database:
     """The database class."""
 
@@ -18,20 +28,30 @@ def __init__(self, db_url: str = '', region_name: str = 'us-east-2'):
         :param str db_url: database endpoint URL to connect to
         :param str region_name: AWS region name to use
         """
-        if 'DISEASE_NORM_PROD' in environ or \
-                'DISEASE_NORM_EB_PROD' in environ:
+        disease_concepts_table = "disease_concepts"  # default
+        disease_metadata_table = "disease_metadata"  # default
+
+        if 'DISEASE_NORM_PROD' in environ or 'DISEASE_NORM_EB_PROD' in environ:
             boto_params = {
                 'region_name': region_name
             }
             if 'DISEASE_NORM_EB_PROD' not in environ:
                 # EB Instance should not have to confirm.
                 # This is used only for using production via CLI
-                if click.confirm("Are you sure you want to use the "
-                                 "production database?", default=False):
-                    click.echo("***DISEASE PRODUCTION DATABASE IN USE***")
-                else:
-                    click.echo("Exiting.")
-                    sys.exit()
+                confirm_aws_db_use("PROD")
+        elif "DISEASE_NORM_NONPROD" in environ:
+            # This is a nonprod table. Only to be used for creating backups which
+            # prod will restore. Will need to manually delete / create this table
+            # on an as needed basis.
+            disease_concepts_table = "disease_concepts_nonprod"
+            disease_metadata_table = "disease_metadata_nonprod"
+
+            boto_params = {
+                "region_name": region_name
+            }
+
+            # This is used only for updating nonprod via CLI
+            confirm_aws_db_use("NONPROD")
         else:
             if db_url:
                 endpoint_url = db_url
@@ -48,15 +68,16 @@ def __init__(self, db_url: str = '', region_name: str = 'us-east-2'):
         self.dynamodb = boto3.resource('dynamodb', **boto_params)
         self.dynamodb_client = boto3.client('dynamodb', **boto_params)
 
-        # Table creation for local database
-        if 'DISEASE_NORM_PROD' not in environ and \
-                'DISEASE_NORM_EB_PROD' not in environ:
+        # Only create tables for local instance
+        envs_do_not_create_tables = {"DISEASE_NORM_PROD", "DISEASE_NORM_EB_PROD",
+                                     "DISEASE_NORM_NONPROD"}
+        if not set(envs_do_not_create_tables) & set(environ):
             existing_tables = self.dynamodb_client.list_tables()['TableNames']
             self.create_diseases_table(existing_tables)
             self.create_meta_data_table(existing_tables)
 
-        self.diseases = self.dynamodb.Table('disease_concepts')
-        self.metadata = self.dynamodb.Table('disease_metadata')
+        self.diseases = self.dynamodb.Table(disease_concepts_table)
+        self.metadata = self.dynamodb.Table(disease_metadata_table)
         self.batch = self.diseases.batch_writer()
         self.cached_sources = {}
 

disease/schemas.py

@@ -5,7 +5,7 @@
 from enum import Enum, IntEnum
 from pydantic import BaseModel, StrictBool
 from datetime import datetime
-from ga4gh.vrsatile.pydantic.vrsatile_models import ValueObjectDescriptor
+from ga4gh.vrsatile.pydantic.vrsatile_models import DiseaseDescriptor
 
 
 class MatchType(IntEnum):
@@ -316,7 +316,7 @@ class NormalizationService(BaseModel):
     query: str
     warnings: Optional[Dict]
     match_type: MatchType
-    disease_descriptor: Optional[ValueObjectDescriptor]
+    disease_descriptor: Optional[DiseaseDescriptor]
     source_meta_: Optional[Dict[SourceName, SourceMeta]]
     service_meta_: ServiceMeta
 

disease/version.py

@@ -1,2 +1,2 @@
 """Disease normalizer version"""
-__version__ = "0.2.12"
+__version__ = "0.2.13"

requirements-dev.txt

@@ -1,105 +1,92 @@
-#
-# These requirements were autogenerated by pipenv
-# To regenerate from the project's Pipfile, run:
-#
-#    pipenv lock --requirements --dev
-#
-
-# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
-# requirements. To emit only development requirements, pass "--dev-only".
-
 -i https://pypi.org/simple
--e .
-anyio==3.5.0; python_full_version >= '3.6.2'
-appnope==0.1.2; platform_system == 'Darwin'
-asgiref==3.4.1; python_version >= '3.6'
-asttokens==2.0.5
-attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+anyio==3.6.1
+boto3==1.24.58
+botocore==1.27.58
+certifi==2022.6.15
+charset-normalizer==2.1.1
+click==8.1.3
+fastapi==0.80.0
+ga4gh.vrsatile.pydantic==0.1.dev1
+h11==0.13.0
+httptools==0.4.0
+idna==3.3
+isodate==0.6.1
+jmespath==1.0.1
+owlready2==0.38
+pydantic==1.9.2
+pyparsing==2.4.7
+python-dateutil==2.8.2
+rdflib==6.2.0
+requests==2.28.1
+s3transfer==0.6.0
+setuptools==65.3.0
+six==1.16.0
+sniffio==1.2.0
+starlette==0.19.1
+typing-extensions==4.3.0
+urllib3==1.26.12
+uvicorn==0.18.3
+uvloop==0.16.0
+websockets==10.3
+appnope==0.1.3
+asttokens==2.0.8
+attrs==22.1.0
 backcall==0.2.0
-black==21.12b0; python_full_version >= '3.6.2'
-boto3==1.20.37
-botocore==1.23.37; python_version >= '3.6'
-certifi==2021.10.8
-cfgv==3.3.1; python_full_version >= '3.6.1'
-charset-normalizer==2.0.10; python_version >= '3'
-click==8.0.3
-coverage==6.2
+cfgv==3.3.1
+coverage==6.4.4
 coveralls==3.3.1
-cycler==0.11.0; python_version >= '3.6'
-debugpy==1.5.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-decorator==5.1.1; python_full_version >= '3.5.0'
-distlib==0.3.4
+cycler==0.11.0
+debugpy==1.6.3
+decorator==5.1.1
+-e .
+distlib==0.3.5
 docopt==0.6.2
-entrypoints==0.3; python_version >= '2.7'
-executing==0.8.2
-fastapi==0.72.0
-filelock==3.4.2; python_version >= '3.7'
+entrypoints==0.4
+executing==0.10.0
+filelock==3.8.0
+flake8==5.0.4
 flake8-docstrings==1.6.0
-flake8==4.0.1
-fonttools==4.28.5; python_version >= '3.7'
-ga4gh.vrsatile.pydantic==0.0.6
-h11==0.12.0; python_version >= '3.6'
-httptools==0.3.0
-identify==2.4.4; python_full_version >= '3.6.1'
-idna==3.3; python_version >= '3'
+fonttools==4.37.0
+identify==2.5.3
 iniconfig==1.1.1
-ipykernel==6.7.0
-ipython==8.0.0; python_version >= '3.8'
-isodate==0.6.1
-jedi==0.18.1; python_version >= '3.6'
-jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-jupyter-client==7.1.1; python_full_version >= '3.6.1'
-jupyter-core==4.9.1; python_version >= '3.6'
-kiwisolver==1.3.2; python_version >= '3.7'
-matplotlib-inline==0.1.3; python_full_version >= '3.5.0'
-matplotlib==3.5.1
-mccabe==0.6.1
-mypy-extensions==0.4.3
-nest-asyncio==1.5.4; python_full_version >= '3.5.0'
-nodeenv==1.6.0
-numpy==1.22.1; python_version >= '3.8'
-owlready2==0.35
-packaging==21.3; python_version >= '3.6'
-parso==0.8.3; python_version >= '3.6'
-pathspec==0.9.0
-pexpect==4.8.0; sys_platform != 'win32'
+ipykernel==6.15.1
+ipython==8.4.0
+jedi==0.18.1
+jupyter-client==7.3.4
+jupyter-core==4.11.1
+kiwisolver==1.4.4
+matplotlib==3.5.3
+matplotlib-inline==0.1.6
+mccabe==0.7.0
+nest-asyncio==1.5.5
+nodeenv==1.7.0
+numpy==1.23.2
+packaging==21.3
+parso==0.8.3
+pexpect==4.8.0
 pickleshare==0.7.5
-pillow==9.0.0; python_version >= '3.7'
-platformdirs==2.4.1; python_version >= '3.7'
-pluggy==1.0.0; python_version >= '3.6'
-pre-commit==2.16.0
-prompt-toolkit==3.0.24; python_full_version >= '3.6.2'
-ptyprocess==0.7.0; os_name != 'nt'
-pure-eval==0.2.1
-py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-pycodestyle==2.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-pydantic==1.9.0
-pydocstyle==6.1.1; python_version >= '3.6'
-pyflakes==2.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pygments==2.11.2; python_full_version >= '3.5.0'
-pyparsing==2.4.7
+pillow==9.2.0
+platformdirs==2.5.2
+pluggy==1.0.0
+pre-commit==2.20.0
+prompt-toolkit==3.0.30
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py==1.11.0
+pycodestyle==2.9.1
+pydocstyle==6.1.1
+pyflakes==2.5.0
+pygments==2.13.0
+pytest==7.1.2
 pytest-cov==3.0.0
-pytest==6.2.5
-python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyyaml==6.0; python_version >= '3.6'
-pyzmq==22.3.0; python_version >= '3.6'
-rdflib==6.1.1
-requests==2.27.1
-s3transfer==0.5.0; python_version >= '3.6'
-setuptools==60.5.0; python_version >= '3.7'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sniffio==1.2.0; python_version >= '3.5'
+pyyaml==6.0
+pyzmq==23.2.1
 snowballstemmer==2.2.0
-stack-data==0.1.4
-starlette==0.17.1; python_version >= '3.6'
-toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-tomli==1.2.3
-tornado==6.1; python_full_version >= '3.5.0'
-traitlets==5.1.1; python_version >= '3.7'
-typing-extensions==4.0.1
-urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-uvicorn==0.17.0
-uvloop==0.16.0
-virtualenv==20.13.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+stack-data==0.4.0
+toml==0.10.2
+tomli==2.0.1
+tornado==6.2
+traitlets==5.3.0
+virtualenv==20.16.3
 wcwidth==0.2.5
-websockets==10.1

requirements.txt

@@ -1,38 +1,30 @@
-#
-# These requirements were autogenerated by pipenv
-# To regenerate from the project's Pipfile, run:
-#
-#    pipenv lock --requirements
-#
-
 -i https://pypi.org/simple
-anyio==3.5.0; python_full_version >= '3.6.2'
-asgiref==3.4.1; python_version >= '3.6'
-boto3==1.20.37
-botocore==1.23.37; python_version >= '3.6'
-certifi==2021.10.8
-charset-normalizer==2.0.10; python_version >= '3'
-click==8.0.3
-fastapi==0.72.0
-ga4gh.vrsatile.pydantic==0.0.6
-h11==0.12.0; python_version >= '3.6'
-httptools==0.3.0
-idna==3.3; python_version >= '3'
+anyio==3.6.1
+boto3==1.24.58
+botocore==1.27.58
+certifi==2022.6.15
+charset-normalizer==2.1.1
+click==8.1.3
+fastapi==0.80.0
+ga4gh.vrsatile.pydantic==0.1.dev1
+h11==0.13.0
+httptools==0.4.0
+idna==3.3
 isodate==0.6.1
-jmespath==0.10.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-owlready2==0.35
-pydantic==1.9.0
+jmespath==1.0.1
+owlready2==0.38
+pydantic==1.9.2
 pyparsing==2.4.7
-python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-rdflib==6.1.1
-requests==2.27.1
-s3transfer==0.5.0; python_version >= '3.6'
-setuptools==60.5.0; python_version >= '3.7'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sniffio==1.2.0; python_version >= '3.5'
-starlette==0.17.1; python_version >= '3.6'
-typing-extensions==4.0.1
-urllib3==1.26.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-uvicorn==0.17.0
+python-dateutil==2.8.2
+rdflib==6.2.0
+requests==2.28.1
+s3transfer==0.6.0
+setuptools==65.3.0
+six==1.16.0
+sniffio==1.2.0
+starlette==0.19.1
+typing-extensions==4.3.0
+urllib3==1.26.12
+uvicorn==0.18.3
 uvloop==0.16.0
-websockets==10.1
+websockets==10.3

setup.cfg

@@ -37,7 +37,7 @@ install_requires =
     boto3
     requests
     typing-extensions
-    ga4gh.vrsatile.pydantic >= 0.0.5
+    ga4gh.vrsatile.pydantic >= 0.1.dev1
 
 [tool:pytest]
 addopts = --ignore=setup.py --ignore=codebuild/ --ignore=analysis/ --doctest-modules --cov-report term-missing --cov .