From 06898c09e492970b0c8f25e6c896916afb609f1f Mon Sep 17 00:00:00 2001 From: James Stevenson Date: Fri, 3 Jan 2025 19:49:58 -0500 Subject: [PATCH] more updates --- pyproject.toml | 5 +++++ src/gene/cli.py | 6 +++--- src/gene/database/dynamodb.py | 18 +++++++++--------- src/gene/database/postgresql.py | 6 +++--- src/gene/etl/merge.py | 8 ++++---- src/gene/query.py | 13 ++++++------- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4360f81..ea4901b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,12 +117,14 @@ select = [ "RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse "RET", # https://docs.astral.sh/ruff/rules/#flake8-return-ret "SLF", # https://docs.astral.sh/ruff/rules/#flake8-self-slf + "SLOT", # https://docs.astral.sh/ruff/rules/#flake8-slots-slot "SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim "ARG", # https://docs.astral.sh/ruff/rules/#flake8-unused-arguments-arg "PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth "PGH", # https://docs.astral.sh/ruff/rules/#pygrep-hooks-pgh "PLC", # https://docs.astral.sh/ruff/rules/#convention-c "PLE", # https://docs.astral.sh/ruff/rules/#error-e_1 + "TRY", # https://docs.astral.sh/ruff/rules/#tryceratops-try "PERF", # https://docs.astral.sh/ruff/rules/#perflint-perf "FURB", # https://docs.astral.sh/ruff/rules/#refurb-furb "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf @@ -143,6 +145,7 @@ fixable = [ "SIM", "PLC", "PLE", + "TRY", "PERF", "FURB", "RUF" @@ -161,6 +164,7 @@ fixable = [ # E501 - line-too-long* # W191 - tab-indentation* # S321 - suspicious-ftp-lib-usage +# PLC0206 - dict-index-missing-items # *ignored for compatibility with formatter ignore = [ "ANN003", @@ -168,6 +172,7 @@ ignore = [ "E111", "E114", "E117", "E501", "W191", "S321", + "PLC0206" ] [tool.ruff.lint.per-file-ignores] diff --git a/src/gene/cli.py b/src/gene/cli.py index c328672..6912391 100644 --- a/src/gene/cli.py +++ b/src/gene/cli.py @@ -206,7 +206,7 @@ def _load_source( try: processed_ids += source.perform_etl(use_existing) except GeneNormalizerEtlError as e: - _logger.error(e) + _logger.exception("ETL error while loading %s", n) click.echo(f"Encountered error while loading {n}: {e}.") click.get_current_context().exit() end_load = timer() @@ -322,13 +322,13 @@ def update_normalizer_db( if len(sources_split) == 0: err_msg = "Must enter 1 or more source names to update" - raise Exception(err_msg) + raise ValueError(err_msg) non_sources = set(sources_split) - set(SOURCES) if len(non_sources) != 0: err_msg = f"Not valid source(s): {non_sources}" - raise Exception(err_msg) + raise ValueError(err_msg) parsed_source_names = {SourceName(SOURCES[s]) for s in sources_split} _update_normalizer(parsed_source_names, db, update_merged, use_existing) diff --git a/src/gene/database/dynamodb.py b/src/gene/database/dynamodb.py index 37e0596..e1ebba9 100644 --- a/src/gene/database/dynamodb.py +++ b/src/gene/database/dynamodb.py @@ -105,8 +105,8 @@ def drop_db(self) -> None: try: if not self._check_delete_okay(): return - except DatabaseWriteException as e: - raise e + except DatabaseWriteException: # noqa: TRY203 + raise if self.gene_table in self.list_tables(): self.dynamodb.Table(self.gene_table).delete() @@ -250,9 +250,9 @@ def get_record_by_id( response = self.genes.query(KeyConditionExpression=exp) record = response["Items"][0] del record["label_and_type"] - return record + return record # noqa: TRY300 except ClientError as e: - _logger.error( + _logger.exception( "boto3 client error on get_records_by_id for search term %s: %s", concept_id, e.response["Error"]["Message"], @@ -275,7 +275,7 @@ def get_refs_by_type(self, search_term: str, ref_type: RefType) -> list[str]: matches = self.genes.query(KeyConditionExpression=filter_exp) return [m["concept_id"] for m in matches.get("Items", None)] except ClientError as e: - _logger.error( + _logger.exception( "boto3 client error on get_refs_by_type for search term %s: %s", search_term, e.response["Error"]["Message"], @@ -378,7 +378,7 @@ def add_record(self, record: dict, src_name: SourceName) -> None: try: self.batch.put_item(Item=record) except ClientError as e: - _logger.error( + _logger.exception( "boto3 client error on add_record for %s: %s", concept_id, e.response["Error"]["Message"], @@ -411,7 +411,7 @@ def add_merged_record(self, record: dict) -> None: try: self.batch.put_item(Item=record) except ClientError as e: - _logger.error( + _logger.exception( "boto3 client error on add_record for " "%s: %s", concept_id, e.response["Error"]["Message"], @@ -438,7 +438,7 @@ def _add_ref_record( try: self.batch.put_item(Item=record) except ClientError as e: - _logger.error( + _logger.exception( "boto3 client error adding reference %s for %s with match type %s: %s", term, concept_id, @@ -473,7 +473,7 @@ def update_merge_ref(self, concept_id: str, merge_ref: Any) -> None: # noqa: AN ) raise DatabaseWriteException(err_msg) from e - _logger.error( + _logger.exception( "boto3 client error in `database.update_record()`: %s", e.response["Error"]["Message"], ) diff --git a/src/gene/database/postgresql.py b/src/gene/database/postgresql.py index e3a3bfe..ce5ba60 100644 --- a/src/gene/database/postgresql.py +++ b/src/gene/database/postgresql.py @@ -121,8 +121,8 @@ def drop_db(self) -> None: try: if not self._check_delete_okay(): return - except DatabaseWriteException as e: - raise e + except DatabaseWriteException: # noqa: TRY203 + raise with self.conn.cursor() as cur: cur.execute(self._drop_db_query) @@ -601,7 +601,7 @@ def add_record(self, record: dict, src_name: SourceName) -> None: # noqa: ARG00 cur.execute(self._ins_symbol_query, [record["symbol"], concept_id]) self.conn.commit() except UniqueViolation: - _logger.error("Record with ID %s already exists", concept_id) + _logger.exception("Record with ID %s already exists", concept_id) self.conn.rollback() _add_merged_record_query = b""" diff --git a/src/gene/etl/merge.py b/src/gene/etl/merge.py index e56c92d..8cf4a4c 100644 --- a/src/gene/etl/merge.py +++ b/src/gene/etl/merge.py @@ -57,13 +57,13 @@ def create_merged_concepts(self, record_ids: set[str]) -> None: self._database.update_merge_ref(concept_id, merge_ref) except DatabaseWriteException as dw: if str(dw).startswith("No such record exists"): - _logger.error( + _logger.exception( "Updating nonexistent record: %s for merge ref to %s", concept_id, merge_ref, ) else: - _logger.error(str(dw)) + _logger.exception("Encountered unknown DB write exception") uploaded_ids |= group self._database.complete_write_transaction() _logger.info("Merged concept generation successful.") @@ -135,9 +135,9 @@ def record_order(record: dict) -> tuple: source_rank = SourcePriority[src].value else: err_msg = ( - f"Prohibited source: {src} in concept_id " f"{record['concept_id']}" + f"Prohibited source: {src} in concept_id {record['concept_id']}" ) - raise Exception(err_msg) + raise ValueError(err_msg) return source_rank, record["concept_id"] records.sort(key=record_order) diff --git a/src/gene/query.py b/src/gene/query.py index def5e74..0d996a0 100644 --- a/src/gene/query.py +++ b/src/gene/query.py @@ -168,9 +168,9 @@ def _fetch_record( """ try: match = self.db.get_record_by_id(concept_id, case_sensitive=False) - except DatabaseReadException as e: - _logger.error( - "Encountered DatabaseReadException looking up %s: %s", concept_id, e + except DatabaseReadException: + _logger.exception( + "Encountered DatabaseReadException looking up %s", concept_id ) else: if match: @@ -244,10 +244,9 @@ def _get_search_response(self, query: str, sources: set[str]) -> dict: self._fetch_record(resp, ref, MatchType[item_type.upper()]) matched_concept_ids.append(ref) - except DatabaseReadException as e: - _logger.error( - "Encountered DatabaseReadException %s looking up %s %s: ", - e, + except DatabaseReadException: + _logger.exception( + "Encountered DatabaseReadException looking up %s %s: ", item_type, term, )