Skip to content

Commit

Permalink
improve MetaTable.get_existing_idx
Browse files Browse the repository at this point in the history
  • Loading branch information
elephantum committed Aug 31, 2024
1 parent 0e32abb commit d46fdc8
Showing 1 changed file with 4 additions and 15 deletions.
19 changes: 4 additions & 15 deletions datapipe/meta/sql_meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,21 +240,10 @@ def get_existing_idx(self, idx: Optional[IndexDF] = None) -> IndexDF:
else:
idx_cols = []

if len(idx_cols) > 0:
row_queries = []

# FIXME поправить на сравнение кортежей
assert idx is not None
for _, row in idx.iterrows():
and_params = [
self.sql_table.c[key] == self._get_sql_param(row[key]) # type: ignore
for key in idx_cols
if key in self.primary_keys
]
and_query = sa.and_(*and_params)
row_queries.append(and_query)

sql = sql.where(sa.or_(*row_queries))
if len(idx_cols) > 0 and idx is not None and len(idx) > 0:
sql = sql_apply_idx_filter_to_table(
sql=sql, table=self.sql_table, primary_keys=idx_cols, idx=idx
)

sql = sql.where(self.sql_table.c.delete_ts.is_(None))

Expand Down

0 comments on commit d46fdc8

Please sign in to comment.