From 5aeeb6e732168bf3b7c238fda565a37d4c6f94c2 Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 7 Oct 2024 10:40:46 +0200 Subject: [PATCH 1/6] accept weird tablenames (with spaces or . in it) --- modelbaker/dbconnector/gpkg_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 7654471..240d1c9 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -266,7 +266,7 @@ def _get_tables_info(self): for record in filtered_records: cursor.execute( """ - PRAGMA table_info({}) + PRAGMA table_info("{}") """.format( record["tablename"] ) From 3ad721e14b15a5c688bc55a141ce9c073b460f1b Mon Sep 17 00:00:00 2001 From: signedav Date: Mon, 7 Oct 2024 14:15:52 +0200 Subject: [PATCH 2/6] bump pgserviceparser --- scripts/package_pip_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package_pip_packages.sh b/scripts/package_pip_packages.sh index 101603f..753910d 100755 --- a/scripts/package_pip_packages.sh +++ b/scripts/package_pip_packages.sh @@ -2,7 +2,7 @@ LIBS_DIR="modelbaker/libs" DEPRECATION=("deprecation" "2.1.0") -PGSERVICEPARSER=("pgserviceparser" "2.1.0") +PGSERVICEPARSER=("pgserviceparser" "2.2.1") TOPPINGMAKER=("toppingmaker" "1.4.0") PACKAGES=( From 732d420f8df927787b6cf499ff0ce476ad26f9bc Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 11 Oct 2024 10:29:54 +0200 Subject: [PATCH 3/6] Update mssql image version --- .docker/docker-compose.gh.yml | 2 +- tests/README.md | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.docker/docker-compose.gh.yml b/.docker/docker-compose.gh.yml index ecb59c2..77a46a6 100644 --- a/.docker/docker-compose.gh.yml +++ b/.docker/docker-compose.gh.yml @@ -19,7 +19,7 @@ services: - "POSTGRES_HOST_AUTH_METHOD=md5" mssql: - image: mcr.microsoft.com/mssql/server:2019-CU11-ubuntu-20.04 + image: mcr.microsoft.com/mssql/server:2019-CU28-ubuntu-20.04 environment: ACCEPT_EULA: Y SA_PASSWORD: diff --git a/tests/README.md b/tests/README.md index 081d9ae..b6af7cb 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,8 +27,7 @@ Run all test starting with ``test_array_` These are dirty notes for the quickest way to test mssql queries manually in the way the tests work. 1. Create a new dir. E.g. `.local_docker_test` -2. Copy the original docker-compose file from directory `.docker` and remove everything except the qgis and the mssql container. -3. Copy the original Dockerfile as well. Leave it like it is... +2. Copy the original docker-compose file from directory `.docker` and remove everything except the qgis and the mssql container (remove the postgis dependency in qgis as well). 4. Copy the original `run-docker-tests.sh` and remove everything except: ```bash set -e @@ -36,13 +35,16 @@ These are dirty notes for the quickest way to test mssql queries manually in the /usr/src/tests/testdata/mssql/setup-mssql.sh ``` 5. Do the following: + Go to the local folder ```bash - docker-compose up -d - docker exec -it local_docker_tests_qgis_1 bash + export QGIS_TEST_VERSION=latest + export GITHUB_WORKSPACE=$PWD + docker-compose -f .local_docker_test/docker-compose.gh.yml up -d + docker exec -it local_docker_test_qgis_1 bash ``` And then in the qgis docker: ```bash - /usr/src/.local_docker_tests/run-docker-tests.sh; + /usr/src/.local_docker_test/run-docker-tests.sh java -jar /usr/src/modelbaker/iliwrapper/bin/ili2mssql-5.0.0/ili2mssql-5.0.0.jar --schemaimport --dbhost mssql --dbusr sa --dbpwd '' --dbdatabase gis --dbschema optimal_polymorph_manuel --coalesceCatalogueRef --createEnumTabs --createNumChecks --createUnique --createFk --createFkIdx --coalesceMultiSurface --coalesceMultiLine --coalesceMultiPoint --coalesceArray --beautifyEnumDispName --createGeomIdx --createMetaInfo --expandMultilingual --createTypeConstraint --createEnumTabsWithId --createTidCol --importTid --smart2Inheritance --strokeArcs --createBasketCol --defaultSrsAuth EPSG --defaultSrsCode 2056 --preScript NULL --postScript NULL --models Polymorphic_Ortsplanung_V1_1 --iliMetaAttrs NULL /usr/src/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili ``` (Surely this could be done without this qgis container, but there you have everything for the set up already...) From cdf008627a4d791fd74c310afe4221f36542913c Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 11 Oct 2024 10:44:42 +0200 Subject: [PATCH 4/6] fix mssql error and attachement key --- modelbaker/dbconnector/mssql_connector.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modelbaker/dbconnector/mssql_connector.py b/modelbaker/dbconnector/mssql_connector.py index c8f7fb5..713c7ce 100644 --- a/modelbaker/dbconnector/mssql_connector.py +++ b/modelbaker/dbconnector/mssql_connector.py @@ -940,7 +940,7 @@ def create_dataset(self, datasetname): return True, self.tr('Successfully created dataset "{}".').format( datasetname ) - except pyodbc.errors.UniqueViolation as e: + except pyodbc.UniqueViolation as e: return False, self.tr('Dataset with name "{}" already exists.').format( datasetname ) @@ -965,7 +965,7 @@ def rename_dataset(self, tid, datasetname): return True, self.tr('Successfully created dataset "{}".').format( datasetname ) - except pyodbc.errors.UniqueViolation as e: + except pyodbc.UniqueViolation as e: return False, self.tr('Dataset with name "{}" already exists.').format( datasetname ) @@ -1086,7 +1086,7 @@ def create_basket( cur.execute( """ INSERT INTO {schema}.{basket_table} ({tid_name}, dataset, topic, {tilitid_name}, attachmentkey ) - VALUES (NEXT VALUE FOR {schema}.{sequence}, {dataset_tid}, '{topic}', {tilitid}, {attachment_key}) + VALUES (NEXT VALUE FOR {schema}.{sequence}, {dataset_tid}, '{topic}', {tilitid}, '{attachment_key}') """.format( schema=self.schema, sequence="t_ili2db_seq", @@ -1103,7 +1103,7 @@ def create_basket( return True, self.tr( 'Successfully created basket for topic "{}".' ).format(topic) - except pyodbc.errors.Error as e: + except pyodbc.Error as e: error_message = " ".join(e.args) return False, self.tr( 'Could not create basket for topic "{}": {}' @@ -1139,7 +1139,7 @@ def edit_basket(self, basket_config: dict) -> tuple[bool, str]: return True, self.tr( 'Successfully edited basket for topic "{}" and dataset "{}".' ).format(basket_config["topic"], basket_config["datasetname"]) - except pyodbc.errors.Error as e: + except pyodbc.Error as e: error_message = " ".join(e.args) return False, self.tr( 'Could not edit basket for topic "{}" and dataset "{}": {}' @@ -1215,7 +1215,7 @@ def set_ili2db_sequence_value(self, value): return True, self.tr( 'Successfully reset sequence value to "{}".' ).format(value) - except pyodbc.errors.Error as e: + except pyodbc.Error as e: error_message = " ".join(e.args) return False, self.tr("Could not reset sequence: {}").format( error_message From 0aebe165bd28788503ba87fd94ed1dec649a488d Mon Sep 17 00:00:00 2001 From: signedav Date: Fri, 11 Oct 2024 21:25:18 +0200 Subject: [PATCH 5/6] uppercase all gpkg ili2db tables - no idea why it's needed but it's needed. --- modelbaker/dbconnector/gpkg_connector.py | 50 ++++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/modelbaker/dbconnector/gpkg_connector.py b/modelbaker/dbconnector/gpkg_connector.py index 589851e..0dbd18b 100644 --- a/modelbaker/dbconnector/gpkg_connector.py +++ b/modelbaker/dbconnector/gpkg_connector.py @@ -212,12 +212,12 @@ def _get_tables_info(self): interlis_joins = """LEFT JOIN T_ILI2DB_TABLE_PROP p ON p.tablename = s.name AND p.tag = 'ch.ehi.ili2db.tableKind' - LEFT JOIN t_ili2db_table_prop alias + LEFT JOIN T_ILI2DB_TABLE_PROP alias ON alias.tablename = s.name AND alias.tag = 'ch.ehi.ili2db.dispName' - LEFT JOIN t_ili2db_classname c + LEFT JOIN T_ILI2DB_CLASSNAME c ON s.name == c.sqlname - LEFT JOIN t_ili2db_attrname attrs + LEFT JOIN T_ILI2DB_ATTRNAME attrs ON c.iliname = attrs.iliname """ try: cursor.execute( @@ -336,7 +336,7 @@ def get_fields_info(self, table_name): cursor.execute( """ SELECT * - FROM t_ili2db_column_prop + FROM T_ILI2DB_COLUMN_PROP WHERE tablename = ?;""", (table_name,), ) @@ -347,7 +347,7 @@ def get_fields_info(self, table_name): cursor.execute( """ SELECT SqlName, IliName - FROM t_ili2db_attrname + FROM T_ILI2DB_ATTRNAME WHERE owner = ?;""", (table_name,), ) @@ -355,7 +355,7 @@ def get_fields_info(self, table_name): cursor.execute( """ SELECT SqlName, IliName - FROM t_ili2db_attrname + FROM T_ILI2DB_ATTRNAME WHERE colowner = ?;""", (table_name,), ) @@ -463,7 +463,7 @@ def get_t_type_map_info(self, table_name): cursor.execute( """ SELECT * - FROM t_ili2db_column_prop + FROM T_ILI2DB_COLUMN_PROP WHERE tablename = ? AND tag = 'ch.ehi.ili2db.types';""", (table_name,), @@ -508,8 +508,8 @@ def get_relations_info(self, filter_layer_list=[]): # Get strength cursor.execute( """SELECT META_ATTRS.attr_value as strength - FROM t_ili2db_attrname AS ATTRNAME - INNER JOIN t_ili2db_meta_attrs AS META_ATTRS + FROM T_ILI2DB_ATTRNAME AS ATTRNAME + INNER JOIN T_ILI2DB_META_ATTRS AS META_ATTRS ON META_ATTRS.ilielement = ATTRNAME.iliname AND META_ATTRS.attr_name = 'ili2db.ili.assocKind' WHERE ATTRNAME.sqlname = ? AND ATTRNAME.{colowner} = ? AND ATTRNAME.target = ?; """.format( @@ -529,8 +529,8 @@ def get_relations_info(self, filter_layer_list=[]): # Get cardinality max cursor.execute( """SELECT META_ATTRS.attr_value as cardinality_max - FROM t_ili2db_attrname AS ATTRNAME - INNER JOIN t_ili2db_meta_attrs AS META_ATTRS + FROM T_ILI2DB_ATTRNAME AS ATTRNAME + INNER JOIN T_ILI2DB_META_ATTRS AS META_ATTRS ON META_ATTRS.ilielement = ATTRNAME.iliname AND META_ATTRS.attr_name = 'ili2db.ili.attrCardinalityMax' WHERE ATTRNAME.sqlname = ? AND ATTRNAME.{colowner} = ? AND ATTRNAME.target = ?; """.format( @@ -561,14 +561,14 @@ def get_bags_of_info(self): cursor.execute( """SELECT cprop.tablename as current_layer_name, cprop.columnname as attribute, cprop.setting as target_layer_name, meta_attrs_cardinality_min.attr_value as cardinality_min, meta_attrs_cardinality_max.attr_value as cardinality_max - FROM t_ili2db_column_prop as cprop - LEFT JOIN t_ili2db_classname as cname + FROM T_ILI2DB_COLUMN_PROP as cprop + LEFT JOIN T_ILI2DB_CLASSNAME as cname ON cname.sqlname = cprop.tablename - LEFT JOIN t_ili2db_meta_attrs as meta_attrs_array + LEFT JOIN T_ILI2DB_META_ATTRS as meta_attrs_array ON LOWER(meta_attrs_array.ilielement) = LOWER(cname.iliname||'.'||cprop.columnname) AND meta_attrs_array.attr_name = 'ili2db.mapping' - LEFT JOIN t_ili2db_meta_attrs as meta_attrs_cardinality_min + LEFT JOIN T_ILI2DB_META_ATTRS as meta_attrs_cardinality_min ON LOWER(meta_attrs_cardinality_min.ilielement) = LOWER(cname.iliname||'.'||cprop.columnname) AND meta_attrs_cardinality_min.attr_name = 'ili2db.ili.attrCardinalityMin' - LEFT JOIN t_ili2db_meta_attrs as meta_attrs_cardinality_max + LEFT JOIN T_ILI2DB_META_ATTRS as meta_attrs_cardinality_max ON LOWER(meta_attrs_cardinality_max.ilielement) = LOWER(cname.iliname||'.'||cprop.columnname) AND meta_attrs_cardinality_max.attr_name = 'ili2db.ili.attrCardinalityMax' WHERE cprop.tag = 'ch.ehi.ili2db.foreignKey' AND meta_attrs_array.attr_value = 'ARRAY' """ @@ -596,7 +596,7 @@ def get_iliname_dbname_mapping(self, sqlnames=list()): cursor.execute( """SELECT iliname, sqlname - FROM t_ili2db_classname + FROM T_ILI2DB_CLASSNAME {where} """.format( where=where @@ -616,7 +616,7 @@ def get_classili_classdb_mapping(self, models_info, extended_classes): ) cursor.execute( """SELECT * - FROM t_ili2db_classname + FROM T_ILI2DB_CLASSNAME WHERE iliname IN ({class_names}) """.format( class_names=class_names @@ -630,7 +630,7 @@ def get_attrili_attrdb_mapping(self, attrs_list): attr_names = "'" + "','".join(attrs_list) + "'" cursor.execute( """SELECT iliname, sqlname, owner - FROM t_ili2db_attrname + FROM T_ILI2DB_ATTRNAME WHERE iliname IN ({attr_names}) """.format( attr_names=attr_names @@ -644,7 +644,7 @@ def get_attrili_attrdb_mapping_by_owner(self, owners): owner_names = "'" + "','".join(owners) + "'" cursor.execute( """SELECT iliname, sqlname, owner - FROM t_ili2db_attrname + FROM T_ILI2DB_ATTRNAME WHERE owner IN ({owner_names}) """.format( owner_names=owner_names @@ -653,7 +653,7 @@ def get_attrili_attrdb_mapping_by_owner(self, owners): return cursor def get_models(self): - if not self._table_exists("t_ili2db_trafo"): + if not self._table_exists("T_ILI2DB_TRAFO"): return {} # Get MODELS @@ -661,14 +661,14 @@ def get_models(self): cursor.execute( """SELECT distinct substr(iliname, 1, pos-1) AS modelname from - (SELECT *, instr(iliname,'.') AS pos FROM t_ili2db_trafo)""" + (SELECT *, instr(iliname,'.') AS pos FROM T_ILI2DB_TRAFO)""" ) models = cursor.fetchall() cursor.execute( """SELECT modelname, content - FROM t_ili2db_model""" + FROM T_ILI2DB_MODEL""" ) contents = cursor.fetchall() @@ -696,13 +696,13 @@ def get_models(self): def ili_version(self): cursor = self.conn.cursor() - cursor.execute("""PRAGMA table_info(t_ili2db_attrname)""") + cursor.execute("""PRAGMA table_info(T_ILI2DB_ATTRNAME)""") table_info = cursor.fetchall() result = 0 for column_info in table_info: if column_info[1] == "Owner": result += 1 - cursor.execute("""PRAGMA table_info(t_ili2db_model)""") + cursor.execute("""PRAGMA table_info(T_ILI2DB_MODEL)""") table_info = cursor.fetchall() for column_info in table_info: if column_info[1] == "file": From efa682171b54f88df088b00179186961321cea98 Mon Sep 17 00:00:00 2001 From: signedav Date: Tue, 15 Oct 2024 12:35:52 +0200 Subject: [PATCH 6/6] fix numbers in readme md --- tests/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index b6af7cb..7a9adc8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -28,13 +28,13 @@ These are dirty notes for the quickest way to test mssql queries manually in the 1. Create a new dir. E.g. `.local_docker_test` 2. Copy the original docker-compose file from directory `.docker` and remove everything except the qgis and the mssql container (remove the postgis dependency in qgis as well). -4. Copy the original `run-docker-tests.sh` and remove everything except: +3. Copy the original `run-docker-tests.sh` and remove everything except: ```bash set -e /usr/src/tests/testdata/mssql/setup-mssql.sh ``` -5. Do the following: +4. Do the following: Go to the local folder ```bash export QGIS_TEST_VERSION=latest @@ -48,7 +48,7 @@ These are dirty notes for the quickest way to test mssql queries manually in the java -jar /usr/src/modelbaker/iliwrapper/bin/ili2mssql-5.0.0/ili2mssql-5.0.0.jar --schemaimport --dbhost mssql --dbusr sa --dbpwd '' --dbdatabase gis --dbschema optimal_polymorph_manuel --coalesceCatalogueRef --createEnumTabs --createNumChecks --createUnique --createFk --createFkIdx --coalesceMultiSurface --coalesceMultiLine --coalesceMultiPoint --coalesceArray --beautifyEnumDispName --createGeomIdx --createMetaInfo --expandMultilingual --createTypeConstraint --createEnumTabsWithId --createTidCol --importTid --smart2Inheritance --strokeArcs --createBasketCol --defaultSrsAuth EPSG --defaultSrsCode 2056 --preScript NULL --postScript NULL --models Polymorphic_Ortsplanung_V1_1 --iliMetaAttrs NULL /usr/src/tests/testdata/ilimodels/Polymorphic_Ortsplanung_V1_1.ili ``` (Surely this could be done without this qgis container, but there you have everything for the set up already...) -6. Now connect (with eg. the VS Code extension) with these params: +5. Now connect (with eg. the VS Code extension) with these params: ``` localhost 1433