From b39fb9a74f78583e2de0386697de379a4209f788 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 10 Jul 2024 13:59:06 +0300 Subject: [PATCH 1/8] first commit for the CKAN 2.10 upgrade --- README.md | 2 ++ ckanext/multilang/harvesters/multilang.py | 4 ++-- setup.py | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 797f99b..63c98c9 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ To install ckanext-multilang: cd ckanext-multilang pip install -e . + + python setup.py develop 3. Initialize the DB with the mandatory Tables needed for localized records: diff --git a/ckanext/multilang/harvesters/multilang.py b/ckanext/multilang/harvesters/multilang.py index 3b734e1..110c12e 100644 --- a/ckanext/multilang/harvesters/multilang.py +++ b/ckanext/multilang/harvesters/multilang.py @@ -5,7 +5,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.search as search -from ckan.lib.base import model +from ckan import model from ckan.common import config from ckan.model import Package, Session from ckan.plugins.core import SingletonPlugin @@ -16,7 +16,7 @@ TagMultilang, ) from ckanext.spatial.harvesters.csw import CSWHarvester -from ckanext.spatial.model import ( +from ckanext.spatial.harvested_metadata import ( ISODocument, ISOElement, ISOKeyword, diff --git a/setup.py b/setup.py index 8f6b07c..c7ae3ea 100644 --- a/setup.py +++ b/setup.py @@ -64,9 +64,10 @@ # project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files - install_requires=[ - 'ckan>=2.9', - ], + # Uncomment the following three lines if CKAN isn't isntalled in your system + #install_requires=[ + # 'ckan>=2.9', + #], python_requires=">=3.7", # If there are data files included in your packages that need to be From f6e1806141d545f20c2e3ea856b35252bb13af44 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 10 Jul 2024 18:07:02 +0300 Subject: [PATCH 2/8] fixing the test.yml file to align with the CKAN 2.10 version --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3c79b3..7b52c02 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.10 - name: Install dependencies run: pip install flake8 pytest - name: Lint with flake8 @@ -21,7 +21,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: [2.9] + ckan-version: [2.10] fail-fast: false name: CKAN ${{ matrix.ckan-version }} From ce721df439bf677dfedcfe4a652f858e943d19bf Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 11 Jul 2024 09:18:33 +0300 Subject: [PATCH 3/8] fixing test.yml --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b52c02..f1c41d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,9 +5,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: "3.10" - name: Install dependencies run: pip install flake8 pytest - name: Lint with flake8 From 2d6ec4e7115d26e9809e4ccf17354f8d418faef5 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 11 Jul 2024 09:22:25 +0300 Subject: [PATCH 4/8] fixing the CKAN version in test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f1c41d6..a041636 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: [2.10] + ckan-version: ["2.10"] fail-fast: false name: CKAN ${{ matrix.ckan-version }} From d13b2c2316462aa08baec1a6e72b2426add90935 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 11 Jul 2024 10:12:47 +0300 Subject: [PATCH 5/8] fixing the test.yml file --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a041636..ed5630b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: image: openknowledge/ckan-dev:${{ matrix.ckan-version }} services: solr: - image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 postgres: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: From f1d4f09a0f0ab45ebc572ac4311e1726e9662032 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 11 Jul 2024 10:37:29 +0300 Subject: [PATCH 6/8] fixing test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed5630b..8267921 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,7 +69,7 @@ jobs: run: | crudini --set --verbose --list --list-sep=\ test.ini app:main ckan.plugins harvest ckan_harvester ckan -c test.ini db init - ckan -c test.ini harvester initdb + ckan -c test.ini db upgrade -p harvest - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.multilang --cov-report=xml --cov-append --disable-warnings ckanext/multilang/tests From dbfce499ad93c12a4f9bcf9765013c063ecb08ab Mon Sep 17 00:00:00 2001 From: gpetrak Date: Thu, 11 Jul 2024 12:55:17 +0300 Subject: [PATCH 7/8] removing depracated functions for ckanext-harvest and ckanext-spatial were used for tests --- ckanext/multilang/tests/conftest.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/ckanext/multilang/tests/conftest.py b/ckanext/multilang/tests/conftest.py index b83c154..ad4d998 100644 --- a/ckanext/multilang/tests/conftest.py +++ b/ckanext/multilang/tests/conftest.py @@ -3,8 +3,9 @@ import pytest from ckan.tests.pytest_ckan.fixtures import clean_db -from ckanext.harvest.tests.fixtures import harvest_setup -from ckanext.spatial.tests.conftest import clean_postgis, spatial_setup +# the following functions doesn't exist in the new versions of ckanext-harvest and ckanext-spatial +# from ckanext.harvest.tests.fixtures import harvest_setup +# from ckanext.spatial.tests.conftest import clean_postgis, spatial_setup from ckanext.multilang.model import setup_db as multilang_setup_db @@ -12,7 +13,18 @@ def multilang_setup(): multilang_setup_db() +@pytest.fixture +def clean_multilang_db(clean_db, multilang_setup): + return [ + # clean_postgis, + clean_db, + # clean_index() + # harvest_setup, + # spatial_setup, + multilang_setup, + ] +''' @pytest.fixture def clean_multilang_db(clean_postgis, clean_db, harvest_setup, spatial_setup, multilang_setup): return [ @@ -23,3 +35,4 @@ def clean_multilang_db(clean_postgis, clean_db, harvest_setup, spatial_setup, mu spatial_setup, multilang_setup, ] +''' From 4b3be9c92c35fec753edd4234d1615d7773daa83 Mon Sep 17 00:00:00 2001 From: gpetrak Date: Wed, 17 Jul 2024 16:02:21 +0300 Subject: [PATCH 8/8] fixing MultilangHarvester to use the iso_values values instead of ckan_locales_mapping --- ckanext/multilang/harvesters/multilang.py | 70 +++++++++++++---------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/ckanext/multilang/harvesters/multilang.py b/ckanext/multilang/harvesters/multilang.py index 110c12e..0d6eda4 100644 --- a/ckanext/multilang/harvesters/multilang.py +++ b/ckanext/multilang/harvesters/multilang.py @@ -160,24 +160,27 @@ def get_package_dict(self, iso_values, harvest_object): package_dict = super(MultilangHarvester, self).get_package_dict(iso_values, harvest_object) self._package_dict = {} - + harvester_config = self.source_config.get('ckan_locales_mapping', {}) if harvester_config: self._ckan_locales_mapping = harvester_config log.info('::::: ckan_locales_mapping entry found in harvester configuration :::::') - + if iso_values['abstract-text'] and iso_values['title-text']: log.debug('::::: Collecting localised data from the metadata abstract :::::') localised_abstracts = [] for abstract_entry in iso_values['abstract-text']: if abstract_entry['text'] and abstract_entry['locale'].lower()[1:]: - if self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]: - localised_abstracts.append({ - 'text': abstract_entry['text'], - 'locale': self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for metadata abstract, entry skipped!') + # We comment out this if because abstract_entry['locale'] is equal with the CKAN + # definition so we don't need the ckan_locales_mapping for now + # if self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]: + localised_abstracts.append({ + 'text': abstract_entry['text'], + 'locale': abstract_entry['locale'].lower()[1:] + # 'locale': self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]] + }) + #else: + # log.warning('Locale Mapping not found for metadata abstract, entry skipped!') else: log.warning('TextGroup data not available for metadata abstract, entry skipped!') @@ -185,18 +188,21 @@ def get_package_dict(self, iso_values, harvest_object): localised_titles = [] for title_entry in iso_values['title-text']: if title_entry['text'] and title_entry['locale'].lower()[1:]: - if self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]: - localised_titles.append({ - 'text': title_entry['text'], - 'locale': self._ckan_locales_mapping[title_entry['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for metadata title, entry skipped!') + # the same as above + # if self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]: + localised_titles.append({ + 'text': title_entry['text'], + # 'locale': self._ckan_locales_mapping[title_entry['locale'].lower()[1:]] + 'locale': title_entry['locale'].lower()[1:] + }) + #else: + # log.warning('Locale Mapping not found for metadata title, entry skipped!') else: log.warning('TextGroup data not available for metadata title, entry skipped!') localised_titles.append({ 'text': iso_values['title'], + # because iso_values['metadata-language']=ita we remove the last letter 'locale': self._ckan_locales_mapping[iso_values['metadata-language'].lower()] }) @@ -231,14 +237,15 @@ def get_package_dict(self, iso_values, harvest_object): for tag_localized in tag_localized_entry: if tag_localized['text'] and tag_localized['locale'].lower()[1:]: - if self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]: - localized_tags.append({ - 'text': tag_localized['name'], - 'localized_text': tag_localized['text'], - 'locale': self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for metadata keyword: %r, entry skipped!', tag_localized['name']) + # if self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]: + localized_tags.append({ + 'text': tag_localized['name'], + 'localized_text': tag_localized['text'], + # 'locale': self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]] + 'locale': tag_localized['locale'].lower()[1:] + }) + #else: + # log.warning('Locale Mapping not found for metadata keyword: %r, entry skipped!', tag_localized['name']) else: log.warning('TextGroup data not available for metadata keyword: %r, entry skipped!', tag_localized['name']) @@ -283,13 +290,14 @@ def handle_organization(self, harvest_object, organisation_mapping, values): for entry in party['organisation-name-localized']: if entry['text'] and entry['locale'].lower()[1:]: - if self._ckan_locales_mapping[entry['locale'].lower()[1:]]: - localized_org.append({ - 'text': org.get('value_' + self._ckan_locales_mapping[entry['locale'].lower()[1:]]) or entry['text'], - 'locale': self._ckan_locales_mapping[entry['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for organization name, entry skipped!') + # if self._ckan_locales_mapping[entry['locale'].lower()[1:]]: + localized_org.append({ + 'text': org.get('value_' + self._ckan_locales_mapping[entry['locale'].lower()[1:]]) or entry['text'], + # 'locale': self._ckan_locales_mapping[entry['locale'].lower()[1:]] + 'locale': entry['locale'].lower()[1:] + }) + #else: + # log.warning('Locale Mapping not found for organization name, entry skipped!') else: log.warning('TextGroup data not available for organization name, entry skipped!')