diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b3c79b3..8267921 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,9 +5,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 with: - python-version: 3.9 + python-version: "3.10" - name: Install dependencies run: pip install flake8 pytest - name: Lint with flake8 @@ -21,7 +21,7 @@ jobs: needs: lint strategy: matrix: - ckan-version: [2.9] + ckan-version: ["2.10"] fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -30,7 +30,7 @@ jobs: image: openknowledge/ckan-dev:${{ matrix.ckan-version }} services: solr: - image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 postgres: image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} env: @@ -69,7 +69,7 @@ jobs: run: | crudini --set --verbose --list --list-sep=\ test.ini app:main ckan.plugins harvest ckan_harvester ckan -c test.ini db init - ckan -c test.ini harvester initdb + ckan -c test.ini db upgrade -p harvest - name: Run tests run: pytest --ckan-ini=test.ini --cov=ckanext.multilang --cov-report=xml --cov-append --disable-warnings ckanext/multilang/tests diff --git a/README.md b/README.md index 797f99b..63c98c9 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ To install ckanext-multilang: cd ckanext-multilang pip install -e . + + python setup.py develop 3. Initialize the DB with the mandatory Tables needed for localized records: diff --git a/ckanext/multilang/harvesters/multilang.py b/ckanext/multilang/harvesters/multilang.py index 3b734e1..0d6eda4 100644 --- a/ckanext/multilang/harvesters/multilang.py +++ b/ckanext/multilang/harvesters/multilang.py @@ -5,7 +5,7 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.search as search -from ckan.lib.base import model +from ckan import model from ckan.common import config from ckan.model import Package, Session from ckan.plugins.core import SingletonPlugin @@ -16,7 +16,7 @@ TagMultilang, ) from ckanext.spatial.harvesters.csw import CSWHarvester -from ckanext.spatial.model import ( +from ckanext.spatial.harvested_metadata import ( ISODocument, ISOElement, ISOKeyword, @@ -160,24 +160,27 @@ def get_package_dict(self, iso_values, harvest_object): package_dict = super(MultilangHarvester, self).get_package_dict(iso_values, harvest_object) self._package_dict = {} - + harvester_config = self.source_config.get('ckan_locales_mapping', {}) if harvester_config: self._ckan_locales_mapping = harvester_config log.info('::::: ckan_locales_mapping entry found in harvester configuration :::::') - + if iso_values['abstract-text'] and iso_values['title-text']: log.debug('::::: Collecting localised data from the metadata abstract :::::') localised_abstracts = [] for abstract_entry in iso_values['abstract-text']: if abstract_entry['text'] and abstract_entry['locale'].lower()[1:]: - if self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]: - localised_abstracts.append({ - 'text': abstract_entry['text'], - 'locale': self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for metadata abstract, entry skipped!') + # We comment out this if because abstract_entry['locale'] is equal with the CKAN + # definition so we don't need the ckan_locales_mapping for now + # if self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]]: + localised_abstracts.append({ + 'text': abstract_entry['text'], + 'locale': abstract_entry['locale'].lower()[1:] + # 'locale': self._ckan_locales_mapping[abstract_entry['locale'].lower()[1:]] + }) + #else: + # log.warning('Locale Mapping not found for metadata abstract, entry skipped!') else: log.warning('TextGroup data not available for metadata abstract, entry skipped!') @@ -185,18 +188,21 @@ def get_package_dict(self, iso_values, harvest_object): localised_titles = [] for title_entry in iso_values['title-text']: if title_entry['text'] and title_entry['locale'].lower()[1:]: - if self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]: - localised_titles.append({ - 'text': title_entry['text'], - 'locale': self._ckan_locales_mapping[title_entry['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for metadata title, entry skipped!') + # the same as above + # if self._ckan_locales_mapping[title_entry['locale'].lower()[1:]]: + localised_titles.append({ + 'text': title_entry['text'], + # 'locale': self._ckan_locales_mapping[title_entry['locale'].lower()[1:]] + 'locale': title_entry['locale'].lower()[1:] + }) + #else: + # log.warning('Locale Mapping not found for metadata title, entry skipped!') else: log.warning('TextGroup data not available for metadata title, entry skipped!') localised_titles.append({ 'text': iso_values['title'], + # because iso_values['metadata-language']=ita we remove the last letter 'locale': self._ckan_locales_mapping[iso_values['metadata-language'].lower()] }) @@ -231,14 +237,15 @@ def get_package_dict(self, iso_values, harvest_object): for tag_localized in tag_localized_entry: if tag_localized['text'] and tag_localized['locale'].lower()[1:]: - if self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]: - localized_tags.append({ - 'text': tag_localized['name'], - 'localized_text': tag_localized['text'], - 'locale': self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for metadata keyword: %r, entry skipped!', tag_localized['name']) + # if self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]]: + localized_tags.append({ + 'text': tag_localized['name'], + 'localized_text': tag_localized['text'], + # 'locale': self._ckan_locales_mapping[tag_localized['locale'].lower()[1:]] + 'locale': tag_localized['locale'].lower()[1:] + }) + #else: + # log.warning('Locale Mapping not found for metadata keyword: %r, entry skipped!', tag_localized['name']) else: log.warning('TextGroup data not available for metadata keyword: %r, entry skipped!', tag_localized['name']) @@ -283,13 +290,14 @@ def handle_organization(self, harvest_object, organisation_mapping, values): for entry in party['organisation-name-localized']: if entry['text'] and entry['locale'].lower()[1:]: - if self._ckan_locales_mapping[entry['locale'].lower()[1:]]: - localized_org.append({ - 'text': org.get('value_' + self._ckan_locales_mapping[entry['locale'].lower()[1:]]) or entry['text'], - 'locale': self._ckan_locales_mapping[entry['locale'].lower()[1:]] - }) - else: - log.warning('Locale Mapping not found for organization name, entry skipped!') + # if self._ckan_locales_mapping[entry['locale'].lower()[1:]]: + localized_org.append({ + 'text': org.get('value_' + self._ckan_locales_mapping[entry['locale'].lower()[1:]]) or entry['text'], + # 'locale': self._ckan_locales_mapping[entry['locale'].lower()[1:]] + 'locale': entry['locale'].lower()[1:] + }) + #else: + # log.warning('Locale Mapping not found for organization name, entry skipped!') else: log.warning('TextGroup data not available for organization name, entry skipped!') diff --git a/ckanext/multilang/tests/conftest.py b/ckanext/multilang/tests/conftest.py index b83c154..ad4d998 100644 --- a/ckanext/multilang/tests/conftest.py +++ b/ckanext/multilang/tests/conftest.py @@ -3,8 +3,9 @@ import pytest from ckan.tests.pytest_ckan.fixtures import clean_db -from ckanext.harvest.tests.fixtures import harvest_setup -from ckanext.spatial.tests.conftest import clean_postgis, spatial_setup +# the following functions doesn't exist in the new versions of ckanext-harvest and ckanext-spatial +# from ckanext.harvest.tests.fixtures import harvest_setup +# from ckanext.spatial.tests.conftest import clean_postgis, spatial_setup from ckanext.multilang.model import setup_db as multilang_setup_db @@ -12,7 +13,18 @@ def multilang_setup(): multilang_setup_db() +@pytest.fixture +def clean_multilang_db(clean_db, multilang_setup): + return [ + # clean_postgis, + clean_db, + # clean_index() + # harvest_setup, + # spatial_setup, + multilang_setup, + ] +''' @pytest.fixture def clean_multilang_db(clean_postgis, clean_db, harvest_setup, spatial_setup, multilang_setup): return [ @@ -23,3 +35,4 @@ def clean_multilang_db(clean_postgis, clean_db, harvest_setup, spatial_setup, mu spatial_setup, multilang_setup, ] +''' diff --git a/setup.py b/setup.py index 8f6b07c..c7ae3ea 100644 --- a/setup.py +++ b/setup.py @@ -64,9 +64,10 @@ # project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files - install_requires=[ - 'ckan>=2.9', - ], + # Uncomment the following three lines if CKAN isn't isntalled in your system + #install_requires=[ + # 'ckan>=2.9', + #], python_requires=">=3.7", # If there are data files included in your packages that need to be