Drop support for Python 2

.github/workflows/test.yml

@@ -21,17 +21,11 @@ jobs:
       matrix:
         include:
           - ckan-version: "2.10"
-            solr-image: "2.10-spatial"
-            requirements-file: 'requirements.txt'
+            solr-image: "2.10-solr9-spatial"
             harvester-version: 'master'
           - ckan-version: 2.9
-            solr-image: 2.9-solr8-spatial
-            requirements-file: 'requirements.txt'
+            solr-image: 2.9-solr9-spatial
             harvester-version: 'master'
-          - ckan-version: 2.9-py2
-            solr-image: 2.9-py2-solr8-spatial
-            requirements-file: 'requirements-py2.txt'
-            harvester-version: 'v1.4.2'
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }}
@@ -98,16 +92,10 @@ jobs:
         pip install cython==0.29.36
         pip install --no-use-pep517 pyproj==2.6.1
 
-    - name: Patch to test pyproj
-      if: ${{ matrix.ckan-version == '2.9-py2' }}
-      run: |
-        pip install cython==0.28.4
-        pip install --no-use-pep517 pyproj==2.2.2
-
-    - name: Install dependencies from ${{ matrix.requirements-file }}
+    - name: Install dependencies from requirements.txt
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
-        pip install -r ${{ matrix.requirements-file }}
+        pip install -r requirements.txt
 
     - name: Install harvester
       if: steps.cache.outputs.cache-hit != 'true'

bin/ckan_pycsw.py

@@ -4,7 +4,7 @@
 import io
 import os
 import argparse
-from six.moves.configparser import SafeConfigParser
+from configparser import SafeConfigParser
 
 import requests
 from lxml import etree

ckanext/spatial/harvested_metadata.py

@@ -1,5 +1,4 @@
 from lxml import etree
-import six
 
 import logging
 log = logging.getLogger(__name__)
@@ -38,7 +37,7 @@ def read_value(self, name):
     def get_xml_tree(self):
         if self.xml_tree is None:
             parser = etree.XMLParser(remove_blank_text=True)
-            xml_str = six.ensure_str(self.xml_str)
+            xml_str = str(self.xml_str)
             self.xml_tree = etree.fromstring(xml_str, parser=parser)
         return self.xml_tree
 

ckanext/spatial/harvesters/base.py

@@ -1,6 +1,5 @@
-import six
-from six.moves.urllib.parse import urlparse
-from six.moves.urllib.request import urlopen
+from urllib.parse import urlparse
+from urllib.request import urlopen
 
 import re
 import cgitb
@@ -33,7 +32,7 @@
 from ckanext.spatial.validation import Validators, all_validators
 from ckanext.spatial.harvested_metadata import ISODocument
 from ckanext.spatial.interfaces import ISpatialHarvester
-from ckantoolkit import config, unicode_safe
+from ckantoolkit import config
 
 log = logging.getLogger(__name__)
 
@@ -300,7 +299,7 @@ def get_package_dict(self, context, data_dict):
         if package is None or package.title != iso_values['title']:
             name = self._gen_new_name(iso_values['title'])
             if not name:
-                name = self._gen_new_name(six.text_type(iso_values['guid']))
+                name = self._gen_new_name(str(iso_values['guid']))
             if not name:
                 raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.')
             package_dict['name'] = name
@@ -414,7 +413,7 @@ def _extract_first_license_url(licences):
                 ymin = float(bbox['south'])
                 ymax = float(bbox['north'])
             except ValueError as e:
-                self._save_object_error('Error parsing bounding box value: {0}'.format(six.text_type(e)),
+                self._save_object_error('Error parsing bounding box value: {0}'.format(str(e)),
                                     harvest_object, 'Import')
             else:
                 # Construct a GeoJSON extent so ckanext-spatial can register the extent geometry
@@ -472,7 +471,7 @@ def _extract_first_license_url(licences):
               log.debug('Processing extra %s', key)
               if not key in extras or override_extras:
                  # Look for replacement strings
-                 if isinstance(value,six.string_types):
+                 if isinstance(value,str):
                     value = value.format(harvest_source_id=harvest_object.job.source.id,
                              harvest_source_url=harvest_object.job.source.url.strip('/'),
                              harvest_source_title=harvest_object.job.source.title,
@@ -576,7 +575,7 @@ def import_stage(self, harvest_object):
             iso_parser = ISODocument(harvest_object.content)
             iso_values = iso_parser.read_values()
         except Exception as e:
-            self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, six.text_type(e)),
+            self._save_object_error('Error parsing ISO document for object {0}: {1}'.format(harvest_object.id, str(e)),
                                     harvest_object, 'Import')
             return False
 
@@ -659,7 +658,7 @@ def import_stage(self, harvest_object):
 
             # We need to explicitly provide a package ID, otherwise ckanext-spatial
             # won't be be able to link the extent to the package.
-            package_dict['id'] = six.text_type(uuid.uuid4())
+            package_dict['id'] = str(uuid.uuid4())
             package_schema['id'] = [unicode_safe]
 
             # Save reference to the package on the object
@@ -675,7 +674,7 @@ def import_stage(self, harvest_object):
                 package_id = p.toolkit.get_action('package_create')(context, package_dict)
                 log.info('Created new package %s with guid %s', package_id, harvest_object.guid)
             except p.toolkit.ValidationError as e:
-                self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import')
+                self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import')
                 return False
 
         elif status == 'change':
@@ -721,7 +720,7 @@ def import_stage(self, harvest_object):
                     package_id = p.toolkit.get_action('package_update')(context, package_dict)
                     log.info('Updated package %s with guid %s', package_id, harvest_object.guid)
                 except p.toolkit.ValidationError as e:
-                    self._save_object_error('Validation Error: %s' % six.text_type(e.error_summary), harvest_object, 'Import')
+                    self._save_object_error('Validation Error: %s' % str(e.error_summary), harvest_object, 'Import')
                     return False
 
         model.Session.commit()
@@ -738,7 +737,7 @@ def _is_wms(self, url):
             s = wms.WebMapService(url)
             return isinstance(s.contents, dict) and s.contents != {}
         except Exception as e:
-            log.error('WMS check for %s failed with exception: %s' % (url, six.text_type(e)))
+            log.error('WMS check for %s failed with exception: %s' % (url, str(e)))
         return False
 
     def _get_object_extra(self, harvest_object, key):
@@ -881,7 +880,7 @@ def _validate_document(self, document_string, harvest_object, validator=None):
         try:
             xml = etree.fromstring(document_string)
         except etree.XMLSyntaxError as e:
-            self._save_object_error('Could not parse XML file: {0}'.format(six.text_type(e)), harvest_object, 'Import')
+            self._save_object_error('Could not parse XML file: {0}'.format(str(e)), harvest_object, 'Import')
             return False, None, []
 
         valid, profile, errors = validator.is_valid(xml)

ckanext/spatial/harvesters/csw.py

@@ -1,6 +1,5 @@
 import re
-import six
-from six.moves.urllib.parse import urlparse, urlunparse, urlencode
+from urllib.parse import urlparse, urlunparse, urlencode
 
 import logging
 
@@ -105,7 +104,7 @@ def gather_stage(self, harvest_job):
 
         except Exception as e:
             log.error('Exception: %s' % text_traceback())
-            self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job)
+            self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job)
             return None
 
         new = guids_in_harvest - guids_in_db

ckanext/spatial/harvesters/gemini.py

@@ -8,9 +8,8 @@
     - GeminiWafHarvester - An index page with links to GEMINI resources
 
 '''
-import six
 import os
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 from datetime import datetime
 from numbers import Number
 import uuid
@@ -24,6 +23,7 @@
 from ckan.model import Session, Package
 from ckan.lib.munge import munge_title_to_name
 from ckan.plugins.core import SingletonPlugin, implements
+from ckan.lib.navl.validators import unicode_safe
 from ckan.lib.helpers import json
 
 from ckan import logic
@@ -73,10 +73,10 @@ def import_stage(self, harvest_object):
             return True
         except Exception as e:
             log.error('Exception during import: %s' % text_traceback())
-            if not six.text_type(e).strip():
+            if not str(e).strip():
                 self._save_object_error('Error importing Gemini document.', harvest_object, 'Import')
             else:
-                self._save_object_error('Error importing Gemini document: %s' % six.text_type(e), harvest_object, 'Import')
+                self._save_object_error('Error importing Gemini document: %s' % str(e), harvest_object, 'Import')
             raise
             if debug_exception_mode:
                 raise
@@ -275,7 +275,7 @@ def write_package_from_gemini_string(self, content):
         if package is None or package.title != gemini_values['title']:
             name = self.gen_new_name(gemini_values['title'])
             if not name:
-                name = self.gen_new_name(six.text_type(gemini_guid))
+                name = self.gen_new_name(str(gemini_guid))
             if not name:
                 raise Exception('Could not generate a unique name from the title or the GUID. Please choose a more unique title.')
             package_dict['name'] = name
@@ -320,7 +320,7 @@ def write_package_from_gemini_string(self, content):
 
         extras_as_dict = []
         for key,value in extras.items():
-            if isinstance(value, six.string_types + (Number,)):
+            if isinstance(value, str + (Number,)):
                 extras_as_dict.append({'key':key,'value':value})
             else:
                 extras_as_dict.append({'key':key,'value':json.dumps(value)})
@@ -413,8 +413,8 @@ def gen_new_name(self, title):
         else:
             counter = 1
             while counter < 101:
-                if name+six.text_type(counter) not in taken:
-                    return name+six.text_type(counter)
+                if name+str(counter) not in taken:
+                    return name+str(counter)
                 counter = counter + 1
             return None
 
@@ -454,7 +454,7 @@ def _create_package_from_data(self, package_dict, package = None):
 
         # The default package schema does not like Upper case tags
         tag_schema = logic.schema.default_tags_schema()
-        tag_schema['name'] = [not_empty,six.text_type]
+        tag_schema['name'] = [not_empty, unicode_safe]
         package_schema['tags'] = tag_schema
 
         # TODO: user
@@ -467,8 +467,8 @@ def _create_package_from_data(self, package_dict, package = None):
         if not package:
             # We need to explicitly provide a package ID, otherwise ckanext-spatial
             # won't be be able to link the extent to the package.
-            package_dict['id'] = six.text_type(uuid.uuid4())
-            package_schema['id'] = [six.text_type]
+            package_dict['id'] = str(uuid.uuid4())
+            package_schema['id'] = [unicode_safe]
 
             action_function = get_action('package_create')
         else:
@@ -478,7 +478,7 @@ def _create_package_from_data(self, package_dict, package = None):
         try:
             package_dict = action_function(context, package_dict)
         except ValidationError as e:
-            raise Exception('Validation Error: %s' % six.text_type(e.error_summary))
+            raise Exception('Validation Error: %s' % str(e.error_summary))
             if debug_exception_mode:
                 raise
 
@@ -572,7 +572,7 @@ def gather_stage(self, harvest_job):
 
         except Exception as e:
             log.error('Exception: %s' % text_traceback())
-            self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % six.text_type(e), harvest_job)
+            self._save_gather_error('Error gathering the identifiers from the CSW server [%s]' % str(e), harvest_job)
             return None
 
         if len(ids) == 0:

ckanext/spatial/harvesters/waf.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 import logging
 import hashlib
 
@@ -98,7 +97,7 @@ def gather_stage(self,harvest_job,collection_package_id=None):
 
         url_to_modified_harvest = {} ## mapping of url to last_modified in harvest
         try:
-            for url, modified_date in _extract_waf(six.text_type(content),source_url,scraper):
+            for url, modified_date in _extract_waf(str(content),source_url,scraper):
                 url_to_modified_harvest[url] = modified_date
         except Exception as e:
             msg = 'Error extracting URLs from %s, error was %s' % (source_url, e)
@@ -316,16 +315,16 @@ def _extract_waf(content, base_url, scraper, results = None, depth=0):
                 response = requests.get(new_url)
                 content = response.content
             except Exception as e:
-                print(six.text_type(e))
+                print(str(e))
                 continue
-            _extract_waf(six.text_type(content), new_url, scraper, results, new_depth)
+            _extract_waf(str(content), new_url, scraper, results, new_depth)
             continue
         if not url.endswith('.xml'):
             continue
         date = record.date
         if date:
             try:
-                date = six.text_type(dateutil.parser.parse(date))
+                date = str(dateutil.parser.parse(date))
             except Exception as e:
                 raise
                 date = None

ckanext/spatial/lib/__init__.py

@@ -1,5 +1,4 @@
 import logging
-import six
 import ckantoolkit as tk
 
 config = tk.config
@@ -47,7 +46,7 @@ def normalize_bbox(bbox_values):
     If there are any problems parsing the input it returns None.
     """
 
-    if isinstance(bbox_values, six.string_types):
+    if isinstance(bbox_values, str):
         bbox_values = bbox_values.split(",")
 
     if len(bbox_values) != 4:

ckanext/spatial/lib/csw_client.py

@@ -2,7 +2,6 @@
 Some very thin wrapper classes around those in OWSLib
 for convenience.
 """
-import six
 import logging
 
 from owslib.etree import etree
@@ -33,7 +32,7 @@ def _xmd(self, obj):
                 pass
             elif callable(val):
                 pass
-            elif isinstance(val, six.string_types):
+            elif isinstance(val, str):
                 md[attr] = val
             elif isinstance(val, int):
                 md[attr] = val

ckanext/spatial/lib/report.py

@@ -3,7 +3,7 @@
 and then saved as a CSV.
 '''
 
-from six import text_type, StringIO
+from io import StringIO
 import datetime
 import csv
 
@@ -52,9 +52,9 @@ def get_csv(self):
                 if isinstance(cell, datetime.datetime):
                     cell = cell.strftime('%Y-%m-%d %H:%M')
                 elif isinstance(cell, int):
-                    cell = text_type(cell)
+                    cell = str(cell)
                 elif isinstance(cell, (list, tuple)):
-                    cell = text_type(cell)
+                    cell = str(cell)
                 elif cell is None:
                     cell = ''
                 else:

ckanext/spatial/plugin/__init__.py

@@ -2,7 +2,6 @@
 import mimetypes
 from logging import getLogger
 
-import six
 import geojson
 
 import shapely.geometry
@@ -104,10 +103,10 @@ def check_spatial_extra(self, dataset_dict, update=False):
         try:
             log.debug("Received geometry: {}".format(geometry))
 
-            geometry = geojson.loads(six.text_type(geometry))
+            geometry = geojson.loads(str(geometry))
         except ValueError as e:
             error_dict = {
-                "spatial": ["Error decoding JSON object: {}".format(six.text_type(e))]}
+                "spatial": ["Error decoding JSON object: {}".format(str(e))]}
             raise tk.ValidationError(error_dict)
 
         if not hasattr(geometry, "is_valid") or not geometry.is_valid: