Merge branch 'adobe-apiplatform:v2' into Luci2015-fix827

.changelog/changelog.md

@@ -1,3 +1,40 @@
+| tag        | date       | title                     |
+|------------|------------|---------------------------|
+| v2.10.0rc4 | 2024-08-13 | User Sync Tool v2.10.0rc4 |
+
+# Fixes
+
+* #825 send `start_sync()` signal when querying groups for auto-create purposes
+* #834 make TimeoutException a child of AssertionException so we don't show a stack trace
+* #837 fix typo in sign engine log message
+* #840 push strategy fails with unhandled exception
+* e61ec81 Fix issue with all users setting
+
+# New Features
+
+* 5e9e01b Ability to exclude Sign users in Sign Sync
+* d761c5e Introduce option to limit scope of Adobe-only users to just those that have groups to remove in the current sync. See [the manual](https://github.com/adobe-apiplatform/user-sync.py/blob/8082c987c79eddcc3fc06f31a1c32de300a30cd7/en/user-manual/configuring_user_sync_tool.md#limits-config) for more information
+
+---
+
+| tag        | date       | title                     |
+|------------|------------|---------------------------|
+| v2.10.0rc3 | 2024-07-29 | User Sync Tool v2.10.0rc3 |
+
+# Fixes
+
+* #825 send `start_sync()` signal when querying groups for auto-create purposes
+* #834 make TimeoutException a child of AssertionException so we don't show a stack trace
+* #837 fix typo in sign engine log message
+* #840 push strategy fails with unhandled exception
+* e61ec81 Fix issue with all users setting
+
+# New Features
+
+* d761c5e Introduce option to limit scope of Adobe-only users to just those that have groups to remove in the current sync. See [the manual](https://github.com/adobe-apiplatform/user-sync.py/blob/8082c987c79eddcc3fc06f31a1c32de300a30cd7/en/user-manual/configuring_user_sync_tool.md#limits-config) for more information
+
+---
+
 | tag        | date       | title                     |
 |------------|------------|---------------------------|
 | v2.10.0rc2 | 2023-10-19 | User Sync Tool v2.10.0rc2 |
@@ -8,10 +45,7 @@
 
 # New Features
 
-* d761c5e Introduce option to limit scope of Adobe-only users to just those that
-  have groups to remove in the current sync. See [the
-  manual](https://github.com/adobe-apiplatform/user-sync.py/blob/8082c987c79eddcc3fc06f31a1c32de300a30cd7/en/user-manual/configuring_user_sync_tool.md#limits-config)
-  for more information
+* d761c5e Introduce option to limit scope of Adobe-only users to just those that have groups to remove in the current sync. See [the manual](https://github.com/adobe-apiplatform/user-sync.py/blob/8082c987c79eddcc3fc06f31a1c32de300a30cd7/en/user-manual/configuring_user_sync_tool.md#limits-config) for more information
 
 ---
 

.changelog/latest.md

@@ -1,10 +1,12 @@
 # Fixes
 
+* #825 send `start_sync()` signal when querying groups for auto-create purposes
+* #834 make TimeoutException a child of AssertionException so we don't show a stack trace
+* #837 fix typo in sign engine log message
+* #840 push strategy fails with unhandled exception
 * e61ec81 Fix issue with all users setting
 
 # New Features
 
-* d761c5e Introduce option to limit scope of Adobe-only users to just those that
-  have groups to remove in the current sync. See [the
-  manual](https://github.com/adobe-apiplatform/user-sync.py/blob/8082c987c79eddcc3fc06f31a1c32de300a30cd7/en/user-manual/configuring_user_sync_tool.md#limits-config)
-  for more information
+* 5e9e01b Ability to exclude Sign users in Sign Sync
+* d761c5e Introduce option to limit scope of Adobe-only users to just those that have groups to remove in the current sync. See [the manual](https://github.com/adobe-apiplatform/user-sync.py/blob/8082c987c79eddcc3fc06f31a1c32de300a30cd7/en/user-manual/configuring_user_sync_tool.md#limits-config) for more information

.github/workflows/package.yml

@@ -16,11 +16,11 @@ jobs:
           - extension_support: 1
             variant_tag: ""
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Get version tag
       id: get_version
       uses: battila7/get-version-action@v2
@@ -67,11 +67,11 @@ jobs:
           - extension_support: 1
             variant_tag: ""
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: Get version tag
       id: get_version
       uses: battila7/get-version-action@v2
@@ -116,10 +116,10 @@ jobs:
           - extension_support: 1
             variant_tag: ""
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: Get version tag
       id: get_version
       uses: battila7/get-version-action@v2
@@ -143,39 +143,54 @@ jobs:
         path: dist/*.zip
         retention-days: 30
 
-  centos:
+  rocky:
     runs-on: ubuntu-latest
     env:
       LC_ALL: en_US.utf8
       LANG: en_US.utf8
     container:
-      image: centos:7
+      image: ${{ matrix.container }}
     strategy:
       matrix:
         include:
-          - extension_support: 0
+          - container: "rockylinux:8"
+            extension_support: 0
             variant_tag: "-noext"
-          - extension_support: 1
+          - container: "rockylinux:8"
+            extension_support: 1
+            variant_tag: ""
+          - container: "rockylinux:9"
+            extension_support: 0
+            variant_tag: "-noext"
+          - container: "rockylinux:9"
+            extension_support: 1
             variant_tag: ""
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Get version tag
         id: get_version
         uses: battila7/get-version-action@v2
-      - name: Centos-Install dependencies
+      - name: Update yum dependencies
+        run: yum update -y
+      - name: Rocky 8 dependencies
+        if: ${{ matrix.container == 'rockylinux:8' }}
+        run: yum install -y pkgconfig gcc openssl-devel dbus-glib-devel python3-dbus sqlite-devel libffi-devel wget make findutils glibc-langpack-en
+      - name: Rocky 9 dependencies
+        if: ${{ matrix.container == 'rockylinux:9' }}
+        run: yum install -y pkgconfig gcc openssl-devel dbus-glib-devel python3-dbus sqlite-devel libffi-devel krb5-devel dbus-daemon wget make findutils glibc-langpack-en
+      - name: Build Python 3.12
         run: |
-          yum install -y pkgconfig gcc openssl-devel dbus-glib-devel dbus-python sqlite-devel libffi-devel wget make
           mkdir tmp && cd tmp
-          wget https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz
-          tar xzf Python-3.9.7.tgz
-          cd Python-3.9.7
+          wget https://www.python.org/ftp/python/3.12.4/Python-3.12.4.tgz
+          tar xzf Python-3.12.4.tgz
+          cd Python-3.12.4
           ./configure --enable-optimizations --enable-shared --enable-loadable-sqlite-extensions
           make altinstall
           cd ../..
       - name: Build executable
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
-          /usr/local/bin/python3.9 -m venv venv
+          /usr/local/bin/python3.12 -m venv venv
           source venv/bin/activate
           # for some reason, pycryptodome won't install when
           # setuptools is installed/upgraded, so install it before
@@ -194,21 +209,23 @@ jobs:
       - name: Get build
         run: |
           cd dist
-          tar czf "user-sync-${UST_VERSION}${UST_VARIANT_TAG}-centos.tar.gz" user-sync
+          LABEL=$(echo $CONTAINER | sed "s/linux://g")
+          tar czf "user-sync-${UST_VERSION}${UST_VARIANT_TAG}-${LABEL}.tar.gz" user-sync
         env:
           UST_VARIANT_TAG: ${{matrix.variant_tag}}
           UST_VERSION: ${{ steps.get_version.outputs.version }}
+          CONTAINER: ${{ matrix.container }}
       - name: Make artifact
         uses: actions/upload-artifact@v2
         with:
-          name: centos
+          name: rocky
           path: dist/*.tar.gz
           retention-days: 30
 
   examples:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: user-sync-examples-zip
       run: |
          cd examples
@@ -229,9 +246,9 @@ jobs:
 
   release:
     runs-on: ubuntu-latest
-    needs: [centos, ubuntu-jammy, ubuntu-focal, examples, windows]
+    needs: [rocky, ubuntu-jammy, ubuntu-focal, examples, windows]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Create Release
       id: create_release
       uses: actions/create-release@v1
@@ -254,6 +271,6 @@ jobs:
       uses: battila7/get-version-action@v2
     - uses: AButler/upload-release-assets@v2.0
       with:
-        files: 'examples/*;ubuntu-jammy/*;ubuntu-focal/*;centos/*'
+        files: 'examples/*;ubuntu-jammy/*;ubuntu-focal/*;rocky/*'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         release-tag: ${{ steps.get_version.outputs.version }}

setup.py

@@ -57,6 +57,7 @@
           'configparser==3.7.4',
           'schema==0.7.2',
           'sign-client~=0.2.1',
+          'setuptools~=72.1.0',
       ],
       extras_require={
           ':sys_platform=="linux" or sys_platform=="linux2"': [

sign_client/setup.py

@@ -40,6 +40,6 @@
       packages=find_packages(),
       install_requires=[
         "requests~=2.26.0",
-        "aiohttp~=3.8.1",
+        "aiohttp~=3.9.0",
       ],
       zip_safe=False)

sign_client/sign_client/client.py

@@ -255,7 +255,7 @@ async def _await_calls(self, handle, headers, objects):
             # prepare a list of calls to make * Note: calls are prepared by using call
             # syntax (eg, func() and not func), but they will not be run until executed by the wait
             # split into batches of self.bach_size to avoid taking too much memory
-            calls = [handle(sem, o, headers, session) for o in objects]
+            calls = [asyncio.create_task(handle(sem, o, headers, session)) for o in objects]
             await asyncio.wait(calls)
 
     async def _get_user(self, semaphore, user_id, header, session):

sign_client/sign_client/error.py

@@ -31,13 +31,6 @@ def is_reported(self):
         return self.reported
 
 
-class TimeoutException(Exception):
+class TimeoutException(AssertionException):
     def __init__(self, message):
-        super(AssertionException, self).__init__(message)
-        self.reported = False
-
-    def set_reported(self):        
-        self.reported = True
-
-    def is_reported(self):
-        return self.reported
+        super().__init__(message)

user_sync/config/sign_sync.py

@@ -56,6 +56,19 @@ def config_schema() -> Schema:
         }
     })
 
+def connector_schema() -> Schema:
+    from schema import And, Optional, Or, Regex
+    return Schema({
+        'host': str,
+        Or('integration_key', 'secure_integration_key'): str,
+        'admin_email': str,
+        Optional('create_users'): Optional(bool),
+        Optional('deactivate_users'): Optional(bool),
+        Optional('exclusions'): {
+            Optional('groups'): list,
+            Optional('users'): list,
+        }
+    })
 
 class SignConfigLoader(ConfigLoader):
     """
@@ -96,7 +109,7 @@ def __init__(self, args: dict):
         filename, encoding = self._config_file_info()
         self.config_loader = ConfigFileLoader(encoding, self.ROOT_CONFIG_PATH_KEYS, self.SUB_CONFIG_PATH_KEYS)
         self.raw_config = self._load_raw_config(filename, encoding)
-        self._validate(self.raw_config)
+        self._validate(config_schema, self.raw_config)
         self.main_config = self.load_main_config(filename, self.raw_config)
         self.invocation_options = self.load_invocation_options()
         self.directory_groups = self.load_directory_groups()
@@ -143,10 +156,10 @@ def _load_raw_config(self, filename, encoding) -> dict:
         return self.config_loader.load_root_config(filename)
 
     @staticmethod
-    def _validate(raw_config: dict):
+    def _validate(schm, raw_config: dict):
         from schema import SchemaError
         try:
-            config_schema().validate(raw_config)
+            schm().validate(raw_config)
         except SchemaError as e:
             raise ConfigValidationError(e.code) from e
 
@@ -211,11 +224,14 @@ def get_target_options(self) -> dict[str, dict]:
         if self.DEFAULT_ORG_NAME not in target_configs:
             raise AssertionException(f"'sign_orgs' config must specify a connector with '{self.DEFAULT_ORG_NAME}' key")
         primary_options = self.config_loader.load_sub_config(target_configs[self.DEFAULT_ORG_NAME])
+        self._validate(connector_schema, primary_options)
         all_options = {}
         for target_id, config_file in target_configs.items():
             if target_id == self.DEFAULT_ORG_NAME:
                 continue
-            all_options[target_id] = self.config_loader.load_sub_config(config_file)
+            cfg = self.config_loader.load_sub_config(config_file)
+            self._validate(connector_schema, cfg)
+            all_options[target_id] = cfg
         all_options[self.DEFAULT_ORG_NAME] = primary_options
         return all_options
 

user_sync/connector/connector_sign.py

@@ -27,6 +27,7 @@
 from ..error import AssertionException
 from sign_client.client import SignClient
 from pathlib import Path
+import re
 
 
 class SignConnector(object):
@@ -45,6 +46,20 @@ def __init__(self, caller_options, org_name, test_mode, connection, cache_config
         sign_builder.require_string_value('admin_email')
         self.create_users = sign_builder.require_value('create_users', bool)
         self.deactivate_users = sign_builder.require_value('deactivate_users', bool)
+
+        exclusion_config = caller_config.get_dict_config('exclusions', True)
+        exclusion_builder = OptionsBuilder(exclusion_config)
+        exclusion_builder.set_value('groups', list, [])
+        exclusion_builder.set_value('users', list, [])
+
+        self.exclusion_options = exclusion_builder.get_options()
+
+        if 'users' in self.exclusion_options:
+            compiled_rules = []
+            for rule in self.exclusion_options['users']:
+                compiled_rules.append(re.compile(rule))
+            self.exclusion_options['users'] = compiled_rules
+
         store_path = Path(cache_config['path'])
 
         options = sign_builder.get_options()

user_sync/connector/connector_umapi.py

@@ -183,6 +183,7 @@ def get_groups(self):
         return list(self.iter_groups())
 
     def iter_groups(self):
+        self.connection.start_sync()
         try:
             for g in umapi_client.GroupsQuery(self.connection):
                 yield g