Bump botocore requirement to 1.34.0 (#2206)

SUMMARY
1.34 was released Nov 2023, 1.35 was released Aug 2024
Also fixes up the running the unit tests under tox.
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
requirements.txt
ADDITIONAL INFORMATION
See also: ansible-collections/amazon.aws#2426

Reviewed-by: Mandar Kulkarni <mandar242@gmail.com>

README.md

@@ -53,7 +53,7 @@ As such support for Python less than 3.8 will be removed in a release after 2024
 
 Starting with the 2.0.0 releases of amazon.aws and community.aws, it is generally the collection's policy to support the versions of `botocore` and `boto3` that were released 12 months prior to the most recent major collection release, following semantic versioning (for example, 2.0.0, 3.0.0).
 
-Version 9.0.0 of this collection supports `boto3 >= 1.28.0` and `botocore >= 1.31.0`
+Version 10.0.0 of this collection supports `boto3 >= 1.34.0` and `botocore >= 1.34.0`
 
 All support for the original AWS SDK `boto` was removed in release 4.0.0.
 

changelogs/fragments/botocore.yml

@@ -0,0 +1,7 @@
+---
+major_changes:
+  - community.aws collection - The community.aws collection has dropped support for
+    ``botocore<1.34.0`` and ``boto3<1.34.0``. Most modules will continue to work
+    with older versions of the AWS SDK, however compatibility with older versions
+    of the SDK is not guaranteed and will not be tested. When using older versions
+    of the SDK a warning will be emitted by Ansible (https://github.com/ansible-collections/amazon.aws/pull/2426).

pyproject.toml

@@ -41,3 +41,13 @@ known_ansible_community_aws = ["ansible_collections.community.aws"]
 
 [tool.flynt]
 transform-joins = true
+
+[tool.flake8]
+# E123, E125 skipped as they are invalid PEP-8.
+show-source = true
+ignore = ["E123", "E125", "E203", "E402", "E501", "E741", "F401", "F811", "F841", "W503"]
+max-line-length = 160
+builtins = "_"
+
+[tool.mypy]
+disable_error_code = ["import-untyped"]

requirements.txt

@@ -2,5 +2,5 @@
 # - tests/unit/constraints.txt
 # - tests/integration/constraints.txt
 # - tests/integration/targets/setup_botocore_pip
-botocore>=1.31.0
-boto3>=1.28.0
+botocore>=1.34.0
+boto3>=1.34.0

test-requirements.txt

@@ -1,17 +1,15 @@
 botocore
 boto3
 
+# coverage==4.5.4
 python-dateutil # Used by autoscaling_scheduled_action
-
-coverage==4.5.4
 placebo
 mock
 pytest
 pytest-forked
 pytest-mock
 pytest-xdist
-pytest-ansible ; python_version >= '3.7'
-git+https://github.com/ansible-community/pytest-ansible-units.git ; python_version < '3.7'
+pytest-ansible
 
 # Needed for ansible.utils.ipaddr in tests
 netaddr

tests/integration/constraints.txt

@@ -1,11 +1,7 @@
 # Specifically run tests against the oldest versions that we support
-botocore==1.31.0
-boto3==1.28.0
+botocore==1.34.0
+boto3==1.34.0
 
 # AWS CLI has `botocore==` dependencies, provide the one that matches botocore
 # to avoid needing to download over a years worth of awscli wheels.
-awscli==1.29.0
-
-# AWS CLI depends on PyYAML <5.5,>=3.10; the latest PyYAML release in that range, 5.4.1, fails to install.
-# Use a version in that range that is known to work (https://github.com/yaml/pyyaml/issues/736)
-PyYAML==5.3.1
+awscli==1.32.0

tests/unit/constraints.txt

@@ -1,7 +1,7 @@
 # Specifically run tests against the oldest versions that we support
-botocore==1.31.0
-boto3==1.28.0
+botocore==1.34.0
+boto3==1.34.0
 
 # AWS CLI has `botocore==` dependencies, provide the one that matches botocore
 # to avoid needing to download over a years worth of awscli wheels.
-awscli==1.29.0
+awscli==1.32.0

tox.ini

@@ -1,104 +1,205 @@
+# It would be nice to merge this into pyproject.toml, unfortunately as of 4.23.2 they don't support generative environments when using TOML
+
 [tox]
 skipsdist = True
-envlist = clean,ansible{2.12,2.13}-py{38,39,310}-{with_constraints,without_constraints},linters
-# Tox4 supports labels which allow us to group the environments rather than dumping all commands into a single environment
-labels =
-    format = flynt, black, isort
-    lint = complexity-report, ansible-lint, black-lint, isort-lint, flake8-lint, flynt-lint
-    units = ansible{2.12,2.13}-py{38,39,310}-{with_constraints,without_constraints}
+skip_missing_interpreters = True
+envlist =
+    clean
+    ansible{2.15}-py{39,310,311}-{with_constraints,without_constraints}
+    ansible{2.16,2.17}-py{310,311,312}-{with_constraints,without_constraints}
+    ansible{2.18}-py{311,312,313}-{with_constraints,without_constraints}
+    linters
 
 [common]
 format_dirs = {toxinidir}/plugins {toxinidir}/tests
+lint_dirs = {toxinidir}/plugins
+ansible_desc =
+    ansible2.15: Ansible-core 2.15
+    ansible2.16: Ansible-core 2.16
+    ansible2.17: Ansible-core 2.17
+    ansible2.18: Ansible-core 2.18
+const_desc =
+    with_constraints: (With boto3/botocore constraints)
+collection_path = ansible_collections/community/aws
+collection_name = community.aws
+[mypy]
+mypy_path = {envtmpdir}/mypy
+full_tmp_path = {[mypy]mypy_path}/{[common]collection_path}
+[pyright]
+pyright_path = {envtmpdir}/pyright
+full_tmp_path = {[mypy]pyright_path}/{[common]collection_path}
+[ansible-sanity]
+sanity_tmp_path = {envtmpdir}/ansible-sanity
+full_tmp_path = {[ansible-sanity]sanity_tmp_path}/{[common]collection_path}
+
 
 [testenv]
-description = Run the test-suite and generate a HTML coverage report
+description = Run the unit tests {[common]ansible_desc}/{base_python} {[common]const_desc}
+labels = unit
 deps =
   pytest
   pytest-cov
-  ansible2.12: ansible-core>2.12,<2.13
-  ansible2.13: ansible-core>2.13,<2.14
-  !ansible2.12-!ansible2.13: ansible-core
   pytest-ansible
   -rtest-requirements.txt
+  ansible2.15: ansible-core>2.15,<2.16
+  ansible2.16: ansible-core>2.16,<2.17
+  ansible2.17: ansible-core>2.17,<2.18
+  ansible2.18: ansible-core>2.18,<2.19
   with_constraints: -rtests/unit/constraints.txt
-commands = pytest --cov-report html --cov plugins/callback --cov plugins/inventory --cov plugins/lookup --cov plugins/module_utils --cov plugins/modules --cov plugins/plugin_utils plugins {posargs:tests/}
+commands =
+    pytest \
+        --cov-report html \
+        --cov plugins/callback \
+        --cov plugins/inventory \
+        --cov plugins/lookup \
+        --cov plugins/module_utils \
+        --cov plugins/modules \
+        --cov plugins/plugin_utils \
+        --cov plugins \
+        --ansible-host-pattern localhost \
+        {posargs:tests/}
 
 [testenv:clean]
+description = Remove coverage test data
 deps = coverage
 skip_install = true
-commands = coverage erase
+commands =
+    coverage erase
 
 [testenv:complexity-report]
+labels = lint
 description = Generate a HTML complexity report in the complexity directory
 deps =
-  # See: https://github.com/lordmauve/flake8-html/issues/30
-  flake8>=3.3.0,<5.0.0
+  flake8-pyproject
   flake8-html
-commands = -flake8 --select C90 --max-complexity 10 --format=html --htmldir={posargs:complexity} plugins
+commands =
+    -flake8 \
+        --select C90 \
+        --max-complexity 10 \
+        --format=html \
+        --htmldir={posargs:complexity} \
+        {posargs:{[common]lint_dirs}}
 
 [testenv:ansible-lint]
+labels = lint
+description = Run ansible-lint
 deps =
-  ansible-lint
+  ansible-lint >= 24.7.0
 commands =
-  ansible-lint {toxinidir}/plugins
+  ansible-lint {posargs:{[common]lint_dirs}}
 
 [testenv:black]
+labels = format
+description = Apply "black" formatting
 depends =
   flynt, isort
 deps =
   black >=23.0, <24.0
 commands =
-  black {[common]format_dirs}
+  black {posargs:{[common]format_dirs}}
 
 [testenv:black-lint]
+labels = lint
+description = Lint against "black" formatting standards
 deps =
   {[testenv:black]deps}
 commands =
-  black -v --check --diff {[common]format_dirs}
+  black -v --check --diff {posargs:{[common]format_dirs}}
 
 [testenv:isort]
+labels = format
+description = Sort imports
 deps =
   isort
 commands =
-  isort {[common]format_dirs}
+  isort {posargs:{[common]format_dirs}}
 
 [testenv:isort-lint]
+labels = lint
+description = Lint for import sorting
 deps =
   {[testenv:isort]deps}
 commands =
-  isort --check-only --diff {[common]format_dirs}
-
-[testenv:flake8-lint]
-deps =
-  flake8
-commands =
-  flake8 {posargs} {[common]format_dirs}
+  isort --check-only --diff {posargs:{[common]format_dirs}}
 
 [testenv:flynt]
+labels = format
+description = Apply flint (f-string) formatting
 deps =
   flynt
 commands =
-  flynt {[common]format_dirs}
+  flynt {posargs:{[common]format_dirs}}
 
 [testenv:flynt-lint]
+labels = lint
+description = Run flint (f-string) linting
 deps =
   flynt
 commands =
-  flynt --dry-run {[common]format_dirs}
+  flynt --dry-run --fail-on-change {posargs:{[common]format_dirs}}
 
-[testenv:linters]
+[testenv:flake8-lint]
+labels = lint
+description = Run FLAKE8 linting
 deps =
-  {[testenv:black]deps}
-  {[testenv:isort]deps}
   flake8
+  flake8-pyproject
+commands =
+  flake8 {posargs:{[common]format_dirs}}
+
+[testenv:pylint-lint]
+labels = future-lint
+description = Run pylint tests that are disabled by the default Ansible sanity tests
+deps =
+  pylint
+commands =
+  pylint \
+    --disable R,C,W,E \
+    --enable consider-using-dict-items,assignment-from-no-return,no-else-continue,no-else-break,simplifiable-if-statement,pointless-string-statement,redefined-outer-name,redefined-builtin \
+    {posargs:{[common]lint_dirs}}
+
+[testenv:ansible-sanity]
+allowlist_externals = echo,cd,rm,mkdir,ln,ls
+labels = future-lint
+description = Run latest Ansible sanity tests
+# ansible-sanity expects us to be installed into ansible_collections/community/aws
+# by default we're checked out into community.aws
+commands_pre =
+  rm -rf {[ansible-sanity]sanity_tmp_path}
+  mkdir -p {[ansible-sanity]full_tmp_path}
+  rm -d {[ansible-sanity]full_tmp_path}
+  ln -s {toxinidir} {[ansible-sanity]full_tmp_path}
+deps =
+  git+https://github.com/ansible/ansible.git@devel
+  shellcheck-py
+commands =
+  cd {[ansible-sanity]full_tmp_path}
+  ansible-test sanity
+
+[testenv:mypy-lint]
+allowlist_externals = echo,cd,rm,mkdir,ln
+labels = future-lint
+description = Run mypi type tests
+set_env =
+  MYPYPATH={envtmpdir}/mypy
+deps =
+  mypy
+  # ansible-core
+  git+https://github.com/ansible/ansible.git@devel
+  botocore
+  boto3
+  placebo
+  typing_extensions
+commands_pre =
+  rm -rf {[mypy]mypy_path}
+  mkdir -p {[mypy]full_tmp_path}
+  rm -d {[mypy]full_tmp_path}
+  ln -s {toxinidir} {[mypy]full_tmp_path}
+  ansible-galaxy collection install -p {[mypy]mypy_path}/ansible_collections git+https://github.com/ansible-collections/amazon.aws.git,main
 commands =
-  black -v --check {toxinidir}/plugins {toxinidir}/tests
-  isort --check-only --diff {toxinidir}/plugins {toxinidir}/tests
-  flake8 {posargs} {toxinidir}/plugins {toxinidir}/tests
-
-[flake8]
-# E123, E125 skipped as they are invalid PEP-8.
-show-source = True
-ignore = E123,E125,E203,E402,E501,E741,F401,F811,F841,W503
-max-line-length = 160
-builtins = _
+  cd {[mypy]full_tmp_path}
+  mypy \
+    --namespace-packages --explicit-package-bases \
+    --follow-imports silent \
+    {posargs:plugins/module_utils}
+    # start with module_utils, expand later