Upgrade the default version of black from 23.12 to 24.8 (#21056)

This upgrades the default black version from 23.12.1 to 24.8.0. 

This results in "user-visible" **formatting changes**: we're ticking
over into a new year, and thus pick up the "2024" stable style in
24.1.0: https://black.readthedocs.io/en/stable/change_log.html#id18 .
Depending on the codebase, this potentially results in significant
reformatting (and does so for Pants).

To make it easier for people to upgrade, this retains a copy of the old
lockfile, that can be referenced via a `resources:` URI.

There's various other changes in the release notes, like support for new
syntax, bug fixes and performance improvements.

Because Pants uses its own black backend this PR also reformats the
world, according to the new style.

The commits are independently sensible:

1. Upgrade black itself
2. Adjust the flake8 rules due to a new conflict
(https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#e701-e704)
3. Reformat the whole world
4. Various fixes after review

build-support/bin/generate_builtin_lockfiles.py

@@ -89,13 +89,11 @@ def lockfile_name(self) -> str:
 
 
 @dataclass
-class PythonTool(Tool[PythonToolRequirementsBase]):
-    ...
+class PythonTool(Tool[PythonToolRequirementsBase]): ...
 
 
 @dataclass
-class JvmTool(Tool[JvmToolBase]):
-    ...
+class JvmTool(Tool[JvmToolBase]): ...
 
 
 all_python_tools = tuple(

build-support/flake8/.flake8

@@ -6,6 +6,8 @@ extend-ignore:
   E231,
   # line too long (> 79 characters)
   E501,
+  # multiple statements on one line (colon) (conflicts with Black)
+  E701,
   # Do not assign a lambda expression
   E731,
   # Ambiguous variable name (enable once fixed)

docs/notes/2.25.x.md

@@ -79,6 +79,40 @@ The `pants.backend.experimental.python.lint.ruff.check` backend [now supports](h
 A new experimental `pants.backend.experimental.openapi.codegen.python` backend
 was added to support python codegen for OpenAPI documents.
 
+##### Black
+
+[The `pants.backend.python.lint.black` backend](https://www.pantsbuild.org/2.25/reference/subsystems/black) now uses version 24.8.0 by default, upgrading from 23.12.1. This comes with a new stable style (see [release notes for 24.1.0](https://black.readthedocs.io/en/stable/change_log.html) for details) which may result in extensive reformatting.
+
+To override Pants' default version, use [the `install_from_resolve` option](https://www.pantsbuild.org/2.25/reference/subsystems/black#install_from_resolve) and/or [the `interpreter_constraints` option](https://www.pantsbuild.org/2.25/reference/subsystems/black#interpreter_constraints). The style changes may be extensive, so, to make upgrading to Pants 2.25 easier, we provide the old lockfile as built-in, for now (Pants will remove this in future, so you should switch to your own lockfile, and/or upgrade to the default Black 24, soon). To use this lockfile, and remain on Black 23, configure a resolve as follows:
+
+``` toml
+# pants.toml
+
+[python.resolves]
+...
+black = "resource://pants.backend.python.lint.black/black-23.12.lock"
+
+[black]
+install_from_resolve = "black"
+```
+
+To take control of your Black version independent of Pants' default, configure a resolve similar to the following, and generate the lockfile with `pants generate-lockfiles --resolve=your-resolve-name`:
+
+``` toml
+# pants.toml
+
+[python.resolves]
+...
+your-resolve-name = "3rdparty/python/your-resolve-name.lock"
+
+[black]
+install_from_resolve = "your-resolve-name"
+```
+
+``` python
+# BUILD
+python_requirement(name="black", resolve="your-resolve-name", requirements=["black>=23,<24"])
+```
 #### Shell
 
 The previously deprecated `[shell-setup].tailor` option has now been removed. See [`[shell-setup].tailor_sources`](https://www.pantsbuild.org/2.25/reference/subsystems/shell-setup#tailor_sources) and [`[shell-setup].tailor_shunit2_tests`](https://www.pantsbuild.org/2.25/reference/subsystems/shell#tailor_shunit2_tests) to update.

src/python/pants/backend/docker/target_types.py

@@ -335,8 +335,7 @@ class DockerBuildKitOptionField:
     """Mixin to indicate a BuildKit-specific option."""
 
     @abstractmethod
-    def options(self, value_formatter: OptionValueFormatter) -> Iterator[str]:
-        ...
+    def options(self, value_formatter: OptionValueFormatter) -> Iterator[str]: ...
 
     required_help = "This option requires BuildKit to be enabled via the Docker subsystem options."
 

src/python/pants/backend/docker/util_rules/docker_build_context.py

@@ -218,9 +218,11 @@ def _merge_build_args(
             # values from the Dockerfile however, in order to not having to duplicate them in
             # the BUILD files.
             return {
-                arg_name: arg_value
-                if has_value
-                else build_env.get(arg_name, build_arg_defaults.get(arg_name))
+                arg_name: (
+                    arg_value
+                    if has_value
+                    else build_env.get(arg_name, build_arg_defaults.get(arg_name))
+                )
                 for arg_name, has_value, arg_value in [
                     build_arg.partition("=") for build_arg in build_args
                 ]

src/python/pants/backend/docker/util_rules/docker_build_context_test.py

@@ -96,8 +96,9 @@ def assert_build_context(
     *,
     build_upstream_images: bool = False,
     expected_files: list[str],
-    expected_interpolation_context: dict[str, str | dict[str, str] | InterpolationValue]
-    | None = None,
+    expected_interpolation_context: (
+        dict[str, str | dict[str, str] | InterpolationValue] | None
+    ) = None,
     expected_num_upstream_images: int = 0,
     pants_args: list[str] | None = None,
     runner_options: dict[str, Any] | None = None,

src/python/pants/backend/go/util_rules/build_opts.py

@@ -214,23 +214,29 @@ async def go_extract_build_options_from_target(
                 "the `[go-test].force_race` option is in effect",
             ),
             (
-                test_target_fields.test_race.value,
-                f"{GoTestRaceDetectorEnabledField.alias}={test_target_fields.test_race.value} on target `{request.address}`",
-            )
-            if test_target_fields
-            else None,
+                (
+                    test_target_fields.test_race.value,
+                    f"{GoTestRaceDetectorEnabledField.alias}={test_target_fields.test_race.value} on target `{request.address}`",
+                )
+                if test_target_fields
+                else None
+            ),
             (
-                target_fields.race.value,
-                f"{GoRaceDetectorEnabledField.alias}={target_fields.race.value} on target `{request.address}`",
-            )
-            if target_fields
-            else None,
+                (
+                    target_fields.race.value,
+                    f"{GoRaceDetectorEnabledField.alias}={target_fields.race.value} on target `{request.address}`",
+                )
+                if target_fields
+                else None
+            ),
             (
-                go_mod_target_fields.race.value,
-                f"{GoRaceDetectorEnabledField.alias}={go_mod_target_fields.race.value} on target `{request.address}`",
-            )
-            if go_mod_target_fields
-            else None,
+                (
+                    go_mod_target_fields.race.value,
+                    f"{GoRaceDetectorEnabledField.alias}={go_mod_target_fields.race.value} on target `{request.address}`",
+                )
+                if go_mod_target_fields
+                else None
+            ),
             (False, "default"),
         ]
     )
@@ -250,23 +256,29 @@ async def go_extract_build_options_from_target(
                 "the `[go-test].force_msan` option is in effect",
             ),
             (
-                test_target_fields.test_msan.value,
-                f"{GoTestMemorySanitizerEnabledField.alias}={test_target_fields.test_msan.value} on target `{request.address}`",
-            )
-            if test_target_fields
-            else None,
+                (
+                    test_target_fields.test_msan.value,
+                    f"{GoTestMemorySanitizerEnabledField.alias}={test_target_fields.test_msan.value} on target `{request.address}`",
+                )
+                if test_target_fields
+                else None
+            ),
             (
-                target_fields.msan.value,
-                f"{GoMemorySanitizerEnabledField.alias}={target_fields.msan.value} on target `{request.address}`",
-            )
-            if target_fields
-            else None,
+                (
+                    target_fields.msan.value,
+                    f"{GoMemorySanitizerEnabledField.alias}={target_fields.msan.value} on target `{request.address}`",
+                )
+                if target_fields
+                else None
+            ),
             (
-                go_mod_target_fields.msan.value,
-                f"{GoMemorySanitizerEnabledField.alias}={go_mod_target_fields.msan.value} on target `{request.address}`",
-            )
-            if go_mod_target_fields
-            else None,
+                (
+                    go_mod_target_fields.msan.value,
+                    f"{GoMemorySanitizerEnabledField.alias}={go_mod_target_fields.msan.value} on target `{request.address}`",
+                )
+                if go_mod_target_fields
+                else None
+            ),
             (False, "default"),
         ]
     )
@@ -286,23 +298,29 @@ async def go_extract_build_options_from_target(
                 "the `[go-test].force_asan` option is in effect",
             ),
             (
-                test_target_fields.test_asan.value,
-                f"{GoTestAddressSanitizerEnabledField.alias}={test_target_fields.test_asan.value} on target `{request.address}`",
-            )
-            if test_target_fields
-            else None,
+                (
+                    test_target_fields.test_asan.value,
+                    f"{GoTestAddressSanitizerEnabledField.alias}={test_target_fields.test_asan.value} on target `{request.address}`",
+                )
+                if test_target_fields
+                else None
+            ),
             (
-                target_fields.asan.value,
-                f"{GoAddressSanitizerEnabledField.alias}={target_fields.asan.value} on target `{request.address}`",
-            )
-            if target_fields
-            else None,
+                (
+                    target_fields.asan.value,
+                    f"{GoAddressSanitizerEnabledField.alias}={target_fields.asan.value} on target `{request.address}`",
+                )
+                if target_fields
+                else None
+            ),
             (
-                go_mod_target_fields.asan.value,
-                f"{GoAddressSanitizerEnabledField.alias}={go_mod_target_fields.asan.value} on target `{request.address}`",
-            )
-            if go_mod_target_fields
-            else None,
+                (
+                    go_mod_target_fields.asan.value,
+                    f"{GoAddressSanitizerEnabledField.alias}={go_mod_target_fields.asan.value} on target `{request.address}`",
+                )
+                if go_mod_target_fields
+                else None
+            ),
             (False, "default"),
         ]
     )

src/python/pants/backend/helm/resolve/artifacts.py

@@ -43,8 +43,7 @@ def __init__(self, duplicates: Iterable[tuple[str, Address]]) -> None:
 class HelmArtifactLocationSpec(ABC):
     @property
     @abstractmethod
-    def spec(self) -> str:
-        ...
+    def spec(self) -> str: ...
 
     @property
     def is_url(self) -> bool:

src/python/pants/backend/openapi/util_rules/openapi_bundle.py

@@ -74,9 +74,11 @@ async def bundle_openapi_document(request: _BundleOpenApiDocument, redocly: Redo
     source_root_request = Get(
         SourceRoot,
         SourceRootRequest,
-        SourceRootRequest(PurePath(request.bundle_source_root))
-        if request.bundle_source_root
-        else SourceRootRequest.for_target(request.target),
+        (
+            SourceRootRequest(PurePath(request.bundle_source_root))
+            if request.bundle_source_root
+            else SourceRootRequest.for_target(request.target)
+        ),
     )
 
     target_stripped_sources_request = Get(

src/python/pants/backend/project_info/peek.py

@@ -56,8 +56,7 @@
 class Dictable(Protocol):
     """Make possible to avoid adding concrete types to serialize objects."""
 
-    def asdict(self) -> Mapping[str, Any]:
-        ...
+    def asdict(self) -> Mapping[str, Any]: ...
 
 
 class PeekSubsystem(Outputting, GoalSubsystem):

src/python/pants/backend/project_info/peek_test.py

@@ -359,9 +359,9 @@ def _normalize_fingerprints(tds: Sequence[TargetData]) -> list[TargetData]:
     return [
         dataclasses.replace(
             td,
-            expanded_sources=None
-            if td.expanded_sources is None
-            else _snapshot("", td.expanded_sources.files),
+            expanded_sources=(
+                None if td.expanded_sources is None else _snapshot("", td.expanded_sources.files)
+            ),
         )
         for td in tds
     ]

src/python/pants/backend/python/goals/package_pex_binary_integration_test.py

@@ -215,9 +215,11 @@ def test_layout(rule_runner: PythonRuleRunner, layout: PexLayout) -> None:
     rule_runner.write_digest(result.digest)
     executable = os.path.join(
         rule_runner.build_root,
-        expected_pex_relpath
-        if PexLayout.ZIPAPP is layout
-        else os.path.join(expected_pex_relpath, "__main__.py"),
+        (
+            expected_pex_relpath
+            if PexLayout.ZIPAPP is layout
+            else os.path.join(expected_pex_relpath, "__main__.py")
+        ),
     )
     stdout = dedent(
         """\

src/python/pants/backend/python/goals/run_python_source_integration_test.py

@@ -203,9 +203,11 @@ def my_file():
         f"--debug-adapter-port={debugadapter_port_for_testing()}",
         *(
             (
-                "--python-default-run-goal-use-sandbox"
-                if global_default_value
-                else "--no-python-default-run-goal-use-sandbox",
+                (
+                    "--python-default-run-goal-use-sandbox"
+                    if global_default_value
+                    else "--no-python-default-run-goal-use-sandbox"
+                ),
             )
             if global_default_value is not None
             else ()

src/python/pants/backend/python/lint/black/BUILD

@@ -1,10 +1,18 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-resource(name="lockfile", source="black.lock")
+resources(
+    name="lockfiles",
+    sources=[
+        "black.lock",
+        # Lockfile for the version of Black Pants used to provide (with the '23 stable style), to
+        # make upgrading easier
+        "black-23.12.lock",
+    ],
+)
 
 python_sources(
-    overrides={"subsystem.py": {"dependencies": [":lockfile"]}},
+    overrides={"subsystem.py": {"dependencies": [":lockfiles"]}},
 )
 
 python_tests(