From a2a06524f7c20b2cebf9d9457ca587a5079c329a Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 26 Dec 2024 01:31:02 +0100 Subject: [PATCH] infer all PBS-related metadata from the URL (#21789) Infer most PBS-related metadata from the artifact download URL provided by the user. (The SHA256 checksum and file size are not inferred from the URL.) Adds a three field format for `--python-build-standalone-known-python-versions` which infers all PBS-related metadata from the URL. Format is `SHA256|FILE_SIZE|URL`. The five field format is still supported. Remove the `--python-build-standalone-require-inferrable-release-tag` option added by #21739 because the PBS release tag can now be optionally provided in the version field (first field) of the five field format for `--python-build-standalone-known-python-versions`. --- docs/notes/2.25.x.md | 8 +- .../python_build_standalone/rules.py | 328 ++++++++++++++---- .../rules_integration_test.py | 26 +- .../python_build_standalone/rules_test.py | 128 +++++++ 4 files changed, 412 insertions(+), 78 deletions(-) create mode 100644 src/python/pants/backend/python/providers/python_build_standalone/rules_test.py diff --git a/docs/notes/2.25.x.md b/docs/notes/2.25.x.md index bd0c499c0ad..95c506003ae 100644 --- a/docs/notes/2.25.x.md +++ b/docs/notes/2.25.x.md @@ -64,9 +64,13 @@ Fixed a bug where linting with the Helm backend enabled could induce serializati The AWS Lambda backend now provides built-in complete platforms for the Python 3.13 runtime. -The Python Build Standalone backend (`pants.backend.python.providers.experimental.python_build_standalone`) now supports filtering PBS releases via their "release tag" via [the new `--python-build-standalone-release-constraints` option](https://www.pantsbuild.org/2.25/reference/subsystems/python-build-standalone-python-provider#release_constraints). THe PBS "known versions" database now contains metadata on all known PBS versions, and not just the latest PBS release tag per Python patchlevel. +Several improvements to the Python Build Standalone backend (`pants.backend.python.providers.experimental.python_build_standalone`): -Also, the PBS "release tag" will be inferred for PBS releases supplied via the `--python-build-standalone-known-python-versions` option from the given URLs if those URLs conform to the naming convention used by the PBS project. The new advanced option `--python-build-standalone-require-inferrable-release-tag` controls whether Pants requires the tag to be inferrable. This option currently defaults to `False`, but will be migrated to `True` in a future Pants release. (The release tag cannot currently be specified via `--python-build-standalone-known-python-versions` since changing that option would not be a backward compatible change.) +- The backend now supports filtering PBS releases via their "release tag" via [the new `--python-build-standalone-release-constraints` option](https://www.pantsbuild.org/2.25/reference/subsystems/python-build-standalone-python-provider#release_constraints). THe PBS "known versions" database now contains metadata on all known PBS versions, and not just the latest PBS release tag per Python patchlevel. + +- The backend will now infer metadata for a PBS release from a given URL if the URL conforms to the naming convention used by the PBS project. The inferred metadata is Python version, PBS release tag, and platform. + +- The `--python-build-standalone-known-python-versions` option now accepts a three field format where each value is `SHA256|FILE_SIZE|URL`. All of the PBS release metadata will be parsed from the URL (which must use the naming convention used by the PBS project). (The existing five-field format is still accepted and will now allow the version and platform fields to be blank if that data can be inferred from the URL.) Reverence to Python Build Standalone not refer to the [GitHub organization](https://github.com/astral-sh/python-build-standalone) as described in [Transferring Python Build Standalone Stewardship to Astral](https://gregoryszorc.com/blog/2024/12/03/transferring-python-build-standalone-stewardship-to-astral/). diff --git a/src/python/pants/backend/python/providers/python_build_standalone/rules.py b/src/python/pants/backend/python/providers/python_build_standalone/rules.py index 3143535fc11..16fa2acec51 100644 --- a/src/python/pants/backend/python/providers/python_build_standalone/rules.py +++ b/src/python/pants/backend/python/providers/python_build_standalone/rules.py @@ -4,13 +4,17 @@ import functools import json +import logging import posixpath import re import textwrap import urllib import uuid +from dataclasses import dataclass from pathlib import PurePath -from typing import Iterable, Mapping, TypedDict, cast +from typing import Iterable, Mapping, Sequence, TypedDict, TypeVar, cast + +from packaging.version import InvalidVersion from pants.backend.python.providers.python_build_standalone.constraints import ( ConstraintParseError, @@ -45,7 +49,7 @@ from pants.engine.unions import UnionRule from pants.option.errors import OptionsError from pants.option.global_options import NamedCachesDirOption -from pants.option.option_types import BoolOption, StrListOption, StrOption +from pants.option.option_types import StrListOption, StrOption from pants.option.subsystem import Subsystem from pants.util.docutil import bin_name from pants.util.frozendict import FrozenDict @@ -55,10 +59,14 @@ from pants.util.strutil import softwrap from pants.version import Version +logger = logging.getLogger(__name__) + PBS_SANDBOX_NAME = ".python_build_standalone" PBS_NAMED_CACHE_NAME = "python_build_standalone" PBS_APPEND_ONLY_CACHES = FrozenDict({PBS_NAMED_CACHE_NAME: PBS_SANDBOX_NAME}) +_T = TypeVar("_T") # Define type variable "T" + class PBSPythonInfo(TypedDict): url: str @@ -69,6 +77,225 @@ class PBSPythonInfo(TypedDict): PBSVersionsT = dict[str, dict[str, dict[str, PBSPythonInfo]]] +@dataclass +class _ParsedPBSPython: + py_version: Version + pbs_release_tag: Version + platform: Platform + url: str + sha256: str + size: int + + +def _parse_py_version_and_pbs_release_tag( + version_and_tag: str, +) -> tuple[Version | None, Version | None]: + version_and_tag = version_and_tag.strip() + if not version_and_tag: + return None, None + + parts = version_and_tag.split("+", 1) + py_version: Version | None = None + pbs_release_tag: Version | None = None + + if len(parts) >= 1: + try: + py_version = Version(parts[0]) + except InvalidVersion: + raise ValueError(f"Version `{parts[0]}` is not a valid Python version.") + + if len(parts) == 2: + try: + pbs_release_tag = Version(parts[1]) + except InvalidVersion: + raise ValueError(f"PBS release tag `{parts[1]}` is not a valid version.") + + return py_version, pbs_release_tag + + +def _parse_pbs_url(url: str) -> tuple[Version, Version, Platform]: + parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) + base_path = posixpath.basename(parsed_url.path) + + base_path_no_prefix = base_path.removeprefix("cpython-") + if base_path_no_prefix == base_path: + raise ValueError( + f"Unable to parse the provided URL since it does not have a cpython prefix as per the PBS naming convention: {url}" + ) + + base_path_parts = base_path_no_prefix.split("-", 1) + if len(base_path_parts) != 2: + raise ValueError( + f"Unable to parse the provided URL because it does not follow the PBS naming convention: {url}" + ) + + py_version, pbs_release_tag = _parse_py_version_and_pbs_release_tag(base_path_parts[0]) + if not py_version or not pbs_release_tag: + raise ValueError( + "Unable to parse the Python version and PBS release tag from the provided URL " + f"because it does not follow the PBS naming convention: {url}" + ) + + platform: Platform + match base_path_parts[1].split("-"): + case [ + "x86_64" | "x86_64_v2" | "x86_64_v3" | "x86_64_v4", + "unknown", + "linux", + "gnu" | "musl", + *_, + ]: + platform = Platform.linux_x86_64 + case ["aarch64", "unknown", "linux", "gnu", *_]: + platform = Platform.linux_arm64 + case ["x86_64", "apple", "darwin", *_]: + platform = Platform.macos_x86_64 + case ["aarch64", "apple", "darwin", *_]: + platform = Platform.macos_arm64 + case _: + raise ValueError( + "Unable to parse the platform from the provided URL " + f"because it does not follow the PBS naming convention: {url}" + ) + + return py_version, pbs_release_tag, platform + + +def _parse_from_three_fields(parts: Sequence[str], orig_value: str) -> _ParsedPBSPython: + assert len(parts) == 3 + sha256, size, url = parts + + try: + py_version, pbs_release_tag, platform = _parse_pbs_url(url) + except ValueError as e: + raise ExternalToolError( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` could not be parsed: {e}" + ) + + return _ParsedPBSPython( + py_version=py_version, + pbs_release_tag=pbs_release_tag, + platform=platform, + url=url, + sha256=sha256, + size=int(size), + ) + + +def _parse_from_five_fields(parts: Sequence[str], orig_value: str) -> _ParsedPBSPython: + assert len(parts) == 5 + py_version_and_tag_str, platform_str, sha256, filesize_str, url = (x.strip() for x in parts) + + try: + maybe_py_version, maybe_pbs_release_tag = _parse_py_version_and_pbs_release_tag( + py_version_and_tag_str + ) + except ValueError: + raise ExternalToolError( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` declares version `{py_version_and_tag_str}` in the first field, " + "but it could not be parsed as a PBS release version." + ) + + maybe_platform: Platform | None = None + if not platform_str: + pass + elif platform_str in ( + Platform.linux_x86_64.value, + Platform.linux_arm64.value, + Platform.macos_x86_64.value, + Platform.macos_arm64.value, + ): + maybe_platform = Platform(platform_str) + else: + raise ExternalToolError( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` declares platforn `{platform_str}` in the second field, " + "but that value is not a known Pants platform. It must be one of " + "`linux_x86_64`, `linux_arm64`, `macos_x86_64`, or `macos_arm64`." + ) + + if len(sha256) != 64 or not re.match("^[a-zA-Z0-9]+$", sha256): + raise ExternalToolError( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` declares SHA256 checksum `{sha256}` in the third field, " + "but that value does not parse as a SHA256 checksum." + ) + + try: + filesize: int = int(filesize_str) + except ValueError: + raise ExternalToolError( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` declares file size `{filesize_str}` in the fourth field, " + "but that value does not parse as an integer." + ) + + maybe_inferred_py_version: Version | None = None + maybe_inferred_pbs_release_tag: Version | None = None + maybe_inferred_platform: Platform | None = None + try: + ( + maybe_inferred_py_version, + maybe_inferred_pbs_release_tag, + maybe_inferred_platform, + ) = _parse_pbs_url(url) + except ValueError: + pass + + def _validate_inferred( + *, explicit: _T | None, inferred: _T | None, description: str, field_pos: str + ) -> _T: + if explicit is None: + if inferred is None: + raise ExternalToolError( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` does not declare a {description} in the {field_pos} field, and no {description} " + "could be inferred from the URL." + ) + else: + return inferred + else: + if inferred is not None and explicit != inferred: + logger.warning( + f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " + f"the value `{orig_value}` declares {description} `{explicit}` in the {field_pos} field, but Pants inferred " + f"{description} `{inferred}` from the URL." + ) + return explicit + + maybe_py_version = _validate_inferred( + explicit=maybe_py_version, + inferred=maybe_inferred_py_version, + description="version", + field_pos="first", + ) + + maybe_pbs_release_tag = _validate_inferred( + explicit=maybe_pbs_release_tag, + inferred=maybe_inferred_pbs_release_tag, + description="PBS release tag", + field_pos="first", + ) + + maybe_platform = _validate_inferred( + explicit=maybe_platform, + inferred=maybe_inferred_platform, + description="platform", + field_pos="second", + ) + + return _ParsedPBSPython( + py_version=maybe_py_version, + pbs_release_tag=maybe_pbs_release_tag, + platform=maybe_platform, + url=url, + sha256=sha256, + size=filesize, + ) + + @functools.cache def load_pbs_pythons() -> PBSVersionsT: versions_info = json.loads(read_sibling_resource(__name__, "versions_info.json")) @@ -110,9 +337,11 @@ class PBSPythonProviderSubsystem(Subsystem): f""" Known versions to verify downloads against. - Each element is a pipe-separated string of `version|platform|sha256|length|url`, where: + Each element is a pipe-separated string of either `py_version+pbs_release_tag|platform|sha256|length|url` or + `sha256|length|url`, where: - - `version` is the version string + - `py_version` is the Python version string + - `pbs_release_tag` is the PBS release tag (i.e., the PBS-specific version) - `platform` is one of `[{','.join(Platform.__members__.keys())}]` - `sha256` is the 64-character hex representation of the expected sha256 digest of the download file, as emitted by `shasum -a 256` @@ -120,9 +349,13 @@ class PBSPythonProviderSubsystem(Subsystem): `wc -c` - `url` is the download URL to the `.tar.gz` archive - E.g., `3.1.2|macos_x86_64|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813|https://`. + E.g., `3.1.2|macos_x86_64|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813|https://` + or `https://|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813`. - Values are space-stripped, so pipes can be indented for readability if necessary. + Values are space-stripped, so pipes can be indented for readability if necessary. If the three field + format is used, then Pants will infer the `py_version`, `pbs_release_tag`, and `platform` fields from + the URL. With the five field format, one or more of `py_version`, `pbs_release_tag`, and `platform` + may be left blank if Pants can infer the field from the URL. Additionally, any versions you specify here will override the default Pants metadata for that version. @@ -141,18 +374,6 @@ class PBSPythonProviderSubsystem(Subsystem): ), ) - require_inferrable_release_tag = BoolOption( - default=False, - help=textwrap.dedent( - """ - Normally, Pants will try to infer the PBS release "tag" from URLs supplied to the - `--python-build-standalone-known-python-versions` option. If this option is True, - then it is an error if Pants cannot infer the tag from the URL. - """ - ), - advanced=True, - ) - @memoized_property def release_constraints(self) -> ConstraintsList: rcs = self._release_constraints @@ -166,68 +387,49 @@ def release_constraints(self) -> ConstraintsList: f"The `[{PBSPythonProviderSubsystem.options_scope}].release_constraints option` is not valid: {e}" ) from None - def get_user_supplied_pbs_pythons(self, require_tag: bool) -> PBSVersionsT: - extract_re = re.compile(r"^cpython-([0-9.]+)\+([0-9]+)-.*\.tar\.\w+$") - - def extract_version_and_tag(url: str) -> tuple[str, str] | None: - parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) - base_path = posixpath.basename(parsed_url.path) - - nonlocal extract_re - if m := extract_re.fullmatch(base_path): - return (m.group(1), m.group(2)) - - return None - + def get_user_supplied_pbs_pythons(self) -> PBSVersionsT: user_supplied_pythons: dict[str, dict[str, dict[str, PBSPythonInfo]]] = {} for version_info in self.known_python_versions or []: - version_parts = version_info.split("|") - if len(version_parts) != 5: + version_parts = [x.strip() for x in version_info.split("|")] + if len(version_parts) not in (3, 5): raise ExternalToolError( f"Each value for the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option " - "must be five values separated by a `|` character as follows: PYTHON_VERSION|PLATFORM|SHA256|FILE_SIZE|URL " - f"\n\nInstead, the following value was provided: {version_info}" + "must be a set of three or five values separated by `|` characters as follows:\n\n" + "- 3 fields: URL|SHA256|FILE_SIZE\n\n" + "- 5 fields: PYTHON_VERSION+PBS_RELEASE|PLATFORM|SHA256|FILE_SIZE|URL\n\n" + "\n\nIf 3 fields are provided, Pants will attempt to infer values based on the URL which must " + "follow the PBS naming conventions.\n\n" + f"Instead, the following value was provided: {version_info}" ) - py_version, platform, sha256, filesize, url = (x.strip() for x in version_parts) - - tag: str | None = None - maybe_inferred_py_version_and_tag = extract_version_and_tag(url) - if maybe_inferred_py_version_and_tag: - inferred_py_version, inferred_tag = maybe_inferred_py_version_and_tag - if inferred_py_version != py_version: - raise ExternalToolError( - f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " - f"the value `{version_info}` declares Python version `{py_version}` in the first field, but the URL" - f"provided references Python version `{inferred_py_version}`. These must be the same." - ) - tag = inferred_tag - - if tag is None: - raise ExternalToolError( - f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, " - f'the PBS release "tag" could not be inferred from the supplied URL: {url}' - "\n\nThis is an error because the option" - f"`[{PBSPythonProviderSubsystem.options_scope}].require_inferrable_release_tag` is set to True." - ) + info = ( + _parse_from_three_fields(version_parts, orig_value=version_info) + if len(version_parts) == 3 + else _parse_from_five_fields(version_parts, orig_value=version_info) + ) + + py_version: str = str(info.py_version) + pbs_release_tag: str = str(info.pbs_release_tag) if py_version not in user_supplied_pythons: user_supplied_pythons[py_version] = {} - if tag not in user_supplied_pythons[py_version]: - user_supplied_pythons[py_version][tag] = {} + if pbs_release_tag not in user_supplied_pythons[py_version]: + user_supplied_pythons[py_version][pbs_release_tag] = {} + + pbs_python_info = PBSPythonInfo(url=info.url, sha256=info.sha256, size=info.size) - pbs_python_info = PBSPythonInfo(url=url, sha256=sha256, size=int(filesize)) - user_supplied_pythons[py_version][tag][platform] = pbs_python_info + user_supplied_pythons[py_version][pbs_release_tag][ + info.platform.value + ] = pbs_python_info return user_supplied_pythons def get_all_pbs_pythons(self) -> PBSVersionsT: all_pythons = load_pbs_pythons().copy() - user_supplied_pythons: PBSVersionsT = self.get_user_supplied_pbs_pythons( - require_tag=self.require_inferrable_release_tag - ) + user_supplied_pythons: PBSVersionsT = self.get_user_supplied_pbs_pythons() + for py_version, release_metadatas_for_py_version in user_supplied_pythons.items(): for ( release_tag, diff --git a/src/python/pants/backend/python/providers/python_build_standalone/rules_integration_test.py b/src/python/pants/backend/python/providers/python_build_standalone/rules_integration_test.py index 3fe14984c06..b5e9231c517 100644 --- a/src/python/pants/backend/python/providers/python_build_standalone/rules_integration_test.py +++ b/src/python/pants/backend/python/providers/python_build_standalone/rules_integration_test.py @@ -143,8 +143,8 @@ def test_additional_versions(rule_runner, mock_empty_versions_resource): "--python-build-standalone-python-provider-known-python-versions=[" + "'3.9.16|linux_arm64|75f3d10ae8933e17bf27e8572466ff8a1e7792f521d33acba578cc8a25d82e0b|24540128|https://github.com/astral-sh/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-aarch64-unknown-linux-gnu-install_only.tar.gz'," + "'3.9.16|macos_arm64|73bad3a610a0ff14166fbd5045cd186084bd2ce99edd2c6327054509e790b9ab|16765350|https://github.com/astral-sh/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-aarch64-apple-darwin-install_only.tar.gz'," - + "'3.9.16|linux_x86_64|f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a|26767987|https://github.com/astral-sh/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-unknown-linux-gnu-install_only.tar.gz'," - + "'3.9.16|macos_x86_64|69331e93656b179fcbfec0d506dfca11d899fe5dced990b28915e41755ce215c|17151321|https://github.com/astral-sh/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-apple-darwin-install_only.tar.gz'," + + "'f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a|26767987|https://github.com/astral-sh/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-unknown-linux-gnu-install_only.tar.gz'," + + "'69331e93656b179fcbfec0d506dfca11d899fe5dced990b28915e41755ce215c|17151321|https://github.com/astral-sh/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-apple-darwin-install_only.tar.gz'," + "]" ], ) @@ -157,14 +157,14 @@ def test_tag_inference_from_url() -> None: subsystem = create_subsystem( pbs.PBSPythonProviderSubsystem, known_python_versions=[ - "3.10.13|linux_arm|abc123|123|https://github.com/astral-sh/python-build-standalone/releases/download/20240224/cpython-3.10.13%2B20240224-aarch64-unknown-linux-gnu-install_only.tar.gz", + "3.10.13|linux_arm64|e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|123|https://github.com/astral-sh/python-build-standalone/releases/download/20240224/cpython-3.10.13%2B20240224-aarch64-unknown-linux-gnu-install_only.tar.gz", ], ) - user_supplied_pbs_versions = subsystem.get_user_supplied_pbs_pythons(require_tag=False) - assert user_supplied_pbs_versions["3.10.13"]["20240224"]["linux_arm"] == pbs.PBSPythonInfo( + user_supplied_pbs_versions = subsystem.get_user_supplied_pbs_pythons() + assert user_supplied_pbs_versions["3.10.13"]["20240224"]["linux_arm64"] == pbs.PBSPythonInfo( url="https://github.com/astral-sh/python-build-standalone/releases/download/20240224/cpython-3.10.13%2B20240224-aarch64-unknown-linux-gnu-install_only.tar.gz", - sha256="abc123", + sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", size=123, ) @@ -172,13 +172,13 @@ def test_tag_inference_from_url() -> None: subsystem = create_subsystem( pbs.PBSPythonProviderSubsystem, known_python_versions=[ - "3.10.13|linux_arm|abc123|123|file:///releases/20240224/cpython.tar.gz", + "3.10.13|linux_arm64|e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|123|file:///releases/20240224/cpython.tar.gz", ], ) with pytest.raises( - ExternalToolError, match='the PBS release "tag" could not be inferred from the supplied URL' + ExternalToolError, match="no PBS release tag could be inferred from the URL" ): - _ = subsystem.get_user_supplied_pbs_pythons(require_tag=True) + _ = subsystem.get_user_supplied_pbs_pythons() def test_venv_pex_reconstruction(rule_runner): @@ -221,22 +221,22 @@ def test_release_constraint_evaluation(rule_runner: RuleRunner) -> None: def make_platform_metadata(): return { "linux_arm64": { - "sha256": "abc123", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 1, "url": "foobar", }, "linux_x86_64": { - "sha256": "abc123", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 1, "url": "https://example.com/foo.zip", }, "macos_arm64": { - "sha256": "abc123", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 1, "url": "https://example.com/foo.zip", }, "macos_x86_64": { - "sha256": "abc123", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 1, "url": "https://example.com/foo.zip", }, diff --git a/src/python/pants/backend/python/providers/python_build_standalone/rules_test.py b/src/python/pants/backend/python/providers/python_build_standalone/rules_test.py new file mode 100644 index 00000000000..3406d7a3de7 --- /dev/null +++ b/src/python/pants/backend/python/providers/python_build_standalone/rules_test.py @@ -0,0 +1,128 @@ +# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import pytest + +from pants.backend.python.providers.python_build_standalone.rules import ( + _parse_from_five_fields, + _parse_from_three_fields, + _parse_pbs_url, + _parse_py_version_and_pbs_release_tag, + _ParsedPBSPython, +) +from pants.core.util_rules.external_tool import ExternalToolError +from pants.engine.platform import Platform +from pants.version import Version + + +def test_parse_py_version_and_pbs_release_tag() -> None: + result1 = _parse_py_version_and_pbs_release_tag("") + assert result1 == (None, None) + + result2 = _parse_py_version_and_pbs_release_tag("1.2.3") + assert result2 == (Version("1.2.3"), None) + + result3 = _parse_py_version_and_pbs_release_tag("1.2.3+20241201") + assert result3 == (Version("1.2.3"), Version("20241201")) + + with pytest.raises(ValueError): + _parse_py_version_and_pbs_release_tag("xyzzy+20241201") + + with pytest.raises(ValueError): + _parse_py_version_and_pbs_release_tag("1.2.3+xyzzy") + + +def test_parse_pbs_url() -> None: + result1 = _parse_pbs_url( + "https://github.com/indygreg/python-build-standalone/releases/download/20240726/cpython-3.12.4%2B20240726-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" + ) + assert result1 == (Version("3.12.4"), Version("20240726"), Platform.linux_x86_64) + + with pytest.raises(ValueError, match="Unable to parse the Python version and PBS release tag"): + _parse_pbs_url( + "https://example.com/cpython-3.12.4-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz" + ) + + with pytest.raises(ValueError, match="Unable to parse the platform"): + _parse_pbs_url( + "https://example.com/cpython-3.12.4%2B20240205-s390-unknown-linux-gnu-install_only_stripped.tar.gz" + ) + + +def test_parse_from_three_fields() -> None: + def invoke(s: str) -> _ParsedPBSPython: + parts = s.split("|") + return _parse_from_three_fields(parts, orig_value=s) + + result1 = invoke( + "f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a|26767987|https://github.com/indygreg/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-unknown-linux-gnu-install_only.tar.gz" + ) + assert result1 == _ParsedPBSPython( + py_version=Version("3.9.16"), + pbs_release_tag=Version("20221220"), + platform=Platform.linux_x86_64, + url="https://github.com/indygreg/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-unknown-linux-gnu-install_only.tar.gz", + sha256="f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a", + size=26767987, + ) + + with pytest.raises(ExternalToolError, match="since it does not have a cpython prefix"): + invoke( + "f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a|26767987|https://dl.example.com/cpython.tar.gz" + ) + + +def test_parse_from_five_fields() -> None: + def invoke(s: str) -> _ParsedPBSPython: + parts = s.split("|") + return _parse_from_five_fields(parts, orig_value=s) + + result1 = invoke( + "3.9.16|linux_x86_64|f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a|26767987|https://github.com/indygreg/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-unknown-linux-gnu-install_only.tar.gz" + ) + assert result1 == _ParsedPBSPython( + py_version=Version("3.9.16"), + pbs_release_tag=Version("20221220"), + platform=Platform.linux_x86_64, + url="https://github.com/indygreg/python-build-standalone/releases/download/20221220/cpython-3.9.16%2B20221220-x86_64-unknown-linux-gnu-install_only.tar.gz", + sha256="f885f3d011ab08e4d9521a7ae2662e9e0073acc0305a1178984b5a1cf057309a", + size=26767987, + ) + + with pytest.raises( + ExternalToolError, + match="does not declare a version in the first field, and no version could be inferred from the URL", + ): + invoke( + "||e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|123|https://dl.example.com/cpython.tar.gz" + ) + + with pytest.raises( + ExternalToolError, + match="does not declare a PBS release tag in the first field, and no PBS release tag could be inferred from the URL", + ): + invoke( + "3.10.1||e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|123|https://dl.example.com/cpython.tar.gz" + ) + + with pytest.raises( + ExternalToolError, + match="does not declare a platform in the second field, and no platform could be inferred from the URL", + ): + invoke( + "3.10.1+20240601||e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|123|https://dl.example.com/cpython.tar.gz" + ) + + result2 = invoke( + "3.10.1+20240601|linux_x86_64|e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855|123|https://dl.example.com/cpython.tar.gz" + ) + assert result2 == _ParsedPBSPython( + py_version=Version("3.10.1"), + pbs_release_tag=Version("20240601"), + platform=Platform.linux_x86_64, + url="https://dl.example.com/cpython.tar.gz", + sha256="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + size=123, + )