From 68d0159b6dfce07f144045d56639c52066e8b90e Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 23 Oct 2023 14:41:34 +1100 Subject: [PATCH 1/6] Fixed #34903, Refs #34825 -- Made workers initialization respect empty set of used connections. Thanks to David Smith for the investigation & patch. Regression in 2128a73713735fb794ca6565fd5d7792293f5cfa. Follow up to a5905b164dbf52e59fa646af9c3d523c0804d86a. Co-authored-by: David Sanders --- django/test/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/test/runner.py b/django/test/runner.py index ecd10d5d9168..8bb40a341365 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -431,7 +431,7 @@ def _init_worker( django.setup() setup_test_environment(debug=debug_mode) - db_aliases = used_aliases or connections + db_aliases = used_aliases if used_aliases is not None else connections for alias in db_aliases: connection = connections[alias] if start_method == "spawn": From 7fcf4f2f0f19c353fe3ee9fe2f6c4baeda4f03c8 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 23 Oct 2023 08:58:30 +0200 Subject: [PATCH 2/6] Bumped versions in pre-commit and npm configurations. --- .pre-commit-config.yaml | 8 ++++---- package.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a87425e0170..6304b41cc9c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ repos: - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.0 + rev: 23.10.0 hooks: - id: black exclude: \.py-tpl$ - repo: https://github.com/adamchainz/blacken-docs - rev: 1.13.0 + rev: 1.16.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.9.0 + - black==23.10.0 - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: @@ -19,6 +19,6 @@ repos: hooks: - id: flake8 - repo: https://github.com/pre-commit/mirrors-eslint - rev: v8.49.0 + rev: v8.52.0 hooks: - id: eslint diff --git a/package.json b/package.json index 510d50ad5b18..e9b3982dc68d 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,11 @@ "npm": ">=1.3.0" }, "devDependencies": { - "eslint": "^8.49.0", - "puppeteer": "^21.1.1", + "eslint": "^8.52.0", + "puppeteer": "^21.4.0", "grunt": "^1.6.1", "grunt-cli": "^1.4.3", - "grunt-contrib-qunit": "^7.0.0", - "qunit": "^2.19.4" + "grunt-contrib-qunit": "^8.0.1", + "qunit": "^2.20.0" } } From e2922b0d5f18169d1d0115a6db5d2ed8c42d0692 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Sun, 22 Oct 2023 23:17:33 +0100 Subject: [PATCH 3/6] Refs #34118 -- Avoided repeat coroutine checks in MiddlewareMixin. --- django/utils/deprecation.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/django/utils/deprecation.py b/django/utils/deprecation.py index 77ff6b1eaaa3..4d136dfa1631 100644 --- a/django/utils/deprecation.py +++ b/django/utils/deprecation.py @@ -100,7 +100,13 @@ def __init__(self, get_response): if get_response is None: raise ValueError("get_response must be provided.") self.get_response = get_response - self._async_check() + # If get_response is a coroutine function, turns us into async mode so + # a thread is not consumed during a whole request. + self.async_mode = iscoroutinefunction(self.get_response) + if self.async_mode: + # Mark the class as async-capable, but do the actual switch inside + # __call__ to avoid swapping out dunder methods. + markcoroutinefunction(self) super().__init__() def __repr__(self): @@ -113,19 +119,9 @@ def __repr__(self): ), ) - def _async_check(self): - """ - If get_response is a coroutine function, turns us into async mode so - a thread is not consumed during a whole request. - """ - if iscoroutinefunction(self.get_response): - # Mark the class as async-capable, but do the actual switch - # inside __call__ to avoid swapping out dunder methods - markcoroutinefunction(self) - def __call__(self, request): # Exit out to async mode, if needed - if iscoroutinefunction(self): + if self.async_mode: return self.__acall__(request) response = None if hasattr(self, "process_request"): From 07fa79ef2bb3e8cace7bd87b292c6c85230eed05 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Mon, 16 Oct 2023 19:25:17 +0100 Subject: [PATCH 4/6] Refs #31262 -- Added __eq__() and __getitem__() to BaseChoiceIterator. This makes it easier to work with lazy iterators used for callables, etc. when extracting items or comparing to lists, e.g. during testing. Also added `BaseChoiceIterator.__iter__()` to make it clear that subclasses must implement this and added `__all__` to the module. Co-authored-by: Adam Johnson Co-authored-by: Natalia Bidart <124304+nessita@users.noreply.github.com> --- django/utils/choices.py | 26 +++++++++++++ tests/utils_tests/test_choices.py | 64 +++++++++++++++++++++++++++---- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/django/utils/choices.py b/django/utils/choices.py index a0611d96f15c..734b9331a1d0 100644 --- a/django/utils/choices.py +++ b/django/utils/choices.py @@ -1,11 +1,37 @@ from collections.abc import Callable, Iterable, Iterator, Mapping +from itertools import islice, zip_longest from django.utils.functional import Promise +__all__ = [ + "BaseChoiceIterator", + "CallableChoiceIterator", + "normalize_choices", +] + class BaseChoiceIterator: """Base class for lazy iterators for choices.""" + def __eq__(self, other): + if isinstance(other, Iterable): + return all(a == b for a, b in zip_longest(self, other, fillvalue=object())) + return super().__eq__(other) + + def __getitem__(self, index): + if index < 0: + # Suboptimally consume whole iterator to handle negative index. + return list(self)[index] + try: + return next(islice(self, index, index + 1)) + except StopIteration: + raise IndexError("index out of range") from None + + def __iter__(self): + raise NotImplementedError( + "BaseChoiceIterator subclasses must implement __iter__()." + ) + class CallableChoiceIterator(BaseChoiceIterator): """Iterator to lazily normalize choices generated by a callable.""" diff --git a/tests/utils_tests/test_choices.py b/tests/utils_tests/test_choices.py index d96c3d49c4f4..a2ad5541a4e5 100644 --- a/tests/utils_tests/test_choices.py +++ b/tests/utils_tests/test_choices.py @@ -2,10 +2,60 @@ from django.db.models import TextChoices from django.test import SimpleTestCase -from django.utils.choices import CallableChoiceIterator, normalize_choices +from django.utils.choices import ( + BaseChoiceIterator, + CallableChoiceIterator, + normalize_choices, +) from django.utils.translation import gettext_lazy as _ +class SimpleChoiceIterator(BaseChoiceIterator): + def __iter__(self): + return ((i, f"Item #{i}") for i in range(1, 4)) + + +class ChoiceIteratorTests(SimpleTestCase): + def test_not_implemented_error_on_missing_iter(self): + class InvalidChoiceIterator(BaseChoiceIterator): + pass # Not overriding __iter__(). + + msg = "BaseChoiceIterator subclasses must implement __iter__()." + with self.assertRaisesMessage(NotImplementedError, msg): + iter(InvalidChoiceIterator()) + + def test_eq(self): + unrolled = [(1, "Item #1"), (2, "Item #2"), (3, "Item #3")] + self.assertEqual(SimpleChoiceIterator(), unrolled) + self.assertEqual(unrolled, SimpleChoiceIterator()) + + def test_eq_instances(self): + self.assertEqual(SimpleChoiceIterator(), SimpleChoiceIterator()) + + def test_not_equal_subset(self): + self.assertNotEqual(SimpleChoiceIterator(), [(1, "Item #1"), (2, "Item #2")]) + + def test_not_equal_superset(self): + self.assertNotEqual( + SimpleChoiceIterator(), + [(1, "Item #1"), (2, "Item #2"), (3, "Item #3"), None], + ) + + def test_getitem(self): + choices = SimpleChoiceIterator() + for i, expected in [(0, (1, "Item #1")), (-1, (3, "Item #3"))]: + with self.subTest(index=i): + self.assertEqual(choices[i], expected) + + def test_getitem_indexerror(self): + choices = SimpleChoiceIterator() + for i in (4, -4): + with self.subTest(index=i): + with self.assertRaises(IndexError) as ctx: + choices[i] + self.assertTrue(str(ctx.exception).endswith("index out of range")) + + class NormalizeFieldChoicesTests(SimpleTestCase): expected = [ ("C", _("Club")), @@ -84,7 +134,7 @@ def get_choices(): get_choices_spy.assert_not_called() self.assertIsInstance(output, CallableChoiceIterator) - self.assertEqual(list(output), self.expected) + self.assertEqual(output, self.expected) get_choices_spy.assert_called_once() def test_mapping(self): @@ -134,7 +184,7 @@ def get_media_choices(): get_media_choices_spy.assert_not_called() self.assertIsInstance(output, CallableChoiceIterator) - self.assertEqual(list(output), self.expected_nested) + self.assertEqual(output, self.expected_nested) get_media_choices_spy.assert_called_once() def test_nested_mapping(self): @@ -185,7 +235,7 @@ def get_choices(): get_choices_spy.assert_not_called() self.assertIsInstance(output, CallableChoiceIterator) - self.assertEqual(list(output), self.expected) + self.assertEqual(output, self.expected) get_choices_spy.assert_called_once() def test_iterable_non_canonical(self): @@ -230,7 +280,7 @@ def get_media_choices(): get_media_choices_spy.assert_not_called() self.assertIsInstance(output, CallableChoiceIterator) - self.assertEqual(list(output), self.expected_nested) + self.assertEqual(output, self.expected_nested) get_media_choices_spy.assert_called_once() def test_nested_iterable_non_canonical(self): @@ -294,12 +344,12 @@ def test_unsupported_values_returned_unmodified(self): def test_unsupported_values_from_callable_returned_unmodified(self): for value in self.invalid_iterable + self.invalid_nested: with self.subTest(value=value): - self.assertEqual(list(normalize_choices(lambda: value)), value) + self.assertEqual(normalize_choices(lambda: value), value) def test_unsupported_values_from_iterator_returned_unmodified(self): for value in self.invalid_nested: with self.subTest(value=value): self.assertEqual( - list(normalize_choices((lambda: (yield from value))())), + normalize_choices((lambda: (yield from value))()), value, ) From 74afcee234f8be989623ccc7c28b9fb97fb548f0 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Mon, 16 Oct 2023 19:07:49 +0100 Subject: [PATCH 5/6] Refs #34899 -- Extracted Field.flatchoices to flatten_choices helper function. Co-authored-by: Natalia Bidart <124304+nessita@users.noreply.github.com> --- django/db/models/fields/__init__.py | 21 ++++++--------- django/utils/choices.py | 10 +++++++ tests/utils_tests/test_choices.py | 42 +++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index f15b5856bfb1..6174b7bc98cb 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -15,7 +15,11 @@ from django.db.models.constants import LOOKUP_SEP from django.db.models.query_utils import DeferredAttribute, RegisterLookupMixin from django.utils import timezone -from django.utils.choices import CallableChoiceIterator, normalize_choices +from django.utils.choices import ( + CallableChoiceIterator, + flatten_choices, + normalize_choices, +) from django.utils.datastructures import DictWrapper from django.utils.dateparse import ( parse_date, @@ -1080,19 +1084,10 @@ def value_to_string(self, obj): """ return str(self.value_from_object(obj)) - def _get_flatchoices(self): + @property + def flatchoices(self): """Flattened version of choices tuple.""" - if self.choices is None: - return [] - flat = [] - for choice, value in self.choices: - if isinstance(value, (list, tuple)): - flat.extend(value) - else: - flat.append((choice, value)) - return flat - - flatchoices = property(_get_flatchoices) + return list(flatten_choices(self.choices)) def save_form_data(self, instance, data): setattr(instance, self.name, data) diff --git a/django/utils/choices.py b/django/utils/choices.py index 734b9331a1d0..54dbdcb3aac7 100644 --- a/django/utils/choices.py +++ b/django/utils/choices.py @@ -6,6 +6,7 @@ __all__ = [ "BaseChoiceIterator", "CallableChoiceIterator", + "flatten_choices", "normalize_choices", ] @@ -43,6 +44,15 @@ def __iter__(self): yield from normalize_choices(self.func()) +def flatten_choices(choices): + """Flatten choices by removing nested values.""" + for value_or_group, label_or_nested in choices or (): + if isinstance(label_or_nested, (list, tuple)): + yield from label_or_nested + else: + yield value_or_group, label_or_nested + + def normalize_choices(value, *, depth=0): """Normalize choices values consistently for fields and widgets.""" # Avoid circular import when importing django.forms. diff --git a/tests/utils_tests/test_choices.py b/tests/utils_tests/test_choices.py index a2ad5541a4e5..e3e3766ea92c 100644 --- a/tests/utils_tests/test_choices.py +++ b/tests/utils_tests/test_choices.py @@ -1,3 +1,4 @@ +import collections.abc from unittest import mock from django.db.models import TextChoices @@ -5,6 +6,7 @@ from django.utils.choices import ( BaseChoiceIterator, CallableChoiceIterator, + flatten_choices, normalize_choices, ) from django.utils.translation import gettext_lazy as _ @@ -56,6 +58,46 @@ def test_getitem_indexerror(self): self.assertTrue(str(ctx.exception).endswith("index out of range")) +class FlattenChoicesTests(SimpleTestCase): + def test_empty(self): + def generator(): + yield from () + + for choices in ({}, [], (), set(), frozenset(), generator(), None, ""): + with self.subTest(choices=choices): + result = flatten_choices(choices) + self.assertIsInstance(result, collections.abc.Generator) + self.assertEqual(list(result), []) + + def test_non_empty(self): + choices = [ + ("C", _("Club")), + ("D", _("Diamond")), + ("H", _("Heart")), + ("S", _("Spade")), + ] + result = flatten_choices(choices) + self.assertIsInstance(result, collections.abc.Generator) + self.assertEqual(list(result), choices) + + def test_nested_choices(self): + choices = [ + ("Audio", [("vinyl", _("Vinyl")), ("cd", _("CD"))]), + ("Video", [("vhs", _("VHS Tape")), ("dvd", _("DVD"))]), + ("unknown", _("Unknown")), + ] + expected = [ + ("vinyl", _("Vinyl")), + ("cd", _("CD")), + ("vhs", _("VHS Tape")), + ("dvd", _("DVD")), + ("unknown", _("Unknown")), + ] + result = flatten_choices(choices) + self.assertIsInstance(result, collections.abc.Generator) + self.assertEqual(list(result), expected) + + class NormalizeFieldChoicesTests(SimpleTestCase): expected = [ ("C", _("Club")), From 171f91d9ef5177850c2f12b26dd732785f6ac034 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Mon, 16 Oct 2023 19:11:18 +0100 Subject: [PATCH 6/6] Fixed #34899 -- Added blank choice to forms' callable choices lazily. --- django/db/models/fields/__init__.py | 10 +++------ django/utils/choices.py | 17 ++++++++++++++- tests/model_forms/tests.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py index 6174b7bc98cb..205a41c1935f 100644 --- a/django/db/models/fields/__init__.py +++ b/django/db/models/fields/__init__.py @@ -16,6 +16,7 @@ from django.db.models.query_utils import DeferredAttribute, RegisterLookupMixin from django.utils import timezone from django.utils.choices import ( + BlankChoiceIterator, CallableChoiceIterator, flatten_choices, normalize_choices, @@ -1055,14 +1056,9 @@ def get_choices( as