From bb24e61e103cb94a9cea43dc741702140fcec8a1 Mon Sep 17 00:00:00 2001 From: KlemenSpruk Date: Mon, 20 Nov 2023 13:50:02 +0100 Subject: [PATCH] tox fix and working tests --- django_project_base/celery/celery.py | 4 +- .../notifications/base/channels/channel.py | 25 +++++++-- .../base/channels/integrations/aws_ses.py | 4 +- .../integrations/aws_sns_single_sms.py | 4 +- .../base/channels/integrations/nexmo_sms.py | 4 +- .../integrations/provider_integration.py | 4 +- .../base/channels/integrations/t2.py | 4 +- .../base/channels/mail_channel.py | 14 ++++- .../base/channels/sms_channel.py | 13 ++++- .../notifications/base/notification.py | 56 ++++++++----------- .../notifications/base/phone_number_parser.py | 14 ++--- .../base/send_notification_service.py | 11 +--- .../notifications/email_notification.py | 2 +- ...0_djangoprojectbasenotification_send_at.py | 5 -- django_project_base/notifications/models.py | 1 - .../notifications/rest/notification.py | 3 +- .../tests/api/test_remainig_license.py | 6 +- .../tests/unit/test_is_mail_sent.py | 6 +- django_project_base/settings.py | 2 + example/setup/settings.py | 1 - requirements.txt | 1 - 21 files changed, 98 insertions(+), 86 deletions(-) diff --git a/django_project_base/celery/celery.py b/django_project_base/celery/celery.py index a49612a6..87f7db42 100644 --- a/django_project_base/celery/celery.py +++ b/django_project_base/celery/celery.py @@ -1,10 +1,10 @@ import os -from celery import Celery, bootsteps +from celery import bootsteps, Celery +from click import Option from django.apps import apps from django.utils.crypto import get_random_string from kombu import Exchange, Queue -from click import Option from kombu.entity import TRANSIENT_DELIVERY_MODE from django_project_base.celery.settings import NOTIFICATIONS_QUEUE_VISIBILITY_TIMEOUT diff --git a/django_project_base/notifications/base/channels/channel.py b/django_project_base/notifications/base/channels/channel.py index c8454b6d..7b4b378e 100644 --- a/django_project_base/notifications/base/channels/channel.py +++ b/django_project_base/notifications/base/channels/channel.py @@ -25,13 +25,22 @@ class Recipient: email: str unique_attribute: str - def __init__(self, identifier: str, phone_number: str, email: str, unique_attribute: str = "identifier") -> None: + def __init__( + self, + identifier: str, + phone_number: str, + email: str, + unique_attribute: str = "identifier", + phone_number_validator=None, + ) -> None: super().__init__() self.identifier = identifier self.phone_number = ( next( iter( - PhoneNumberParser.valid_phone_numbers([phone_number]) if phone_number and len(phone_number) else "" + PhoneNumberParser.valid_phone_numbers([phone_number], phone_number_validator) + if phone_number and len(phone_number) + else "" ), None, ) @@ -79,7 +88,6 @@ def _find_provider( exclude = [] def get_first_provider(val: Union[str, List]): - print("PROVIDER", val, setting_name, settings) if val and isinstance(val, (list, tuple)): prov = next(filter(lambda i: i not in exclude, val), None) @@ -113,7 +121,9 @@ def create_delivery_report( ) @abstractmethod - def get_recipients(self, notification: DjangoProjectBaseNotification, unique_identifier="email") -> List[Recipient]: + def get_recipients( + self, notification: DjangoProjectBaseNotification, unique_identifier="email", phone_number_validator=None + ) -> List[Recipient]: rec_obj = notification.email_list if not rec_obj: rec_obj = notification.recipients_list @@ -131,6 +141,7 @@ def get_recipients(self, notification: DjangoProjectBaseNotification, unique_ide email=u.get("email", "") or "", phone_number=u.get("phone_number", "") or "", unique_attribute=unique_identifier, + phone_number_validator=phone_number_validator, ) for u in rec_obj ] @@ -177,7 +188,9 @@ def send(self, notification: DjangoProjectBaseNotification, extra_data, settings try: message = self.provider.get_message(notification) - recipients = self.get_recipients(notification) + recipients = self.get_recipients( + notification, phone_number_validator=getattr(settings, "IS_PHONE_NUMBER_ALLOWED_FUNCTION", None) + ) if not recipients: raise ValueError("No valid recipients") @@ -222,7 +235,9 @@ def send(self, notification: DjangoProjectBaseNotification, extra_data, settings a_sender=notification.sender, a_extra_data=extra_data, a_recipients_list=notification.recipients_list, + a_settings=settings, delay=int(datetime.datetime.now().timestamp()), + user=extra_data["user"], ).send() self.create_delivery_report( notification, recipient, dlr__uuid, auxiliary_notification=a_notification.pk diff --git a/django_project_base/notifications/base/channels/integrations/aws_ses.py b/django_project_base/notifications/base/channels/integrations/aws_ses.py index b2314017..534488e7 100644 --- a/django_project_base/notifications/base/channels/integrations/aws_ses.py +++ b/django_project_base/notifications/base/channels/integrations/aws_ses.py @@ -1,7 +1,7 @@ -from typing import Union, Optional +from typing import Optional, Union import boto3 -from django.conf import settings, Settings +from django.conf import Settings from rest_framework.status import is_success from django_project_base.notifications.base.channels.channel import Recipient diff --git a/django_project_base/notifications/base/channels/integrations/aws_sns_single_sms.py b/django_project_base/notifications/base/channels/integrations/aws_sns_single_sms.py index a772f957..e276add8 100644 --- a/django_project_base/notifications/base/channels/integrations/aws_sns_single_sms.py +++ b/django_project_base/notifications/base/channels/integrations/aws_sns_single_sms.py @@ -1,7 +1,7 @@ -from typing import Union, Optional +from typing import Optional, Union import boto3 -from django.conf import settings, Settings +from django.conf import Settings from rest_framework.status import is_success from django_project_base.notifications.base.channels.channel import Recipient diff --git a/django_project_base/notifications/base/channels/integrations/nexmo_sms.py b/django_project_base/notifications/base/channels/integrations/nexmo_sms.py index 0c1c4afb..c65c04d1 100644 --- a/django_project_base/notifications/base/channels/integrations/nexmo_sms.py +++ b/django_project_base/notifications/base/channels/integrations/nexmo_sms.py @@ -1,7 +1,7 @@ -from typing import Union, Optional +from typing import Optional, Union import requests -from django.conf import settings, Settings +from django.conf import Settings from rest_framework.status import is_success from django_project_base.notifications.base.channels.channel import Recipient diff --git a/django_project_base/notifications/base/channels/integrations/provider_integration.py b/django_project_base/notifications/base/channels/integrations/provider_integration.py index b5d77f1f..a3cda661 100644 --- a/django_project_base/notifications/base/channels/integrations/provider_integration.py +++ b/django_project_base/notifications/base/channels/integrations/provider_integration.py @@ -1,7 +1,7 @@ import re from abc import ABC, abstractmethod from html import unescape -from typing import Union, Optional +from typing import Optional, Union import swapper from django.conf import Settings @@ -19,7 +19,7 @@ class ProviderIntegration(ABC): def __init__(self, settings: object) -> None: super().__init__() - self.settings = Settings + self.settings = settings @abstractmethod def validate_send(self, response: dict): diff --git a/django_project_base/notifications/base/channels/integrations/t2.py b/django_project_base/notifications/base/channels/integrations/t2.py index 8d754c40..613c7da7 100644 --- a/django_project_base/notifications/base/channels/integrations/t2.py +++ b/django_project_base/notifications/base/channels/integrations/t2.py @@ -1,9 +1,9 @@ import json -from typing import Union, Optional +from typing import Optional, Union import requests import swapper -from django.conf import settings, Settings +from django.conf import Settings from django.contrib.auth import get_user_model from requests.auth import HTTPBasicAuth from rest_framework.status import is_success diff --git a/django_project_base/notifications/base/channels/mail_channel.py b/django_project_base/notifications/base/channels/mail_channel.py index db1c66d4..e8eb246f 100644 --- a/django_project_base/notifications/base/channels/mail_channel.py +++ b/django_project_base/notifications/base/channels/mail_channel.py @@ -1,7 +1,7 @@ import uuid from typing import List -from django.conf import settings, Settings +from django.conf import Settings from django.core.exceptions import ValidationError from django.core.validators import validate_email @@ -32,8 +32,16 @@ def send(self, notification: DjangoProjectBaseNotification, extra_data, settings ) return super().send(notification=notification, extra_data=extra_data, settings=settings) - def get_recipients(self, notification: DjangoProjectBaseNotification, unique_identifier=""): - return list(set(super().get_recipients(notification, unique_identifier="email"))) + def get_recipients( + self, notification: DjangoProjectBaseNotification, unique_identifier="", phone_number_validator=None + ): + return list( + set( + super().get_recipients( + notification, unique_identifier="email", phone_number_validator=phone_number_validator + ) + ) + ) def clean_email_recipients(self, recipients: List[Recipient]) -> List[Recipient]: valid = [] diff --git a/django_project_base/notifications/base/channels/sms_channel.py b/django_project_base/notifications/base/channels/sms_channel.py index 655e83c7..8b495d2a 100644 --- a/django_project_base/notifications/base/channels/sms_channel.py +++ b/django_project_base/notifications/base/channels/sms_channel.py @@ -1,6 +1,7 @@ from typing import List from django.conf import Settings + from django_project_base.notifications.base.channels.channel import Channel, Recipient from django_project_base.notifications.base.enums import ChannelIdentifier from django_project_base.notifications.models import DjangoProjectBaseNotification @@ -15,8 +16,16 @@ class SmsChannel(Channel): provider_setting_name = "NOTIFICATIONS_SMS_PROVIDER" - def get_recipients(self, notification: DjangoProjectBaseNotification, unique_identifier=""): - return list(set(super().get_recipients(notification, unique_identifier="phone_number"))) + def get_recipients( + self, notification: DjangoProjectBaseNotification, unique_identifier="", phone_number_validator=None + ): + return list( + set( + super().get_recipients( + notification, unique_identifier="phone_number", phone_number_validator=phone_number_validator + ) + ) + ) def send( self, notification: DjangoProjectBaseNotification, extra_data: dict, settings: Settings, **kwargs diff --git a/django_project_base/notifications/base/notification.py b/django_project_base/notifications/base/notification.py index 6c92146b..0cb281fb 100644 --- a/django_project_base/notifications/base/notification.py +++ b/django_project_base/notifications/base/notification.py @@ -94,9 +94,7 @@ def __init__( @staticmethod def resend(notification: DjangoProjectBaseNotification, user_pk: Optional[str] = None): - notification.user = user_pk or ( - notification.extra_data or {} - ) # TODO: refactor this to read from @prop which always returns {} + notification.user = user_pk from django_project_base.notifications.rest.notification import MessageToListField recipients: List[str] = MessageToListField.parse(json.loads(notification.recipients_original_payload)) @@ -112,7 +110,9 @@ def resend(notification: DjangoProjectBaseNotification, user_pk: Optional[str] = ) notification.email_fallback = mail_fallback notification.save(update_fields=["recipients", "recipients_original_payload_search"]) - SendNotificationService(settings=settings).make_send(notification, notification.extra_data or {}, resend=True) + SendNotificationService(settings=settings, use_default_db_connection=True).make_send( + notification, {}, resend=True + ) def __set_via_channels(self, val): self._via_channels = val @@ -175,7 +175,6 @@ def send(self) -> DjangoProjectBaseNotification: send_notification_sms=self.send_notification_sms, send_notification_sms_text=None, send_at=self.send_at, - extra_data=self._extra_data, ) notification = self._ensure_channels(channels=required_channels, notification=notification, settings=settings) @@ -210,8 +209,6 @@ def send(self) -> DjangoProjectBaseNotification: ) notification.recipients_list = rec_list ext_data = self._extra_data.get("a_extra_data") or self._extra_data - - notification.save(update_fields=["extra_data"]) self.enqueue_notification(notification, extra_data=ext_data) return notification @@ -219,31 +216,25 @@ def send(self) -> DjangoProjectBaseNotification: SendNotificationService(settings=settings, use_default_db_connection=True).make_send( notification, self._extra_data, resend=False ) - else: - if not self.persist: - raise Exception("Delayed notification must be persisted") - mail_fallback: bool = ( - swapper.load_model("django_project_base", "ProjectSettings") - .objects.get(name=USE_EMAIL_IF_RECIPIENT_HAS_NO_PHONE_NUMBER, project__slug=notification.project_slug) - .python_value - if notification.project_slug - else False - ) - notification.extra_data["mail-fallback"] = mail_fallback - rec_list = [] - for usr in self._recipients: - rec_list.append( - { - k: v - for k, v in get_user_model().objects.get(pk=usr).userprofile.__dict__.items() - if not k.startswith("_") and isinstance(v, (str, list, tuple, int, float)) - } - ) - notification.extra_data["recipients-list"] = rec_list - notification.extra_data["sender"] = notification.sender - - notification.save(update_fields=["extra_data"]) - + # else: + # if not self.persist: + # raise Exception("Delayed notification must be persisted") + # mail_fallback: bool = ( + # swapper.load_model("django_project_base", "ProjectSettings") + # .objects.get(name=USE_EMAIL_IF_RECIPIENT_HAS_NO_PHONE_NUMBER, project__slug=notification.project_slug) + # .python_value + # if notification.project_slug + # else False + # ) + # rec_list = [] + # for usr in self._recipients: + # rec_list.append( + # { + # k: v + # for k, v in get_user_model().objects.get(pk=usr).userprofile.__dict__.items() + # if not k.startswith("_") and isinstance(v, (str, list, tuple, int, float)) + # } + # ) return notification def _ensure_channels( @@ -255,6 +246,7 @@ def _ensure_channels( from django_project_base.notifications.base.channels.mail_channel import MailChannel extra_data = self._extra_data.get("a_extra_data") or self._extra_data + settings = self._extra_data.get("a_settings") or settings for channel_name in channels: # ensure dlr user and check providers channel = ChannelIdentifier.channel(channel_name, settings=settings, project_slug=self._project) diff --git a/django_project_base/notifications/base/phone_number_parser.py b/django_project_base/notifications/base/phone_number_parser.py index 8c1ee472..66d2d00f 100644 --- a/django_project_base/notifications/base/phone_number_parser.py +++ b/django_project_base/notifications/base/phone_number_parser.py @@ -1,22 +1,18 @@ from typing import List -from django.core.cache import cache - class PhoneNumberParser: @staticmethod - def is_allowed(phone_number: str) -> bool: - if (allowed_function := cache.get("IS_PHONE_NUMBER_ALLOWED_FUNCTION".lower(), "")) and allowed_function: - from dill import loads as dloads - - return dloads(allowed_function)(phone_number) + def is_allowed(phone_number: str, allowed_validator=None) -> bool: + if allowed_validator: + return allowed_validator(phone_number) return phone_number and len(phone_number) >= 8 @staticmethod - def valid_phone_numbers(candidates: List[str]) -> List[str]: + def valid_phone_numbers(candidates: List[str], allowed_validator=None) -> List[str]: valid: List[str] = [] for number in candidates: - if not PhoneNumberParser.is_allowed(number): + if not PhoneNumberParser.is_allowed(number, allowed_validator): continue if number.startswith("+"): valid.append(number.lstrip("+")) diff --git a/django_project_base/notifications/base/send_notification_service.py b/django_project_base/notifications/base/send_notification_service.py index 92eb2dea..7ed3b230 100644 --- a/django_project_base/notifications/base/send_notification_service.py +++ b/django_project_base/notifications/base/send_notification_service.py @@ -1,10 +1,9 @@ import logging from django import db -from django.core.cache import cache +from django.conf import Settings from django.utils import timezone -from django.conf import Settings from django_project_base.constants import NOTIFICATION_QUEUE_NAME from django_project_base.notifications.base.enums import ChannelIdentifier from django_project_base.notifications.models import DjangoProjectBaseNotification @@ -33,13 +32,6 @@ def make_send( if notification.required_channels is None: return notification - if ( - self.settings - and (phn_allowed := getattr(self.settings, "IS_PHONE_NUMBER_ALLOWED_FUNCTION", "")) - and phn_allowed - ): - cache.set("IS_PHONE_NUMBER_ALLOWED_FUNCTION".lower(), phn_allowed, timeout=None) - already_sent_channels = set( filter( lambda i: i not in (None, "") and i, @@ -65,7 +57,6 @@ def make_send( required_channels.add(SmsChannel.name) for channel_identifier in required_channels: - print(self.settings) channel = ChannelIdentifier.channel( channel_identifier, settings=self.settings, diff --git a/django_project_base/notifications/email_notification.py b/django_project_base/notifications/email_notification.py index 261c7920..309b76e5 100644 --- a/django_project_base/notifications/email_notification.py +++ b/django_project_base/notifications/email_notification.py @@ -2,8 +2,8 @@ import uuid from typing import List, Optional, Type -from django.core.cache import cache from django.conf import settings +from django.core.cache import cache from rest_framework.exceptions import PermissionDenied from django_project_base.notifications.base.channels.channel import Channel diff --git a/django_project_base/notifications/migrations/0010_djangoprojectbasenotification_send_at.py b/django_project_base/notifications/migrations/0010_djangoprojectbasenotification_send_at.py index b5960e8e..bd9bd366 100644 --- a/django_project_base/notifications/migrations/0010_djangoprojectbasenotification_send_at.py +++ b/django_project_base/notifications/migrations/0010_djangoprojectbasenotification_send_at.py @@ -14,9 +14,4 @@ class Migration(migrations.Migration): name="send_at", field=models.BigIntegerField(null=True), ), - migrations.AddField( - model_name="djangoprojectbasenotification", - name="extra_data", - field=models.JSONField(null=True), - ), ] diff --git a/django_project_base/notifications/models.py b/django_project_base/notifications/models.py index eec42914..9b2b60c9 100644 --- a/django_project_base/notifications/models.py +++ b/django_project_base/notifications/models.py @@ -80,7 +80,6 @@ class AbstractDjangoProjectBaseNotification(models.Model): send_notification_sms = models.BooleanField(null=False, blank=False, default=False) send_notification_sms_text = models.TextField(blank=False, null=True) send_at = models.BigIntegerField(blank=False, null=True) - extra_data = models.JSONField(null=True, blank=False) class Meta: abstract = True diff --git a/django_project_base/notifications/rest/notification.py b/django_project_base/notifications/rest/notification.py index db97277d..8b7ad930 100644 --- a/django_project_base/notifications/rest/notification.py +++ b/django_project_base/notifications/rest/notification.py @@ -5,9 +5,9 @@ import pytz import swapper +from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType -from django.conf import settings from django.core.cache import cache from django.db.models import ForeignKey, QuerySet from django.http import Http404, HttpResponse @@ -253,7 +253,6 @@ class Meta: "delayed_to", "recipients_original_payload_search", "exceptions", - "extra_data", ) layout = Layout( Row(Column("message_to")), diff --git a/django_project_base/notifications/tests/api/test_remainig_license.py b/django_project_base/notifications/tests/api/test_remainig_license.py index b5997027..4b114cde 100644 --- a/django_project_base/notifications/tests/api/test_remainig_license.py +++ b/django_project_base/notifications/tests/api/test_remainig_license.py @@ -1,3 +1,5 @@ +from django.conf import settings + from django_project_base.licensing.logic import LogAccessService from django_project_base.notifications.base.enums import ChannelIdentifier from django_project_base.notifications.tests.notifications_transaction_test_case import NotificationsTransactionTestCase @@ -9,5 +11,7 @@ def test_remaining_license(self): log_service = LogAccessService() usage = log_service.report(user=self.test_user) credit = log_service._user_access_user_inflow(self.test_user.pk) - channel = ChannelIdentifier.channel(mail.required_channels.split(",")[0], ensure_dlr_user=False) + channel = ChannelIdentifier.channel( + mail.required_channels.split(",")[0], ensure_dlr_user=False, settings=settings + ) self.assertEqual(credit - channel.notification_price, usage["remaining_credit"]) diff --git a/django_project_base/notifications/tests/unit/test_is_mail_sent.py b/django_project_base/notifications/tests/unit/test_is_mail_sent.py index bef37bc7..98d8532a 100644 --- a/django_project_base/notifications/tests/unit/test_is_mail_sent.py +++ b/django_project_base/notifications/tests/unit/test_is_mail_sent.py @@ -1,5 +1,7 @@ import uuid +from django.conf import settings + from django_project_base.licensing.logic import LogAccessService from django_project_base.notifications.base.channels.channel import Recipient from django_project_base.notifications.base.enums import ChannelIdentifier @@ -27,7 +29,9 @@ def test_resend_email(self): self.assertIsNotNone(notification.sent_at) self.assertFalse(bool(notification.failed_channels)) self.assertEqual(notification.required_channels, notification.sent_channels) - channel = ChannelIdentifier.channel(notification.required_channels.split(",")[0], ensure_dlr_user=False) + channel = ChannelIdentifier.channel( + notification.required_channels.split(",")[0], ensure_dlr_user=False, settings=settings + ) dlr_pk = str(uuid.uuid4()) dlr_count = DeliveryReport.objects.all().count() resend_data = channel._make_send( diff --git a/django_project_base/settings.py b/django_project_base/settings.py index 2024bf8d..d9c36899 100644 --- a/django_project_base/settings.py +++ b/django_project_base/settings.py @@ -41,6 +41,8 @@ }, }, {"name": "NOTIFICATION_SENDERS", "default": {}}, + {"name": "SYSTEM_EMAIL_SENDER_ID", "default": ""}, + {"name": "SYSTEM_SMS_SENDER_ID", "default": ""}, { "name": "NOTIFICATIONS_EMAIL_PROVIDER", "default": "django_project_base.notifications.base.channels.integrations.aws_ses.AwsSes", diff --git a/example/setup/settings.py b/example/setup/settings.py index 35d5539d..59b8a479 100644 --- a/example/setup/settings.py +++ b/example/setup/settings.py @@ -239,4 +239,3 @@ } DEFAULT_AUTO_FIELD = "django.db.models.AutoField" - diff --git a/requirements.txt b/requirements.txt index 9812a6d4..10c34d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,5 +18,4 @@ natural celery==5.2.7 django-redis-cache pandas -dill==0.3.7 click==8.1.7