Celery deprecations, closes #243

celeryconfig.py

@@ -0,0 +1,60 @@
+from celery.schedules import crontab
+
+
+timezone = 'Europe/Berlin'
+
+include = (
+    'gitmostwanted.tasks.repo_most_starred', 'gitmostwanted.tasks.repo_stars',
+    'gitmostwanted.tasks.repo_metadata', 'gitmostwanted.tasks.repo_status',
+    'gitmostwanted.tasks.github'
+)
+
+# celery beat
+beat_schedule = {
+    'metadata-refresh': {
+        'task': 'gitmostwanted.tasks.repo_metadata.metadata_refresh',
+        'schedule': crontab(minute=30, hour=0),
+        'args': [14]
+    },
+    'metadata-erase': {
+        'task': 'gitmostwanted.tasks.repo_metadata.metadata_erase',
+        'schedule': crontab(minute=50, hour=0)
+    },
+    'metadata-maturity': {
+        'task': 'gitmostwanted.tasks.repo_metadata.metadata_maturity',
+        'schedule': crontab(minute=30, hour=1),
+        'args': [3]
+    },
+    'metadata-trend': {
+        'task': 'gitmostwanted.tasks.repo_metadata.metadata_trend',
+        'schedule': crontab(minute=0, hour=5, day_of_month=1),
+        'args': [56]
+    },
+    'most-starred-day': {
+        'task': 'gitmostwanted.tasks.repo_most_starred.most_starred_day',
+        'schedule': crontab(minute=0, hour=8)
+    },
+    'most-starred-week': {
+        'task': 'gitmostwanted.tasks.repo_most_starred.most_starred_week',
+        'schedule': crontab(minute=0, hour=9, day_of_week=0)
+    },
+    'most-starred-month': {
+        'task': 'gitmostwanted.tasks.repo_most_starred.most_starred_month',
+        'schedule': crontab(minute=0, hour=10, day_of_month=1)
+    },
+    'status-refresh': {
+        'task': 'gitmostwanted.tasks.repo_status.status_refresh',
+        'schedule': crontab(minute=0, hour=3),
+        'args': [28]
+    },
+    'status-detect': {
+        'task': 'gitmostwanted.tasks.repo_status.status_detect',
+        'schedule': crontab(minute=30, hour=4),
+        'args': [28, 4]
+    },
+    'stars-mature': {
+        'task': 'gitmostwanted.tasks.repo_stars.stars_mature',
+        'schedule': crontab(minute=30, hour=3, day_of_week='*/2'),
+        'args': [28]
+    }
+}

gitmostwanted/app.py

@@ -3,12 +3,10 @@
 from flask import Flask
 
 from gitmostwanted.services import \
-    celery as service_celery, db as service_db, log as service_log, sentry as service_sentry
+    celery_dtq as service_celery, db as service_db, log as service_log, sentry as service_sentry
 
 env = environ.get('GMW_APP_ENV', 'development').capitalize()
 
-print("env:\n\tGMW_APP_ENV: " + env)
-
 app = Flask(__name__)
 app.config.from_object('gitmostwanted.config.Config' + env)
 app.config.from_envvar('GMW_APP_SETTINGS', silent=True)
@@ -19,4 +17,6 @@
 db = service_db.instance(app)
 log = service_log.instance(app)
 
+log.info("Environment used: %s", app.config.get('ENVIRONMENT'))
+
 del environ, Flask, service_celery, service_db, service_log, service_sentry

gitmostwanted/config.py

@@ -1,8 +1,9 @@
 # pylint: disable=C1001
 class Config:
+    ENVIRONMENT = 'development'
+
     # Custom
     DEBUG_FILE = '/tmp/gmw.log'
-    ENVIRONMENT = 'development'
 
     # Application related
     REPOSITORY_WORTH_SOLID = 6
@@ -19,10 +20,6 @@ class Config:
     SQLALCHEMY_POOL_RECYCLE = 3600
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 
-    # Celery
-    CELERY_TIMEZONE = 'Europe/Berlin'
-    CELERY_BROKER_URL = ''
-
     # Oauth
     GITHUB_AUTH = (None, None)
     GITHUB_OAUTH = {}
@@ -34,13 +31,16 @@ class ConfigDevelopment(Config):
 
 
 class ConfigTesting(Config):
-    SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
     ENVIRONMENT = 'testing'
+
+    SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
     GITHUB_AUTH = ('Test', '')
     SECRET_KEY = 'testing'  # noqa
     TESTING = True
 
 
 class ConfigProduction(Config):
-    DEBUG = False
     ENVIRONMENT = 'production'
+
+    DEBUG = False
+

gitmostwanted/lib/celery/flask_context_task.py

@@ -0,0 +1,11 @@
+from abc import ABC
+
+from celery import Task
+
+from gitmostwanted.app import app
+
+
+class FlaskContextTask(Task, ABC):
+    def __call__(self, *args, **kwargs):
+        with app.app_context():
+            return super().__call__(*args, **kwargs)

gitmostwanted/services/celery_dtq.py

@@ -0,0 +1,11 @@
+from celery import Celery
+
+
+def instance(app):
+    """:rtype: Celery"""
+    return Celery(
+        app.import_name,
+        broker=app.config.get('CELERY_URL_BROKER'),
+        backend=app.config.get('CELERY_URL_BACKEND'),
+        task_cls='lib.celery.flask_context_task:FlaskContextTask'
+    )

instance.cfg.distr

@@ -1,69 +1,12 @@
-from celery.schedules import crontab
-
 # flask
 SECRET_KEY = 'secret_key'
 
 # SQLAlchemy
 SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://gitmostwanted:gitmostwanted_pwd@gmw_db/gitmostwanted?charset=utf8mb4'
 
 # celery
-CELERY_BROKER_URL = 'redis://gmw_redis:6379/0',
-CELERY_RESULT_BACKEND = 'redis://gmw_redis:6379/0'
-
-# celery beat
-CELERY_INCLUDE = (
-    'gitmostwanted.tasks.repo_most_starred', 'gitmostwanted.tasks.repo_stars',
-    'gitmostwanted.tasks.repo_metadata', 'gitmostwanted.tasks.repo_status',
-    'gitmostwanted.tasks.github'
-)
-CELERYBEAT_SCHEDULE = {
-    'metadata-refresh': {
-        'task': 'gitmostwanted.tasks.repo_metadata.metadata_refresh',
-        'schedule': crontab(minute=30, hour=0),
-        'args': [14]
-    },
-    'metadata-erase': {
-        'task': 'gitmostwanted.tasks.repo_metadata.metadata_erase',
-        'schedule': crontab(minute=50, hour=0)
-    },
-    'metadata-maturity': {
-        'task': 'gitmostwanted.tasks.repo_metadata.metadata_maturity',
-        'schedule': crontab(minute=30, hour=1),
-        'args': [3]
-    },
-    'metadata-trend': {
-        'task': 'gitmostwanted.tasks.repo_metadata.metadata_trend',
-        'schedule': crontab(minute=0, hour=5, day_of_month=1),
-        'args': [56]
-    },
-    'most-starred-day': {
-        'task': 'gitmostwanted.tasks.repo_most_starred.most_starred_day',
-        'schedule': crontab(minute=0, hour=8)
-    },
-    'most-starred-week': {
-        'task': 'gitmostwanted.tasks.repo_most_starred.most_starred_week',
-        'schedule': crontab(minute=0, hour=9, day_of_week=0)
-    },
-    'most-starred-month': {
-        'task': 'gitmostwanted.tasks.repo_most_starred.most_starred_month',
-        'schedule': crontab(minute=0, hour=10, day_of_month=1)
-    },
-    'status-refresh': {
-        'task': 'gitmostwanted.tasks.repo_status.status_refresh',
-        'schedule': crontab(minute=0, hour=3),
-        'args': [28]
-    },
-    'status-detect': {
-        'task': 'gitmostwanted.tasks.repo_status.status_detect',
-        'schedule': crontab(minute=30, hour=4),
-        'args': [28, 4]
-    },
-    'stars-mature': {
-        'task': 'gitmostwanted.tasks.repo_stars.stars_mature',
-        'schedule': crontab(minute=30, hour=3, day_of_week='*/2'),
-        'args': [28]
-    }
-}
+CELERY_URL_BROKER = 'redis://gmw_redis:6379/0',
+CELERY_URL_BACKEND = 'redis://gmw_redis:6379/0'
 
 # oauth
 GITHUB_OAUTH = {

tests/conftest.py

@@ -0,0 +1 @@
+pytest_plugins = ["celery.contrib.pytest"]

tests/unit/services/test_celery.py

@@ -1,25 +1,26 @@
-from gitmostwanted.services.celery import instance
-from unittest import TestCase, mock
+from unittest import mock
+
+import pytest
+
 from gitmostwanted.web import app
 
 
-class ServicesCeleryTestCase(TestCase):
-    def test_use_flask_context(self):
-        context = mock.MagicMock()
+@pytest.fixture(scope='session')
+def celery_parameters():
+    return {'task_cls': 'lib.celery.flask_context_task:FlaskContextTask'}
+
 
-        app.app_context = lambda: context
-        app.config['CELERY_ALWAYS_EAGER'] = True
-        app.config['CELERY_IGNORE_RESULT'] = True
+def test_use_flask_context(celery_app, celery_worker):
+    context = mock.MagicMock()
 
-        cell = instance(app)
+    app.app_context = lambda: context
 
-        class FakeTask(cell.Task):
-            name = 'test'
+    @celery_app.task
+    def test():
+        return 1
 
-            def run(self): pass
+    celery_worker.reload()
 
-        task = FakeTask()
-        cell.tasks.register(task)
-        task.delay()
+    test.delay().get(timeout=10)
 
-        context.__enter__.assert_called_once_with()
+    context.__enter__.assert_called_once_with()