From 7f23f0e891455324d1832a44e0053f3ffdadb489 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 1/3] Establishment each hour reindexing --- apps/establishment/models.py | 5 +++++ apps/establishment/tasks.py | 12 ++++++++++-- apps/search_indexes/documents/establishment.py | 2 +- project/settings/base.py | 10 +++++++++- requirements/base.txt | 3 ++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 304ea2a6..4a6a95e4 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -120,6 +120,11 @@ class EstablishmentQuerySet(models.QuerySet): def with_type_related(self): return self.prefetch_related('establishment_subtypes') + def with_es_related(self): + """Return qs with related for ES indexing objects.""" + return self.select_related('address', 'establishment_type', 'address__city', 'address__city__country').\ + prefetch_related('tags', 'schedule') + def search(self, value, locale=None): """Search text in JSON fields.""" if locale is not None: diff --git a/apps/establishment/tasks.py b/apps/establishment/tasks.py index cf23a7e6..978f29ef 100644 --- a/apps/establishment/tasks.py +++ b/apps/establishment/tasks.py @@ -1,10 +1,13 @@ """Establishment app tasks.""" import logging + from celery import shared_task +from django.core import management +from django_elasticsearch_dsl.management.commands import search_index + from establishment import models from location.models import Country - logger = logging.getLogger(__name__) @@ -12,10 +15,15 @@ logger = logging.getLogger(__name__) def recalculate_price_levels_by_country(country_id): try: country = Country.objects.get(pk=country_id) - except Country.DoesNotExist as ex: + except Country.DoesNotExist as _: logger.error(f'ESTABLISHMENT. Country does not exist. ID {country_id}') else: qs = models.Establishment.objects.filter(address__city__country=country) for establishment in qs: establishment.recalculate_price_level(low_price=country.low_price, high_price=country.high_price) + +@shared_task +def rebuild_establishment_indices(): + management.call_command(search_index.Command(), action='rebuild', models=[models.Establishment.__name__], + force=True) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 5d858321..665fc85d 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -106,4 +106,4 @@ class EstablishmentDocument(Document): ) def get_queryset(self): - return super().get_queryset().published() + return super().get_queryset().published().with_es_related() diff --git a/project/settings/base.py b/project/settings/base.py index f9524f94..8a65c1bb 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -13,6 +13,7 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ import os import sys from datetime import timedelta +from celery.schedules import crontab # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -98,7 +99,8 @@ EXTERNAL_APPS = [ 'timezone_field', 'storages', 'sorl.thumbnail', - 'timezonefinder' + 'timezonefinder', + 'django_celery_beat', ] @@ -324,6 +326,12 @@ CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = TIME_ZONE +CELERY_BEAT_SCHEDULE = { + 'send-summary-every-hour': { + 'task': 'establishment.tasks.rebuild_establishment_indices', + 'schedule': crontab(hour=1), + }, +} # Django FCM (Firebase push notifications) FCM_DJANGO_SETTINGS = { diff --git a/requirements/base.txt b/requirements/base.txt index c95055de..19c29b7d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -44,4 +44,5 @@ sorl-thumbnail==12.5.0 # temp solution redis==3.2.0 amqp>=2.4.0 -celery==4.3.0rc2 \ No newline at end of file +celery==4.3.0rc2 +django-celery-beat==1.5.0 \ No newline at end of file From dd0b5951328fa09ea47ab1e221950b28b082110e Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 2/3] Review fixes --- apps/establishment/tasks.py | 4 +++- project/settings/base.py | 8 -------- requirements/base.txt | 3 +-- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/apps/establishment/tasks.py b/apps/establishment/tasks.py index 978f29ef..b7a39d2d 100644 --- a/apps/establishment/tasks.py +++ b/apps/establishment/tasks.py @@ -2,6 +2,8 @@ import logging from celery import shared_task +from celery.schedules import crontab +from celery.task import periodic_task from django.core import management from django_elasticsearch_dsl.management.commands import search_index @@ -23,7 +25,7 @@ def recalculate_price_levels_by_country(country_id): establishment.recalculate_price_level(low_price=country.low_price, high_price=country.high_price) -@shared_task +@periodic_task(run_every=crontab(minute=60)) def rebuild_establishment_indices(): management.call_command(search_index.Command(), action='rebuild', models=[models.Establishment.__name__], force=True) diff --git a/project/settings/base.py b/project/settings/base.py index 8a65c1bb..a8334839 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -13,7 +13,6 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ import os import sys from datetime import timedelta -from celery.schedules import crontab # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -100,7 +99,6 @@ EXTERNAL_APPS = [ 'storages', 'sorl.thumbnail', 'timezonefinder', - 'django_celery_beat', ] @@ -326,12 +324,6 @@ CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = TIME_ZONE -CELERY_BEAT_SCHEDULE = { - 'send-summary-every-hour': { - 'task': 'establishment.tasks.rebuild_establishment_indices', - 'schedule': crontab(hour=1), - }, -} # Django FCM (Firebase push notifications) FCM_DJANGO_SETTINGS = { diff --git a/requirements/base.txt b/requirements/base.txt index 19c29b7d..c95055de 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -44,5 +44,4 @@ sorl-thumbnail==12.5.0 # temp solution redis==3.2.0 amqp>=2.4.0 -celery==4.3.0rc2 -django-celery-beat==1.5.0 \ No newline at end of file +celery==4.3.0rc2 \ No newline at end of file From 4eaa600fc1baee60176e9a37937670420d331e1a Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 3/3] Fix all tests --- apps/establishment/tasks.py | 2 +- apps/utils/tests/tests_translated.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/establishment/tasks.py b/apps/establishment/tasks.py index b7a39d2d..fdc10933 100644 --- a/apps/establishment/tasks.py +++ b/apps/establishment/tasks.py @@ -25,7 +25,7 @@ def recalculate_price_levels_by_country(country_id): establishment.recalculate_price_level(low_price=country.low_price, high_price=country.high_price) -@periodic_task(run_every=crontab(minute=60)) +@periodic_task(run_every=crontab(minute=59)) def rebuild_establishment_indices(): management.call_command(search_index.Command(), action='rebuild', models=[models.Establishment.__name__], force=True) diff --git a/apps/utils/tests/tests_translated.py b/apps/utils/tests/tests_translated.py index 4569ecf2..557c8b5d 100644 --- a/apps/utils/tests/tests_translated.py +++ b/apps/utils/tests/tests_translated.py @@ -9,6 +9,7 @@ from account.models import User from news.models import News, NewsType from establishment.models import Establishment, EstablishmentType, Employee +from location.models import Country class BaseTestCase(APITestCase): @@ -39,7 +40,13 @@ class TranslateFieldTests(BaseTestCase): self.news_type = NewsType.objects.create(name="Test news type") self.news_type.save() + + self.country_ru = Country.objects.get( + name={"en-GB": "Russian"} + ) + self.news_item = News.objects.create( + id=8, created_by=self.user, modified_by=self.user, title={ @@ -52,6 +59,7 @@ class TranslateFieldTests(BaseTestCase): news_type=self.news_type, slug='test', state=News.PUBLISHED, + country=self.country_ru, ) self.news_item.save()