From e922117c6172cb44c9d643a00bb2dcdd6fdbd64d Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 15 Nov 2019 10:09:26 +0300 Subject: [PATCH 01/11] refactor subscriber model --- .../migrations/0002_subscriber_old_id.py | 18 +++++++++ apps/notification/models.py | 38 +++++++++++-------- apps/transfer/models.py | 1 + 3 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 apps/notification/migrations/0002_subscriber_old_id.py diff --git a/apps/notification/migrations/0002_subscriber_old_id.py b/apps/notification/migrations/0002_subscriber_old_id.py new file mode 100644 index 00000000..bf92831d --- /dev/null +++ b/apps/notification/migrations/0002_subscriber_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-11-15 07:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('notification', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='subscriber', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/notification/models.py b/apps/notification/models.py index 85176d24..3e6f7f3a 100644 --- a/apps/notification/models.py +++ b/apps/notification/models.py @@ -74,21 +74,29 @@ class Subscriber(ProjectBaseMixin): (USABLE, _('Usable')), ) - user = models.OneToOneField(User, blank=True, null=True, default=None, - on_delete=models.SET_NULL, related_name='subscriber', - verbose_name=_('User')) - email = models.EmailField(blank=True, null=True, default=None, unique=True, - verbose_name=_('Email')) - ip_address = models.GenericIPAddressField(blank=True, null=True, default=None, - verbose_name=_('IP address')) - country_code = models.CharField(max_length=10, blank=True, null=True, default=None, - verbose_name=_('Country code')) - locale = models.CharField(blank=True, null=True, default=None, - max_length=10, verbose_name=_('Locale identifier')) - state = models.PositiveIntegerField(choices=STATE_CHOICES, default=USABLE, - verbose_name=_('State')) - update_code = models.CharField(max_length=254, blank=True, null=True, default=None, - db_index=True, verbose_name=_('Token')) + user = models.OneToOneField( + User, + blank=True, + null=True, + default=None, + on_delete=models.SET_NULL, + related_name='subscriber', + verbose_name=_('User'), + ) + email = models.EmailField(blank=True, null=True, default=None, unique=True, verbose_name=_('Email')) + ip_address = models.GenericIPAddressField(blank=True, null=True, default=None, verbose_name=_('IP address')) + country_code = models.CharField(max_length=10, blank=True, null=True, default=None, verbose_name=_('Country code')) + locale = models.CharField(blank=True, null=True, default=None, max_length=10, verbose_name=_('Locale identifier')) + state = models.PositiveIntegerField(choices=STATE_CHOICES, default=USABLE, verbose_name=_('State')) + update_code = models.CharField( + max_length=254, + blank=True, + null=True, + default=None, + db_index=True, + verbose_name=_('Token'), + ) + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) objects = SubscriberManager.from_queryset(SubscriberQuerySet)() diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 43084804..25fdb527 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -1194,6 +1194,7 @@ class NewsletterSubscriber(MigrateMixin): site = models.ForeignKey(Sites, models.DO_NOTHING, blank=True, null=True) email_address = models.ForeignKey(EmailAddresses, models.DO_NOTHING, blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) + consent_purpose = models.CharField(max_length=255, blank=True, null=True) consent_at = models.DateTimeField() created_at = models.DateTimeField() updated_at = models.DateTimeField() From 2931381126532dfc904f59e4e683d7047dfec472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Fri, 15 Nov 2019 10:12:19 +0300 Subject: [PATCH 02/11] Fix update employee --- apps/establishment/management/commands/update_employee.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/establishment/management/commands/update_employee.py b/apps/establishment/management/commands/update_employee.py index 41831f0d..888c121e 100644 --- a/apps/establishment/management/commands/update_employee.py +++ b/apps/establishment/management/commands/update_employee.py @@ -1,9 +1,12 @@ from django.core.management.base import BaseCommand from django.db import connections from establishment.management.commands.add_position import namedtuplefetchall -from establishment.models import Employee +from establishment.models import Employee, EstablishmentEmployee +from django.utils import timezone +from django.db.models import Q from tqdm import tqdm + class Command(BaseCommand): help = 'Add employee from old db to new db.' @@ -30,4 +33,7 @@ class Command(BaseCommand): for e in tqdm(self.employees_sql()): empl = Employee.objects.filter(old_id=e.profile_id).update(name=e.name) + EstablishmentEmployee.objects.filter(from_date__isnull=True, old_id__isnull=True)\ + .update(from_date=timezone.now()-timezone.timedelta(days=1)) + self.stdout.write(self.style.WARNING(f'Update employee name objects.')) From 418ef2769485139a561b8b2c9298a634db577605 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 15 Nov 2019 10:21:28 +0300 Subject: [PATCH 03/11] transfer newsletter_subscriber --- apps/notification/transfer_data.py | 32 ++++++++++--- apps/transfer/management/commands/transfer.py | 1 + apps/transfer/serializers/notification.py | 45 +++++++++++++++++++ 3 files changed, 72 insertions(+), 6 deletions(-) diff --git a/apps/notification/transfer_data.py b/apps/notification/transfer_data.py index 49ae6887..dcc5b395 100644 --- a/apps/notification/transfer_data.py +++ b/apps/notification/transfer_data.py @@ -1,24 +1,44 @@ from pprint import pprint -from transfer.models import EmailAddresses -from transfer.serializers.notification import SubscriberSerializer +from account.models import User +from transfer.models import EmailAddresses, NewsletterSubscriber +from transfer.serializers.notification import SubscriberSerializer, NewsletterSubscriberSerializer def transfer_subscriber(): - queryset = EmailAddresses.objects.filter(state="usable") + queryset = EmailAddresses.objects.filter(state='usable') serialized_data = SubscriberSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + pprint(f'News serializer errors: {serialized_data.errors}') def transfer_newsletter_subscriber(): - pass + users = User.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + queryset = NewsletterSubscriber.objects.filter( + email_address__account_id__in=list(users) + ).values( + 'id', + 'email_address__email', + 'email_address__account_id', + 'email_address__ip', + 'email_address__country_code', + 'email_address__locale', + 'state', + 'created_at', + ) + + serialized_data = NewsletterSubscriberSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f'NewsletterSubscriber serializer errors: {serialized_data.errors}') data_types = { - "subscriber": [transfer_subscriber] + 'subscriber': [transfer_subscriber], + 'newsletter_subscriber': [transfer_newsletter_subscriber], } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 28e40c50..f30a8467 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -38,6 +38,7 @@ class Command(BaseCommand): 'assemblage', 'rating_count', 'product_review', + 'newsletter_subscriber', # подписчики на рассылку - переносить после переноса пользователей №1 ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py index c179dd2a..bd568239 100644 --- a/apps/transfer/serializers/notification.py +++ b/apps/transfer/serializers/notification.py @@ -33,3 +33,48 @@ class SubscriberSerializer(serializers.ModelSerializer): def get_country_code(self, obj): return obj["country_code"] + + +class NewsletterSubscriberSerializer(serializers.Serializer): + pass + # vintage = serializers.IntegerField() + # mark = serializers.FloatField(allow_null=True) + # establishment_id = serializers.IntegerField() + # created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + # aasm_state = serializers.CharField(allow_null=True) + # reviewer_id = serializers.IntegerField() + # id = serializers.IntegerField() + # + # def validate(self, data): + # data.update({ + # 'reviewer': self.get_reviewer(data), + # 'status': Review.READY if data['aasm_state'] == 'published' else Review.TO_INVESTIGATE, + # 'published_at': data.pop('created_at'), + # 'old_id': data.pop('id'), + # 'content_object': self.get_establishment(data), + # }) + # data.pop('reviewer_id') + # data.pop('establishment_id') + # data.pop('aasm_state') + # return data + # + # def create(self, validated_data): + # obj, _ = Review.objects.update_or_create( + # old_id=validated_data['old_id'], + # defaults=validated_data, + # ) + # return obj + # + # @staticmethod + # def get_reviewer(data): + # user = User.objects.filter(old_id=data['reviewer_id']).first() + # if not user: + # raise ValueError(f"User account not found with old_id {data['reviewer_id']}") + # return user + # + # @staticmethod + # def get_establishment(data): + # establishment = Establishment.objects.filter(old_id=data['establishment_id']).first() + # if not establishment: + # raise ValueError(f"Establishment not found with old_id {data['establishment_id']}: ") + # return establishment From 8951de7399f9662c630bf8afd5365ddeb17b95eb Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Fri, 15 Nov 2019 10:23:16 +0300 Subject: [PATCH 04/11] updated requirements, updated ES task --- apps/establishment/tasks.py | 2 +- requirements/base.txt | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/establishment/tasks.py b/apps/establishment/tasks.py index fdc10933..80d9cca8 100644 --- a/apps/establishment/tasks.py +++ b/apps/establishment/tasks.py @@ -27,5 +27,5 @@ def recalculate_price_levels_by_country(country_id): @periodic_task(run_every=crontab(minute=59)) def rebuild_establishment_indices(): - management.call_command(search_index.Command(), action='rebuild', models=[models.Establishment.__name__], + management.call_command(search_index.Command(), action='populate', models=[models.Establishment.__name__], force=True) diff --git a/requirements/base.txt b/requirements/base.txt index d7050a29..b4f25189 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -50,7 +50,5 @@ PyYAML==5.1.2 # temp solution redis==3.2.0 -amqp>=2.4.0 - -kombu==4.5.0 -celery==4.3.0rc2 +kombu==4.3.0 +celery==4.2.2 From b8df1643698809cc626d140e22aee081d93403ab Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Fri, 15 Nov 2019 10:33:33 +0300 Subject: [PATCH 05/11] update celery to 4.3.0 --- requirements/base.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index b4f25189..4489d755 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -50,5 +50,5 @@ PyYAML==5.1.2 # temp solution redis==3.2.0 -kombu==4.3.0 -celery==4.2.2 +kombu==4.6.6 +celery==4.3.0 From 1348a1def6ed767bc70eebb8fb1728788f6a2832 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 15 Nov 2019 10:36:23 +0300 Subject: [PATCH 06/11] notification serializer start --- apps/notification/transfer_data.py | 7 +------ apps/transfer/serializers/notification.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/apps/notification/transfer_data.py b/apps/notification/transfer_data.py index dcc5b395..cc5542b4 100644 --- a/apps/notification/transfer_data.py +++ b/apps/notification/transfer_data.py @@ -1,6 +1,5 @@ from pprint import pprint -from account.models import User from transfer.models import EmailAddresses, NewsletterSubscriber from transfer.serializers.notification import SubscriberSerializer, NewsletterSubscriberSerializer @@ -17,17 +16,13 @@ def transfer_subscriber(): def transfer_newsletter_subscriber(): - users = User.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) - queryset = NewsletterSubscriber.objects.filter( - email_address__account_id__in=list(users) - ).values( + queryset = NewsletterSubscriber.objects.all().values( 'id', 'email_address__email', 'email_address__account_id', 'email_address__ip', 'email_address__country_code', 'email_address__locale', - 'state', 'created_at', ) diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py index bd568239..dc685a0c 100644 --- a/apps/transfer/serializers/notification.py +++ b/apps/transfer/serializers/notification.py @@ -36,15 +36,14 @@ class SubscriberSerializer(serializers.ModelSerializer): class NewsletterSubscriberSerializer(serializers.Serializer): - pass - # vintage = serializers.IntegerField() - # mark = serializers.FloatField(allow_null=True) - # establishment_id = serializers.IntegerField() - # created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') - # aasm_state = serializers.CharField(allow_null=True) - # reviewer_id = serializers.IntegerField() - # id = serializers.IntegerField() - # + id = serializers.IntegerField() + email_address__email = serializers.CharField() + email_address__account_id = serializers.IntegerField(allow_null=True) + email_address__ip = serializers.CharField(allow_null=True) + email_address__country_code = serializers.CharField(allow_null=True) + email_address__locale = serializers.CharField(allow_null=True) + created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + # def validate(self, data): # data.update({ # 'reviewer': self.get_reviewer(data), From 0dbb2c64cd0949748aba3c47dfc905e2a4f3ae65 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 15 Nov 2019 10:46:21 +0300 Subject: [PATCH 07/11] fix new in_fav detail --- apps/news/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/news/views.py b/apps/news/views.py index 9c0a17e5..fbc0e455 100644 --- a/apps/news/views.py +++ b/apps/news/views.py @@ -44,10 +44,10 @@ class NewsDetailView(NewsMixinView, generics.RetrieveAPIView): lookup_field = 'slug' serializer_class = serializers.NewsDetailWebSerializer - queryset = models.News.objects.all() - def get_queryset(self): - return self.queryset + """Override get_queryset method.""" + qs = models.News.objects.all().annotate_in_favorites(self.request.user) + return qs class NewsTypeListView(generics.ListAPIView): From b590a762c1b619a72e0cb35838d33e1a3ed32320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Fri, 15 Nov 2019 10:48:04 +0300 Subject: [PATCH 08/11] Fix update --- apps/establishment/management/commands/update_employee.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/management/commands/update_employee.py b/apps/establishment/management/commands/update_employee.py index 888c121e..e6d00cc8 100644 --- a/apps/establishment/management/commands/update_employee.py +++ b/apps/establishment/management/commands/update_employee.py @@ -33,7 +33,7 @@ class Command(BaseCommand): for e in tqdm(self.employees_sql()): empl = Employee.objects.filter(old_id=e.profile_id).update(name=e.name) - EstablishmentEmployee.objects.filter(from_date__isnull=True, old_id__isnull=True)\ + EstablishmentEmployee.objects.filter(from_date__isnull=True, old_id__isnull=False)\ .update(from_date=timezone.now()-timezone.timedelta(days=1)) self.stdout.write(self.style.WARNING(f'Update employee name objects.')) From 4d302395a90f2c8b7447cdc695e29450371557b0 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 15 Nov 2019 10:53:28 +0300 Subject: [PATCH 09/11] fix product in_fav --- apps/product/views/common.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/apps/product/views/common.py b/apps/product/views/common.py index e8f3a966..daa46fd7 100644 --- a/apps/product/views/common.py +++ b/apps/product/views/common.py @@ -10,14 +10,15 @@ from comment.serializers import CommentRUDSerializer class ProductBaseView(generics.GenericAPIView): """Product base view""" - permission_classes = (permissions.AllowAny, ) + permission_classes = (permissions.AllowAny,) def get_queryset(self): """Override get_queryset method.""" return Product.objects.published() \ - .with_base_related() \ - .by_country_code(self.request.country_code) \ - .order_by('-created') + .with_base_related() \ + .annotate_in_favorites(self.request.user) \ + .by_country_code(self.request.country_code) \ + .order_by('-created') class ProductListView(ProductBaseView, generics.ListAPIView): @@ -67,9 +68,9 @@ class ProductCommentListView(generics.ListAPIView): """Override get_queryset method""" product = get_object_or_404(Product, slug=self.kwargs['slug']) return Comment.objects.by_content_type(app_label='product', - model='product')\ - .by_object_id(object_id=product.pk)\ - .order_by('-created') + model='product') \ + .by_object_id(object_id=product.pk) \ + .order_by('-created') class ProductCommentRUDView(generics.RetrieveUpdateDestroyAPIView): From ff006eb6fbccb7ca842fe47ed6db6aa3b81c5d4e Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Fri, 15 Nov 2019 11:56:16 +0300 Subject: [PATCH 10/11] Try this style of update tasks of the establishments --- apps/establishment/tasks.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/establishment/tasks.py b/apps/establishment/tasks.py index 80d9cca8..67572612 100644 --- a/apps/establishment/tasks.py +++ b/apps/establishment/tasks.py @@ -4,11 +4,14 @@ import logging from celery import shared_task from celery.schedules import crontab from celery.task import periodic_task + from django.core import management from django_elasticsearch_dsl.management.commands import search_index +from django_elasticsearch_dsl.registries import registry from establishment import models from location.models import Country +from search_indexes.documents.establishment import EstablishmentDocument logger = logging.getLogger(__name__) @@ -25,7 +28,18 @@ def recalculate_price_levels_by_country(country_id): establishment.recalculate_price_level(low_price=country.low_price, high_price=country.high_price) +# @periodic_task(run_every=crontab(minute=59)) +# def rebuild_establishment_indices(): +# management.call_command(search_index.Command(), action='populate', models=[models.Establishment.__name__], +# force=True) + + @periodic_task(run_every=crontab(minute=59)) -def rebuild_establishment_indices(): - management.call_command(search_index.Command(), action='populate', models=[models.Establishment.__name__], - force=True) +def update_establishment_indices(): + try: + doc = registry.get_documents([models.Establishment]).pop() + except KeyError: + pass + else: + qs = doc().get_indexing_queryset() + doc().update(qs) From 36b47a6cba7bca4606ae8a1bf04f540d6ba8cea6 Mon Sep 17 00:00:00 2001 From: Semyon Yekhmenin Date: Fri, 15 Nov 2019 09:13:28 +0000 Subject: [PATCH 11/11] Added establishment subtype to product view --- apps/establishment/serializers/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 0053a424..46a8619e 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -131,7 +131,8 @@ class EstablishmentSubTypeBaseSerializer(serializers.ModelSerializer): 'id', 'name', 'name_translated', - 'establishment_type' + 'establishment_type', + 'index_name', ] extra_kwargs = { 'name': {'write_only': True}, @@ -173,6 +174,7 @@ class EstablishmentShortSerializer(serializers.ModelSerializer): """Short serializer for establishment.""" city = CitySerializer(source='address.city', allow_null=True) establishment_type = EstablishmentTypeGeoSerializer() + establishment_subtypes = EstablishmentSubTypeBaseSerializer(many=True) class Meta: """Meta class.""" @@ -183,7 +185,8 @@ class EstablishmentShortSerializer(serializers.ModelSerializer): 'index_name', 'slug', 'city', - 'establishment_type' + 'establishment_type', + 'establishment_subtypes', ]