From c725da6a9b09274a63a00d1e23682ae01b296b9b Mon Sep 17 00:00:00 2001 From: "a.feteleu" Date: Sat, 16 Nov 2019 01:08:31 +0300 Subject: [PATCH 01/22] refactored transfer ads --- apps/transfer/serializers/advertisement.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/transfer/serializers/advertisement.py b/apps/transfer/serializers/advertisement.py index c7bd50f8..761a1879 100644 --- a/apps/transfer/serializers/advertisement.py +++ b/apps/transfer/serializers/advertisement.py @@ -14,8 +14,6 @@ class AdvertisementSerializer(serializers.ModelSerializer): href = serializers.CharField() site_id = serializers.PrimaryKeyRelatedField( queryset=Sites.objects.all()) - start_at = serializers.DateTimeField(allow_null=True) - expire_at = serializers.DateTimeField(allow_null=True) class Meta: """Meta class.""" @@ -24,29 +22,27 @@ class AdvertisementSerializer(serializers.ModelSerializer): 'id', 'href', 'site_id', - 'start_at', - 'expire_at', ] def validate(self, data): data.update({ 'old_id': data.pop('id'), 'url': data.pop('href'), - 'site': self.get_site(data.pop('site_id')), - 'start': data.pop('start_at', None), - 'end': data.pop('expire_at', None), + 'site_settings': self.get_site_settings(data.pop('site_id')), }) return data def create(self, validated_data): - site = validated_data.pop('site') - obj, _ = self.Meta.model.objects.get_or_create(validated_data) + site = validated_data.pop('site_settings') + url = validated_data.get('url') + + obj, _ = self.Meta.model.objects.get_or_create(url=url, defaults=validated_data) if site and site not in obj.sites.all(): obj.sites.add(site) return obj - def get_site(self, subdomain): + def get_site_settings(self, subdomain): subdomain = subdomain.country_code_2 if isinstance(subdomain, Sites) else subdomain qs = SiteSettings.objects.filter(subdomain=subdomain) if qs.exists(): @@ -89,5 +85,5 @@ class AdvertisementImageSerializer(AdvertisementSerializer): image_url = validated_data.get('image_url') if advertisement and image_url: - Page.objects.get_or_create(advertisement=advertisement, image_url=image_url, source=Page.MOBILE) - Page.objects.get_or_create(advertisement=advertisement, image_url=image_url, source=Page.WEB) \ No newline at end of file + self.Meta.model.objects.get_or_create(source=Page.MOBILE, **validated_data) + self.Meta.model.objects.get_or_create(source=Page.WEB, **validated_data) From 8b656b9bb63c19f9d31b3d52eb316070df377f7f Mon Sep 17 00:00:00 2001 From: Semyon Yekhmenin Date: Sat, 16 Nov 2019 09:38:33 +0000 Subject: [PATCH 02/22] Fixed problem with no required period in guest online service response --- apps/booking/views.py | 48 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/apps/booking/views.py b/apps/booking/views.py index 653503e7..cdadc768 100644 --- a/apps/booking/views.py +++ b/apps/booking/views.py @@ -13,10 +13,43 @@ from utils.methods import get_user_ip class CheckWhetherBookingAvailable(generics.GenericAPIView): """ Checks which service to use if establishmend is managed by any """ + _VALID_GUESTONLINE_PERIODS = {'lunch', 'dinner', 'afternoon', 'breakfast'} + permission_classes = (permissions.AllowAny,) serializer_class = CheckBookingSerializer pagination_class = None + def _fill_period_template(self, period_template, period_name): + period_template_copy = period_template.copy() + period_template_copy['period'] = period_name + return period_template_copy + + def _preprocess_guestonline_response(self, response): + periods = response['periods'] + periods_by_name = {period['period']: period for period in periods if 'period' in period} + if not periods_by_name: + raise ValueError('Empty guestonline response') + + period_template = iter(periods_by_name.values()).__next__().copy() + period_template.pop('total_left_seats') + period_template.pop('hours') + period_template.pop('period') + + processed_periods = [ + periods_by_name[period_name] + if period_name in periods_by_name + else self._fill_period_template(period_template, period_name) + for period_name in CheckWhetherBookingAvailable._VALID_GUESTONLINE_PERIODS + ] + + unnamed_periods = filter(lambda period: 'period' not in period, periods) + for unnamed_period in unnamed_periods: + processed_periods.append(unnamed_period) + + response['periods'] = processed_periods + + return response + def get(self, request, *args, **kwargs): is_booking_available = False establishment = get_object_or_404(Establishment, pk=kwargs['establishment_id']) @@ -24,12 +57,12 @@ class CheckWhetherBookingAvailable(generics.GenericAPIView): date = request.query_params.get('date') g_service = GuestonlineService() l_service = LastableService() - if (not establishment.lastable_id is None) and l_service \ + if establishment.lastable_id is not None and l_service \ .check_whether_booking_available(establishment.lastable_id, date): is_booking_available = True service = l_service service.service_id = establishment.lastable_id - elif (not establishment.guestonline_id is None) and g_service \ + elif establishment.guestonline_id is not None and g_service \ .check_whether_booking_available(establishment.guestonline_id, **g_service.get_certain_keys(request.query_params, {'date', 'persons'})): @@ -41,7 +74,11 @@ class CheckWhetherBookingAvailable(generics.GenericAPIView): 'available': is_booking_available, 'type': service.service if service else None, } - response.update({'details': service.response} if service and service.response else {}) + + service_response = self._preprocess_guestonline_response(service.response) \ + if establishment.guestonline_id is not None \ + else service.response + response.update({'details': service_response} if service and service.response else {}) return Response(data=response, status=200) @@ -97,8 +134,9 @@ class UpdatePendingBooking(generics.UpdateAPIView): r = service.update_booking(service.get_certain_keys(data, { 'email', 'phone', 'last_name', 'first_name', 'country_code', 'pending_booking_id', 'note', }, { - 'email', 'phone', 'last_name', 'first_name', 'country_code', 'pending_booking_id', - })) + 'email', 'phone', 'last_name', 'first_name', + 'country_code', 'pending_booking_id', + })) if isinstance(r, Response): return r if data.get('newsletter'): From 36d16625ea177db1d64184b20222acb3745e4a2e Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 13:28:03 +0300 Subject: [PATCH 03/22] change filter for Establishment --- apps/tag/management/commands/add_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index 37fdbbea..0fba46f8 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -11,7 +11,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): existing_establishment = Establishment.objects.filter( - old_id__isnull=False, tags__isnull=True + old_id__isnull=False ) ESTABLISHMENT = 1 SHOP = 2 From ca61359a81287940efe21c3569898d6e99a04412 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 04/22] Fix product type serializer --- apps/search_indexes/serializers.py | 18 +++++++++++++----- apps/search_indexes/views.py | 3 +++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index dffd6e77..f926ff55 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -79,6 +79,18 @@ class WineColorDocumentSerializer(serializers.Serializer): return get_translated_value(obj.label) +class ProductTypeDocumentSerializer(serializers.Serializer): + """Product type ES document serializer.""" + + id = serializers.IntegerField() + index_name = serializers.CharField() + name_translated = serializers.SerializerMethodField() + + @staticmethod + def get_name_translated(obj): + return get_translated_value(obj.name) + + class ProductEstablishmentDocumentSerializer(serializers.Serializer): """Related to Product Establishment ES document serializer.""" @@ -202,13 +214,9 @@ class ProductDocumentSerializer(DocumentSerializer): subtypes = ProductSubtypeDocumentSerializer(many=True) wine_region = WineRegionDocumentSerializer(allow_null=True) wine_colors = WineColorDocumentSerializer(many=True) - product_type = serializers.SerializerMethodField() + product_type = ProductTypeDocumentSerializer(allow_null=True) establishment_detail = ProductEstablishmentDocumentSerializer(source='establishment', allow_null=True) - @staticmethod - def get_product_type(obj): - return get_translated_value(obj.product_type.name if obj.product_type else {}) - class Meta: """Meta class.""" diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index 1a898cbc..a40b13ed 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -251,6 +251,9 @@ class ProductDocumentViewSet(BaseDocumentViewSet): 'type': { 'field': 'product_type.index_name', }, + 'product_type': { + 'field': 'product_type.index_name', + }, 'subtype': { 'field': 'subtypes.index_name', 'lookups': [ From bff533e7d5c741464518ede884c6e4f3e0f57277 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 05/22] Tag types --- apps/search_indexes/views.py | 20 ++++++-------------- apps/tag/filters.py | 7 ++++++- apps/tag/models.py | 4 ++++ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index a40b13ed..85bd7c01 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -89,10 +89,6 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): search_fields = { 'name': {'fuzziness': 'auto:2,5', 'boost': 4}, - 'transliterated_name': {'fuzziness': 'auto:2,5', - 'boost': 3}, - 'index_name': {'fuzziness': 'auto:2,5', - 'boost': 2}, 'description': {'fuzziness': 'auto:2,5'}, } translated_search_fields = ( @@ -199,7 +195,7 @@ class ProductDocumentViewSet(BaseDocumentViewSet): """Product document ViewSet.""" document = ProductDocument - lookup_field = 'slug' + # lookup_field = 'slug' pagination_class = ProjectMobilePagination permission_classes = (permissions.AllowAny,) serializer_class = serializers.ProductDocumentSerializer @@ -212,8 +208,8 @@ class ProductDocumentViewSet(BaseDocumentViewSet): filter_backends = [ FilteringFilterBackend, filters.CustomSearchFilterBackend, - GeoSpatialFilteringFilterBackend, - DefaultOrderingFilterBackend, + # GeoSpatialFilteringFilterBackend, +# DefaultOrderingFilterBackend, ] search_fields = { @@ -221,10 +217,9 @@ class ProductDocumentViewSet(BaseDocumentViewSet): 'boost': 4}, 'transliterated_name': {'fuzziness': 'auto:2,5', 'boost': 3}, - 'index_name': {'fuzziness': 'auto:2,5', - 'boost': 2}, 'description': {'fuzziness': 'auto:2,5'}, } + translated_search_fields = ( 'description', ) @@ -248,9 +243,6 @@ class ProductDocumentViewSet(BaseDocumentViewSet): 'for_establishment': { 'field': 'establishment.slug', }, - 'type': { - 'field': 'product_type.index_name', - }, 'product_type': { 'field': 'product_type.index_name', }, @@ -262,5 +254,5 @@ class ProductDocumentViewSet(BaseDocumentViewSet): ] } } - geo_spatial_filter_fields = { - } \ No newline at end of file + # geo_spatial_filter_fields = { + # } \ No newline at end of file diff --git a/apps/tag/filters.py b/apps/tag/filters.py index 0b1fb829..08d14e94 100644 --- a/apps/tag/filters.py +++ b/apps/tag/filters.py @@ -31,13 +31,18 @@ class TagCategoryFilterSet(TagsBaseFilterSet): """TagCategory filterset.""" establishment_type = filters.CharFilter(method='by_establishment_type') + product_type = filters.CharFilter(method='by_product_type') class Meta: """Meta class.""" model = models.TagCategory fields = ('type', - 'establishment_type', ) + 'establishment_type', + 'product_type', ) + + def by_product_type(self, queryset, name, value): + return queryset.by_product_type(value) # todo: filter by establishment type def by_establishment_type(self, queryset, name, value): diff --git a/apps/tag/models.py b/apps/tag/models.py index a35425e8..2505b9ca 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -100,6 +100,10 @@ class TagCategoryQuerySet(models.QuerySet): """Filter by establishment type index name.""" return self.filter(establishment_types__index_name=index_name) + def by_product_type(self, index_name): + """Filter by product type index name.""" + return self.filter(product_types__index_name=index_name) + def with_tags(self, switcher=True): """Filter by existing tags.""" return self.exclude(tags__isnull=switcher) From 5a014ca7fe87e0c4027fb1040744f7e9c2a55434 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 06/22] Fix some product search issues --- apps/search_indexes/serializers.py | 8 ++++++-- apps/search_indexes/views.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index f926ff55..a26b80fc 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -39,7 +39,8 @@ class ProductSubtypeDocumentSerializer(serializers.Serializer): id = serializers.IntegerField() name_translated = serializers.SerializerMethodField() - get_name_translated = lambda obj: get_translated_value(obj.name) + def get_name_translated(self, obj): + return get_translated_value(obj.name) class WineRegionCountryDocumentSerialzer(serializers.Serializer): @@ -64,6 +65,9 @@ class WineRegionDocumentSerializer(serializers.Serializer): name = serializers.CharField() country = WineRegionCountryDocumentSerialzer(allow_null=True) + def get_attribute(self, instance): + return instance.wine_region if instance and instance.wine_region else None + class WineColorDocumentSerializer(serializers.Serializer): """Wine color ES document serializer,""" @@ -211,7 +215,7 @@ class ProductDocumentSerializer(DocumentSerializer): """Product document serializer""" tags = TagsDocumentSerializer(many=True) - subtypes = ProductSubtypeDocumentSerializer(many=True) + subtypes = ProductSubtypeDocumentSerializer(many=True, allow_null=True) wine_region = WineRegionDocumentSerializer(allow_null=True) wine_colors = WineColorDocumentSerializer(many=True) product_type = ProductTypeDocumentSerializer(allow_null=True) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index 85bd7c01..dcc6f895 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -89,6 +89,8 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): search_fields = { 'name': {'fuzziness': 'auto:2,5', 'boost': 4}, + 'transliterated_name': {'fuzziness': 'auto:2,5', + 'boost': 3}, 'description': {'fuzziness': 'auto:2,5'}, } translated_search_fields = ( From 94797c354ac0f40d514f44d8a24786586756a228 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 07/22] Fix search results produt name --- apps/search_indexes/documents/product.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/documents/product.py b/apps/search_indexes/documents/product.py index b00f3f50..a1179a4f 100644 --- a/apps/search_indexes/documents/product.py +++ b/apps/search_indexes/documents/product.py @@ -95,13 +95,13 @@ class ProductDocument(Document): }, multi=True ) + name = fields.KeywordField(attr='display_name') class Django: model = models.Product fields = ( 'id', 'category', - 'name', 'available', 'public_mark', 'slug', From 144b5abdbffd2da18265c85ee4796964e4cfe9d1 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 08/22] Fix issue w/ product display name --- apps/product/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/product/models.py b/apps/product/models.py index ba9ccd38..53cd0e61 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -278,7 +278,7 @@ class Product(TranslatedFieldsMixin, BaseAttributes): def display_name(self): name = f'{self.name} ' \ f'({self.vintage if self.vintage else "BSA"})' - if self.establishment.name: + if self.establishment and self.establishment.name: name = f'{self.establishment.name} - ' + name return name From 4e005a7a4d589c21a1bc5f119bc5ab7e1e51cf8a Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 16 Nov 2019 15:56:23 +0300 Subject: [PATCH 09/22] migrate subscribers --- .../migrations/0008_auto_20191116_1135.py | 17 ++++++++ .../migrations/0003_auto_20191116_1248.py | 20 ++++++++++ apps/notification/models.py | 2 +- apps/notification/transfer_data.py | 14 +++---- apps/transfer/serializers/notification.py | 39 ++++++++++++------- 5 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 apps/advertisement/migrations/0008_auto_20191116_1135.py create mode 100644 apps/notification/migrations/0003_auto_20191116_1248.py diff --git a/apps/advertisement/migrations/0008_auto_20191116_1135.py b/apps/advertisement/migrations/0008_auto_20191116_1135.py new file mode 100644 index 00000000..c2a0278c --- /dev/null +++ b/apps/advertisement/migrations/0008_auto_20191116_1135.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.7 on 2019-11-16 11:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('advertisement', '0007_auto_20191115_0750'), + ] + + operations = [ + migrations.AlterModelOptions( + name='advertisement', + options={'verbose_name': 'Advertisement', 'verbose_name_plural': 'Advertisements'}, + ), + ] diff --git a/apps/notification/migrations/0003_auto_20191116_1248.py b/apps/notification/migrations/0003_auto_20191116_1248.py new file mode 100644 index 00000000..2af6a7ae --- /dev/null +++ b/apps/notification/migrations/0003_auto_20191116_1248.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.7 on 2019-11-16 12:48 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('notification', '0002_subscriber_old_id'), + ] + + operations = [ + migrations.AlterField( + model_name='subscriber', + name='user', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='subscriber', to=settings.AUTH_USER_MODEL, verbose_name='User'), + ), + ] diff --git a/apps/notification/models.py b/apps/notification/models.py index 3e6f7f3a..a5cde50b 100644 --- a/apps/notification/models.py +++ b/apps/notification/models.py @@ -74,7 +74,7 @@ class Subscriber(ProjectBaseMixin): (USABLE, _('Usable')), ) - user = models.OneToOneField( + user = models.ForeignKey( User, blank=True, null=True, diff --git a/apps/notification/transfer_data.py b/apps/notification/transfer_data.py index 487501c3..1712b6ea 100644 --- a/apps/notification/transfer_data.py +++ b/apps/notification/transfer_data.py @@ -1,7 +1,5 @@ from pprint import pprint -from django.db.models import Count - from transfer.models import EmailAddresses, NewsletterSubscriber from transfer.serializers.notification import SubscriberSerializer, NewsletterSubscriberSerializer @@ -25,14 +23,14 @@ def transfer_newsletter_subscriber(): 'email_address__ip', 'email_address__country_code', 'email_address__locale', - 'created_at', + 'updated_at', ) - # serialized_data = NewsletterSubscriberSerializer(data=list(queryset.values()), many=True) - # if serialized_data.is_valid(): - # serialized_data.save() - # else: - # pprint(f'NewsletterSubscriber serializer errors: {serialized_data.errors}') + serialized_data = NewsletterSubscriberSerializer(data=list(queryset), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f'NewsletterSubscriber serializer errors: {serialized_data.errors}') data_types = { diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py index 7eb7bdac..dc0ca4f3 100644 --- a/apps/transfer/serializers/notification.py +++ b/apps/transfer/serializers/notification.py @@ -1,3 +1,4 @@ +from django.db import IntegrityError from rest_framework import serializers from account.models import User @@ -41,10 +42,10 @@ class NewsletterSubscriberSerializer(serializers.Serializer): id = serializers.IntegerField() email_address__email = serializers.CharField() email_address__account_id = serializers.IntegerField(allow_null=True) - email_address__ip = serializers.CharField(allow_null=True) - email_address__country_code = serializers.CharField(allow_null=True) - email_address__locale = serializers.CharField(allow_null=True) - created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + email_address__ip = serializers.CharField(allow_null=True, allow_blank=True) + email_address__country_code = serializers.CharField(allow_null=True, allow_blank=True) + email_address__locale = serializers.CharField(allow_null=True, allow_blank=True) + updated_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') def validate(self, data): data.update({ @@ -53,18 +54,28 @@ class NewsletterSubscriberSerializer(serializers.Serializer): 'ip_address': data.pop('email_address__ip'), 'country_code': data.pop('email_address__country_code'), 'locale': data.pop('email_address__locale'), - 'created': data.pop('created_at'), + 'created': data.pop('updated_at'), 'user_id': self.get_user(data), }) data.pop('email_address__account_id') return data - # def create(self, validated_data): - # obj, _ = Review.objects.update_or_create( - # old_id=validated_data['old_id'], - # defaults=validated_data, - # ) - # return obj + def create(self, validated_data): + try: + obj = Subscriber.objects.get(email=validated_data['email']) + except Subscriber.DoesNotExist: + obj = Subscriber.objects.create(**validated_data) + else: + current_data = obj.created + if validated_data['created'] > current_data: + obj.ip_address = validated_data['ip_address'] + obj.locale = validated_data['locale'] + obj.country_code = validated_data['country_code'] + obj.old_id = validated_data['old_id'] + obj.created = validated_data['created'] + obj.user_id = validated_data['user_id'] + obj.save() + return obj @staticmethod def get_user(data): @@ -73,6 +84,6 @@ class NewsletterSubscriberSerializer(serializers.Serializer): return None user = User.objects.filter(old_id=data['email_address__account_id']).first() - if not user: - raise ValueError(f"User account not found with old_id {data['email_address__account_id']}") - return user.id + if user: + return user.id + return None From 2e463feb8dbd8f314d8347e44e4f3805425cddc2 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 10/22] Sort countries by name accordingly to locale --- apps/location/views/common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/location/views/common.py b/apps/location/views/common.py index a1e6f011..99cd4c92 100644 --- a/apps/location/views/common.py +++ b/apps/location/views/common.py @@ -1,8 +1,9 @@ """Location app views.""" from rest_framework import generics from rest_framework import permissions - +from django.db.models.expressions import RawSQL from location import models, serializers +from utils.models import get_current_locale # Mixins @@ -37,7 +38,9 @@ class CountryListView(CountryViewMixin, generics.ListAPIView): """List view for model Country.""" pagination_class = None - + def get_queryset(self): + qs = super().get_queryset().order_by(RawSQL("name->>%s", (get_current_locale(),))) + return qs class CountryRetrieveView(CountryViewMixin, generics.RetrieveAPIView): """Retrieve view for model Country.""" From 78e42550c1c635d6e4e91781f454da031db92d7b Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 16 Nov 2019 16:57:26 +0300 Subject: [PATCH 11/22] add address to establishment similar and collection establishment --- apps/collection/views/common.py | 8 ++++---- apps/establishment/serializers/common.py | 11 +++++++++-- apps/establishment/views/web.py | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/apps/collection/views/common.py b/apps/collection/views/common.py index 5bf8f70e..3d06d861 100644 --- a/apps/collection/views/common.py +++ b/apps/collection/views/common.py @@ -1,11 +1,11 @@ +from django.shortcuts import get_object_or_404 from rest_framework import generics from rest_framework import permissions from collection import models -from utils.pagination import ProjectPageNumberPagination -from django.shortcuts import get_object_or_404 -from establishment.serializers import EstablishmentBaseSerializer from collection.serializers import common as serializers +from establishment.serializers import EstablishmentSimilarSerializer +from utils.pagination import ProjectPageNumberPagination # Mixins @@ -53,7 +53,7 @@ class CollectionEstablishmentListView(CollectionListView): """Retrieve list of establishment for collection.""" lookup_field = 'slug' pagination_class = ProjectPageNumberPagination - serializer_class = EstablishmentBaseSerializer + serializer_class = EstablishmentSimilarSerializer def get_queryset(self): """ diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 1d405be7..7051300f 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -17,6 +17,7 @@ from review.serializers import ReviewShortSerializer class ContactPhonesSerializer(serializers.ModelSerializer): """Contact phone serializer""" + class Meta: model = models.ContactPhone fields = [ @@ -26,6 +27,7 @@ class ContactPhonesSerializer(serializers.ModelSerializer): class ContactEmailsSerializer(serializers.ModelSerializer): """Contact email serializer""" + class Meta: model = models.ContactEmail fields = [ @@ -35,6 +37,7 @@ class ContactEmailsSerializer(serializers.ModelSerializer): class SocialNetworkRelatedSerializers(serializers.ModelSerializer): """Social network serializers.""" + class Meta: model = models.SocialNetwork fields = [ @@ -45,7 +48,6 @@ class SocialNetworkRelatedSerializers(serializers.ModelSerializer): class PlateSerializer(ProjectModelSerializer): - name_translated = TranslatedField() currency = CurrencySerializer(read_only=True) @@ -316,6 +318,12 @@ class EstablishmentDetailSerializer(EstablishmentBaseSerializer): ] +class EstablishmentSimilarSerializer(EstablishmentBaseSerializer): + """Serializer for Establishment model.""" + + address = AddressDetailSerializer(read_only=True) + + class EstablishmentCommentCreateSerializer(comment_serializers.CommentSerializer): """Create comment serializer""" mark = serializers.IntegerField() @@ -379,4 +387,3 @@ class EstablishmentFavoritesCreateSerializer(FavoritesCreateSerializer): 'content_object': validated_data.pop('establishment') }) return super().create(validated_data) - diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 2a6fd215..20e8f81a 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -72,7 +72,7 @@ class EstablishmentRecentReviewListView(EstablishmentListView): class EstablishmentSimilarListView(EstablishmentListView): """Resource for getting a list of establishments.""" - serializer_class = serializers.EstablishmentBaseSerializer + serializer_class = serializers.EstablishmentSimilarSerializer pagination_class = EstablishmentPortionPagination def get_queryset(self): From cc67499601dfd61d701b33ce2513d80b78cd7be8 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 12/22] Fix wines db queries --- apps/establishment/serializers/common.py | 24 +++++++++++++++++++++++- apps/location/serializers/common.py | 14 ++++++++++++++ apps/product/models.py | 7 ++++++- apps/product/serializers/common.py | 11 ++++++----- apps/product/views/common.py | 4 ++++ apps/tag/serializers.py | 15 +++++++++++++++ 6 files changed, 68 insertions(+), 7 deletions(-) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 7051300f..858a203a 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -5,7 +5,7 @@ from rest_framework import serializers from comment import models as comment_models from comment.serializers import common as comment_serializers from establishment import models -from location.serializers import AddressBaseSerializer, CitySerializer, AddressDetailSerializer +from location.serializers import AddressBaseSerializer, CitySerializer, AddressDetailSerializer, CityShortSerializer from main.serializers import AwardSerializer, CurrencySerializer from tag.serializers import TagBaseSerializer from timetable.serialziers import ScheduleRUDSerializer @@ -193,6 +193,28 @@ class EstablishmentShortSerializer(serializers.ModelSerializer): ] +class EstablishmentProductShortSerializer(serializers.ModelSerializer): + """SHORT Serializer for displaying info about an establishment on product page.""" + establishment_type = EstablishmentTypeGeoSerializer() + establishment_subtypes = EstablishmentSubTypeBaseSerializer(many=True) + address = AddressBaseSerializer() + city = CityShortSerializer(source='address.city', allow_null=True) + + class Meta: + """Meta class.""" + model = models.Establishment + fields = [ + 'id', + 'name', + 'index_name', + 'slug', + 'city', + 'establishment_type', + 'establishment_subtypes', + 'address', + ] + + class EstablishmentProductSerializer(EstablishmentShortSerializer): """Serializer for displaying info about an establishment on product page.""" diff --git a/apps/location/serializers/common.py b/apps/location/serializers/common.py index fe27892c..6f03eaed 100644 --- a/apps/location/serializers/common.py +++ b/apps/location/serializers/common.py @@ -54,6 +54,20 @@ class RegionSerializer(serializers.ModelSerializer): 'country_id' ] +class CityShortSerializer(serializers.ModelSerializer): + """Short city serializer""" + country = CountrySerializer(read_only=True) + + class Meta: + """Meta class""" + model = models.City + fields = ( + 'id', + 'name', + 'code', + 'country', + ) + class CitySerializer(serializers.ModelSerializer): """City serializer.""" diff --git a/apps/product/models.py b/apps/product/models.py index 53cd0e61..331cfd72 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -82,7 +82,12 @@ class ProductQuerySet(models.QuerySet): def with_extended_related(self): """Returns qs with almost all related objects.""" return self.with_base_related() \ - .prefetch_related('tags', 'standards', 'classifications', 'classifications__standard', + .prefetch_related('tags', 'tags__category', 'tags__category__country', + 'standards', 'classifications', 'classifications__standard', + 'establishment__address', 'establishment__establishment_type', + 'establishment__address__city', 'establishment__address__city__country', + 'establishment__establishment_subtypes', 'product_gallery', + 'gallery', 'product_type', 'subtypes', 'classifications__classification_type', 'classifications__tags') \ .select_related('wine_region', 'wine_sub_region') diff --git a/apps/product/serializers/common.py b/apps/product/serializers/common.py index da2a2344..0195ee5f 100644 --- a/apps/product/serializers/common.py +++ b/apps/product/serializers/common.py @@ -4,7 +4,7 @@ from rest_framework import serializers from comment.models import Comment from comment.serializers import CommentSerializer -from establishment.serializers import EstablishmentShortSerializer, EstablishmentProductSerializer +from establishment.serializers import EstablishmentShortSerializer, EstablishmentProductSerializer, EstablishmentProductShortSerializer from gallery.models import Image from product import models from review.serializers import ReviewShortSerializer @@ -12,13 +12,13 @@ from utils import exceptions as utils_exceptions from utils.serializers import TranslatedField, FavoritesCreateSerializer from main.serializers import AwardSerializer from location.serializers import WineRegionBaseSerializer, WineSubRegionBaseSerializer -from tag.serializers import TagBaseSerializer, TagCategoryShortSerializer +from tag.serializers import TagBaseSerializer, TagCategoryProductSerializer class ProductTagSerializer(TagBaseSerializer): """Serializer for model Tag.""" - category = TagCategoryShortSerializer(read_only=True) + category = TagCategoryProductSerializer(read_only=True) class Meta(TagBaseSerializer.Meta): """Meta class.""" @@ -88,10 +88,10 @@ class ProductBaseSerializer(serializers.ModelSerializer): name = serializers.CharField(source='display_name', read_only=True) product_type = ProductTypeBaseSerializer(read_only=True) subtypes = ProductSubTypeBaseSerializer(many=True, read_only=True) - establishment_detail = EstablishmentShortSerializer(source='establishment', read_only=True) + establishment_detail = EstablishmentProductShortSerializer(source='establishment', read_only=True) tags = ProductTagSerializer(source='related_tags', many=True, read_only=True) wine_region = WineRegionBaseSerializer(read_only=True) - wine_colors = TagBaseSerializer(many=True, read_only=True) + wine_colors = ProductTagSerializer(many=True, read_only=True) preview_image_url = serializers.URLField(source='preview_main_image_url', allow_null=True, read_only=True) @@ -120,6 +120,7 @@ class ProductBaseSerializer(serializers.ModelSerializer): class ProductDetailSerializer(ProductBaseSerializer): """Product detail serializer.""" description_translated = TranslatedField() + establishment_detail = EstablishmentShortSerializer(source='establishment', read_only=True) review = ReviewShortSerializer(source='last_published_review', read_only=True) awards = AwardSerializer(many=True, read_only=True) classifications = ProductClassificationBaseSerializer(many=True, read_only=True) diff --git a/apps/product/views/common.py b/apps/product/views/common.py index daa46fd7..911d1f0b 100644 --- a/apps/product/views/common.py +++ b/apps/product/views/common.py @@ -26,6 +26,10 @@ class ProductListView(ProductBaseView, generics.ListAPIView): serializer_class = serializers.ProductBaseSerializer filter_class = filters.ProductFilterSet + def get_queryset(self): + qs = super().get_queryset().with_extended_related() + return qs + class ProductDetailView(ProductBaseView, generics.RetrieveAPIView): """Detail view fro model Product.""" diff --git a/apps/tag/serializers.py b/apps/tag/serializers.py index 02f2e4d3..2cc818a9 100644 --- a/apps/tag/serializers.py +++ b/apps/tag/serializers.py @@ -37,6 +37,21 @@ class TagBackOfficeSerializer(TagBaseSerializer): 'category' ) +class TagCategoryProductSerializer(serializers.ModelSerializer): + """SHORT Serializer for TagCategory""" + + label_translated = TranslatedField() + + class Meta: + """Meta class.""" + + model = models.TagCategory + fields = ( + 'id', + 'label_translated', + 'index_name', + ) + class TagCategoryBaseSerializer(serializers.ModelSerializer): """Serializer for model TagCategory.""" From 8a86018a5e3a963ca8a52089c2dd4da55d06aaf7 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 13/22] Fix wines db queries #2 --- apps/product/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/product/models.py b/apps/product/models.py index 331cfd72..44c57057 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -275,9 +275,8 @@ class Product(TranslatedFieldsMixin, BaseAttributes): @property def related_tags(self): - return self.tags.exclude( - category__index_name__in=['sugar-content', 'wine-color', 'bottles-produced', - 'serial-number', 'grape-variety']) + return self.tags.exclude(category__index_name__in=['sugar-content', 'wine-color', 'bottles-produced', + 'serial-number', 'grape-variety']).prefetch_related('category') @property def display_name(self): From be0d63cf079448cef363220b7c3f03c8456d09cd Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 14/22] Fix products search --- apps/search_indexes/documents/product.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/documents/product.py b/apps/search_indexes/documents/product.py index a1179a4f..9e57c607 100644 --- a/apps/search_indexes/documents/product.py +++ b/apps/search_indexes/documents/product.py @@ -95,7 +95,7 @@ class ProductDocument(Document): }, multi=True ) - name = fields.KeywordField(attr='display_name') + name = fields.TextField(attr='display_name') class Django: model = models.Product From 9d4a672d67dd6d03664867e746751a05c9fdde48 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 15/22] Multilingual textfield products analyzers --- apps/search_indexes/documents/product.py | 4 +++- apps/search_indexes/views.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/search_indexes/documents/product.py b/apps/search_indexes/documents/product.py index 9e57c607..cb0131cb 100644 --- a/apps/search_indexes/documents/product.py +++ b/apps/search_indexes/documents/product.py @@ -95,7 +95,9 @@ class ProductDocument(Document): }, multi=True ) - name = fields.TextField(attr='display_name') + name = fields.TextField(attr='display_name', analyzer='english') + name_ru = fields.TextField(attr='display_name', analyzer='russian') + name_fr = fields.TextField(attr='display_name', analyzer='french') class Django: model = models.Product diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index dcc6f895..174b1dd8 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -216,7 +216,11 @@ class ProductDocumentViewSet(BaseDocumentViewSet): search_fields = { 'name': {'fuzziness': 'auto:2,5', - 'boost': 4}, + 'boost': 8}, + 'name_ru': {'fuzziness': 'auto:2,5', + 'boost': 6}, + 'name_fr': {'fuzziness': 'auto:2,5', + 'boost': 7}, 'transliterated_name': {'fuzziness': 'auto:2,5', 'boost': 3}, 'description': {'fuzziness': 'auto:2,5'}, From 70b21107448547e904c9af7e5bad28efeb5139ca Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 16/22] Fix booking slots order --- apps/booking/views.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/booking/views.py b/apps/booking/views.py index cdadc768..73f6f55e 100644 --- a/apps/booking/views.py +++ b/apps/booking/views.py @@ -14,6 +14,12 @@ class CheckWhetherBookingAvailable(generics.GenericAPIView): """ Checks which service to use if establishmend is managed by any """ _VALID_GUESTONLINE_PERIODS = {'lunch', 'dinner', 'afternoon', 'breakfast'} + _GUESTONLINE_PERIODS_TO_PRIOR = { + 'breakfast': 1, + 'lunch': 2, + 'afternoon': 3, + 'dinner': 4, + } permission_classes = (permissions.AllowAny,) serializer_class = CheckBookingSerializer @@ -32,7 +38,7 @@ class CheckWhetherBookingAvailable(generics.GenericAPIView): period_template = iter(periods_by_name.values()).__next__().copy() period_template.pop('total_left_seats') - period_template.pop('hours') + period_template['hours'] = [] period_template.pop('period') processed_periods = [ @@ -46,7 +52,8 @@ class CheckWhetherBookingAvailable(generics.GenericAPIView): for unnamed_period in unnamed_periods: processed_periods.append(unnamed_period) - response['periods'] = processed_periods + response['periods'] = sorted(processed_periods, + key=lambda x: self._GUESTONLINE_PERIODS_TO_PRIOR[x.get('period', 'lunch')]) return response From 45b0b93068b580dcb6f9ce1416a5ac2a4d4566ce Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 22:49:38 +0300 Subject: [PATCH 17/22] add new migrations for tags --- apps/news/views.py | 5 +- apps/search_indexes/__init__.py | 2 +- apps/search_indexes/signals.py | 134 +++++++++---------- apps/tag/management/commands/add_tags.py | 162 +++++++++++++---------- apps/tag/models.py | 1 + project/settings/base.py | 6 +- project/settings/local.py | 10 +- 7 files changed, 171 insertions(+), 149 deletions(-) diff --git a/apps/news/views.py b/apps/news/views.py index f7d18101..d0528132 100644 --- a/apps/news/views.py +++ b/apps/news/views.py @@ -1,11 +1,8 @@ """News app views.""" from django.conf import settings -from django.db.transaction import on_commit from django.shortcuts import get_object_or_404 -from rest_framework import generics, permissions, status -from rest_framework.response import Response +from rest_framework import generics, permissions -from gallery.tasks import delete_image from news import filters, models, serializers from rating.tasks import add_rating from utils.permissions import IsCountryAdmin, IsContentPageManager diff --git a/apps/search_indexes/__init__.py b/apps/search_indexes/__init__.py index 6c812219..c45cbc38 100644 --- a/apps/search_indexes/__init__.py +++ b/apps/search_indexes/__init__.py @@ -1 +1 @@ -from search_indexes.signals import update_document +# from search_indexes.signals import update_document diff --git a/apps/search_indexes/signals.py b/apps/search_indexes/signals.py index 5bee4e17..1c7dbfe0 100644 --- a/apps/search_indexes/signals.py +++ b/apps/search_indexes/signals.py @@ -1,67 +1,67 @@ -"""Search indexes app signals.""" -from django.db.models.signals import post_save -from django.dispatch import receiver -from django_elasticsearch_dsl.registries import registry - - -@receiver(post_save) -def update_document(sender, **kwargs): - from establishment.models import Establishment - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - - app_label_model_name_to_filter = { - ('location','country'): 'address__city__country', - ('location','city'): 'address__city', - ('location', 'address'): 'address', - # todo: remove after migration - ('establishment', 'establishmenttype'): 'establishment_type', - ('establishment', 'establishmentsubtype'): 'establishment_subtypes', - ('tag', 'tag'): 'tags', - } - filter_name = app_label_model_name_to_filter.get((app_label, model_name)) - if filter_name: - qs = Establishment.objects.filter(**{filter_name: instance}) - for product in qs: - registry.update(product) - - -@receiver(post_save) -def update_news(sender, **kwargs): - from news.models import News - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - app_label_model_name_to_filter = { - ('location','country'): 'country', - ('news','newstype'): 'news_type', - ('tag', 'tag'): 'tags', - } - filter_name = app_label_model_name_to_filter.get((app_label, model_name)) - if filter_name: - qs = News.objects.filter(**{filter_name: instance}) - for product in qs: - registry.update(product) - - -@receiver(post_save) -def update_product(sender, **kwargs): - from product.models import Product - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - app_label_model_name_to_filter = { - ('product','productstandard'): 'standards', - ('product', 'producttype'): 'product_type', - ('tag','tag'): 'tags', - ('location', 'wineregion'): 'wine_region', - ('location', 'winesubregion'): 'wine_sub_region', - ('location', 'winevillage'): 'wine_village', - ('establishment', 'establishment'): 'establishment', - } - filter_name = app_label_model_name_to_filter.get((app_label, model_name)) - if filter_name: - qs = Product.objects.filter(**{filter_name: instance}) - for product in qs: - registry.update(product) +# """Search indexes app signals.""" +# from django.db.models.signals import post_save +# from django.dispatch import receiver +# from django_elasticsearch_dsl.registries import registry +# +# +# @receiver(post_save) +# def update_document(sender, **kwargs): +# from establishment.models import Establishment +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# +# app_label_model_name_to_filter = { +# ('location','country'): 'address__city__country', +# ('location','city'): 'address__city', +# ('location', 'address'): 'address', +# # todo: remove after migration +# ('establishment', 'establishmenttype'): 'establishment_type', +# ('establishment', 'establishmentsubtype'): 'establishment_subtypes', +# ('tag', 'tag'): 'tags', +# } +# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) +# if filter_name: +# qs = Establishment.objects.filter(**{filter_name: instance}) +# for product in qs: +# registry.update(product) +# +# +# @receiver(post_save) +# def update_news(sender, **kwargs): +# from news.models import News +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# app_label_model_name_to_filter = { +# ('location','country'): 'country', +# ('news','newstype'): 'news_type', +# ('tag', 'tag'): 'tags', +# } +# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) +# if filter_name: +# qs = News.objects.filter(**{filter_name: instance}) +# for product in qs: +# registry.update(product) +# +# +# @receiver(post_save) +# def update_product(sender, **kwargs): +# from product.models import Product +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# app_label_model_name_to_filter = { +# ('product','productstandard'): 'standards', +# ('product', 'producttype'): 'product_type', +# ('tag','tag'): 'tags', +# ('location', 'wineregion'): 'wine_region', +# ('location', 'winesubregion'): 'wine_sub_region', +# ('location', 'winevillage'): 'wine_village', +# ('establishment', 'establishment'): 'establishment', +# } +# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) +# if filter_name: +# qs = Product.objects.filter(**{filter_name: instance}) +# for product in qs: +# registry.update(product) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index 0fba46f8..b0c54e5c 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -3,87 +3,111 @@ from django.core.management.base import BaseCommand from establishment.models import Establishment, EstablishmentType from transfer import models as legacy from tag.models import Tag, TagCategory +from tqdm import tqdm +from django.db import connections +from collections import namedtuple + + +def namedtuplefetchall(cursor): + "Return all rows from a cursor as a namedtuple" + desc = cursor.description + nt_result = namedtuple('Result', [col[0] for col in desc]) + return [nt_result(*row) for row in cursor.fetchall()] + + +def metadata_category_sql(): + with connections['legacy'].cursor() as cursor: + cursor.execute( + '''SELECT + `key`, + establishments.type, + key_value_metadata.`value_type`, + public, + key_value_metadata.id as 'old_id' + FROM metadata + LEFT JOIN establishments + ON metadata.establishment_id=establishments.id + LEFT JOIN key_value_metadata + ON metadata.key=key_value_metadata.key_name + GROUP BY + establishments.type, + `key`, + key_value_metadata.`value_type`, + public, old_id;''' + ) + return namedtuplefetchall(cursor) + + +def metadata_tags_sql(): + with connections['legacy'].cursor() as cursor: + cursor.execute( + """ + SELECT + value, + `key` as category, + establishment_id +FROM metadata +WHERE establishment_id is not null""" + ) + return namedtuplefetchall(cursor) class Command(BaseCommand): help = 'Add tags values from old db to new db' - def handle(self, *args, **kwargs): + def get_type(self, meta): + meta_type = meta.value_type + if not meta.value_type: + if meta.key == 'wineyard_visits': + meta_type = 'list' + elif meta.key in ['private_room', 'outside_sits']: + meta_type = 'bool' + return meta_type + def handle(self, *args, **kwargs): existing_establishment = Establishment.objects.filter( old_id__isnull=False ) - ESTABLISHMENT = 1 - SHOP = 2 - RESTAURANT = 3 - WINEYARD = 4 - MAPPER = { - RESTAURANT: EstablishmentType.RESTAURANT, - WINEYARD: EstablishmentType.PRODUCER, - SHOP: EstablishmentType.ARTISAN + 'Restaurant': EstablishmentType.RESTAURANT, + 'Wineyard': EstablishmentType.PRODUCER, + 'Shop': EstablishmentType.ARTISAN } + # remove old black category + for establishment_tag in tqdm(EstablishmentType.objects.all()): + establishment_tag.tag_categories.remove(*list( + establishment_tag.tag_categories.exclude( + tags__establishments__isnull=False).distinct())) - mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all() - for key, value in MAPPER.items(): - values_meta_id_list = mapper_values_meta.filter( - key_value_metadatum_establishment_id=key - ).values_list('key_value_metadatum_id') + # created Tag Category + for meta in tqdm(metadata_category_sql()): + category, _ = TagCategory.objects.update_or_create( + index_name=meta.key, + defaults={ + "public": True if meta.public == 1 else False, + "value_type": self.get_type(meta), + "label": {"en-GB": meta.key} + } + ) - est_type, _ = EstablishmentType.objects.get_or_create(index_name=value) + # add to EstablishmentType + est_type = EstablishmentType.objects.get(index_name=MAPPER[meta.type]) + if category not in est_type.tag_categories.all(): + est_type.tag_categories.add(category) - key_value_metadata = legacy.KeyValueMetadata.objects.filter( - id__in=values_meta_id_list) + count = 0 + for meta_tag in tqdm(metadata_tags_sql()): - # create TagCategory - for key_value in key_value_metadata: - tag_category, created = TagCategory.objects.get_or_create( - index_name=key_value.key_name, - ) - - if created: - tag_category.label = { - 'en-GB': key_value.key_name, - 'fr-FR': key_value.key_name, - 'ru-RU': key_value.key_name, - } - tag_category.value_type = key_value.value_type - tag_category.save() - est_type.tag_categories.add( - tag_category - ) - - # create Tag - for tag in key_value.metadata_set.filter( - establishment__id__in=list( - existing_establishment.values_list('old_id', flat=True) - )): - - new_tag, created = Tag.objects.get_or_create( - value=tag.value, - category=tag_category, - ) - if created: - - sp = tag.value.split('_') - value = ' '.join([sp[0].capitalize()] + sp[1:]) - - trans = { - 'en-GB': value, - 'fr-FR': value, - 'ru-RU': value, - } - - aliases = legacy.MetadatumAliases.objects.filter(value=tag.value) - - for alias in aliases: - trans[alias.locale] = alias.meta_alias - - new_tag.label = trans - new_tag.save() - - est = existing_establishment.filter( - old_id=tag.establishment_id).first() - if est: - est.tags.add(new_tag) - est.save() + tag, _ = Tag.objects.update_or_create( + category=TagCategory.objects.get(index_name=meta_tag.category), + value=meta_tag.value, + defaults={ + "label": {"en-GB": meta_tag.value} + } + ) + establishment = existing_establishment.filter(old_id=meta_tag.establishment_id).first() + if establishment: + if tag not in establishment.tags.all(): + establishment.tags.add(tag) + count += 1 + self.stdout.write(self.style.WARNING(f'Created {count} tags to Establishment')) diff --git a/apps/tag/models.py b/apps/tag/models.py index a35425e8..29f18a23 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -37,6 +37,7 @@ class Tag(TranslatedFieldsMixin, models.Model): chosen_tag_settings = models.ManyToManyField(Country, through='ChosenTagSettings') priority = models.PositiveIntegerField(null=True, default=0) + # It does not make sense since in the old base another structure with duplicates old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) old_id_meta_product = models.PositiveIntegerField(_('old id metadata product'), diff --git a/project/settings/base.py b/project/settings/base.py index 70427ef3..4a2d0584 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -66,7 +66,7 @@ PROJECT_APPS = [ 'partner.apps.PartnerConfig', 'product.apps.ProductConfig', 'recipe.apps.RecipeConfig', - 'search_indexes.apps.SearchIndexesConfig', + # 'search_indexes.apps.SearchIndexesConfig', 'translation.apps.TranslationConfig', 'configuration.apps.ConfigurationConfig', 'timetable.apps.TimetableConfig', @@ -79,8 +79,8 @@ PROJECT_APPS = [ EXTERNAL_APPS = [ 'corsheaders', - 'django_elasticsearch_dsl', - 'django_elasticsearch_dsl_drf', + # 'django_elasticsearch_dsl', + # 'django_elasticsearch_dsl_drf', 'django_filters', 'drf_yasg', 'fcm_django', diff --git a/project/settings/local.py b/project/settings/local.py index f9a096fe..605a98c9 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -80,11 +80,11 @@ LOGGING = { 'py.warnings': { 'handlers': ['console'], }, - 'django.db.backends': { - 'handlers': ['console', ], - 'level': 'DEBUG', - 'propagate': False, - }, + # 'django.db.backends': { + # 'handlers': ['console', ], + # 'level': 'DEBUG', + # 'propagate': False, + # }, } } From ad3966b2aa261751c23dc9b4307cac36b22996a9 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 22:56:32 +0300 Subject: [PATCH 18/22] add method for label --- apps/tag/management/commands/add_tags.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index b0c54e5c..3c43a14d 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -43,11 +43,11 @@ def metadata_tags_sql(): cursor.execute( """ SELECT - value, - `key` as category, - establishment_id -FROM metadata -WHERE establishment_id is not null""" + value, + `key` as category, + establishment_id + FROM metadata + WHERE establishment_id is not null""" ) return namedtuplefetchall(cursor) @@ -64,6 +64,11 @@ class Command(BaseCommand): meta_type = 'bool' return meta_type + def get_label(self, text): + sp = text.key.split('_') + label = ' '.join([sp[0].capitalize()] + sp[1:]) + return label + def handle(self, *args, **kwargs): existing_establishment = Establishment.objects.filter( old_id__isnull=False @@ -86,7 +91,7 @@ class Command(BaseCommand): defaults={ "public": True if meta.public == 1 else False, "value_type": self.get_type(meta), - "label": {"en-GB": meta.key} + "label": {"en-GB": self.get_label(meta.key)} } ) @@ -102,7 +107,7 @@ class Command(BaseCommand): category=TagCategory.objects.get(index_name=meta_tag.category), value=meta_tag.value, defaults={ - "label": {"en-GB": meta_tag.value} + "label": {"en-GB": self.get_label(meta_tag.value)} } ) establishment = existing_establishment.filter(old_id=meta_tag.establishment_id).first() From b1d176c583eee7640bf734fae00837715a3c8ea8 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 23:01:22 +0300 Subject: [PATCH 19/22] enable elastic --- apps/search_indexes/__init__.py | 2 +- apps/search_indexes/signals.py | 134 ++++++++++++++++---------------- project/settings/base.py | 6 +- 3 files changed, 71 insertions(+), 71 deletions(-) diff --git a/apps/search_indexes/__init__.py b/apps/search_indexes/__init__.py index c45cbc38..6c812219 100644 --- a/apps/search_indexes/__init__.py +++ b/apps/search_indexes/__init__.py @@ -1 +1 @@ -# from search_indexes.signals import update_document +from search_indexes.signals import update_document diff --git a/apps/search_indexes/signals.py b/apps/search_indexes/signals.py index 1c7dbfe0..5bee4e17 100644 --- a/apps/search_indexes/signals.py +++ b/apps/search_indexes/signals.py @@ -1,67 +1,67 @@ -# """Search indexes app signals.""" -# from django.db.models.signals import post_save -# from django.dispatch import receiver -# from django_elasticsearch_dsl.registries import registry -# -# -# @receiver(post_save) -# def update_document(sender, **kwargs): -# from establishment.models import Establishment -# app_label = sender._meta.app_label -# model_name = sender._meta.model_name -# instance = kwargs['instance'] -# -# app_label_model_name_to_filter = { -# ('location','country'): 'address__city__country', -# ('location','city'): 'address__city', -# ('location', 'address'): 'address', -# # todo: remove after migration -# ('establishment', 'establishmenttype'): 'establishment_type', -# ('establishment', 'establishmentsubtype'): 'establishment_subtypes', -# ('tag', 'tag'): 'tags', -# } -# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) -# if filter_name: -# qs = Establishment.objects.filter(**{filter_name: instance}) -# for product in qs: -# registry.update(product) -# -# -# @receiver(post_save) -# def update_news(sender, **kwargs): -# from news.models import News -# app_label = sender._meta.app_label -# model_name = sender._meta.model_name -# instance = kwargs['instance'] -# app_label_model_name_to_filter = { -# ('location','country'): 'country', -# ('news','newstype'): 'news_type', -# ('tag', 'tag'): 'tags', -# } -# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) -# if filter_name: -# qs = News.objects.filter(**{filter_name: instance}) -# for product in qs: -# registry.update(product) -# -# -# @receiver(post_save) -# def update_product(sender, **kwargs): -# from product.models import Product -# app_label = sender._meta.app_label -# model_name = sender._meta.model_name -# instance = kwargs['instance'] -# app_label_model_name_to_filter = { -# ('product','productstandard'): 'standards', -# ('product', 'producttype'): 'product_type', -# ('tag','tag'): 'tags', -# ('location', 'wineregion'): 'wine_region', -# ('location', 'winesubregion'): 'wine_sub_region', -# ('location', 'winevillage'): 'wine_village', -# ('establishment', 'establishment'): 'establishment', -# } -# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) -# if filter_name: -# qs = Product.objects.filter(**{filter_name: instance}) -# for product in qs: -# registry.update(product) +"""Search indexes app signals.""" +from django.db.models.signals import post_save +from django.dispatch import receiver +from django_elasticsearch_dsl.registries import registry + + +@receiver(post_save) +def update_document(sender, **kwargs): + from establishment.models import Establishment + app_label = sender._meta.app_label + model_name = sender._meta.model_name + instance = kwargs['instance'] + + app_label_model_name_to_filter = { + ('location','country'): 'address__city__country', + ('location','city'): 'address__city', + ('location', 'address'): 'address', + # todo: remove after migration + ('establishment', 'establishmenttype'): 'establishment_type', + ('establishment', 'establishmentsubtype'): 'establishment_subtypes', + ('tag', 'tag'): 'tags', + } + filter_name = app_label_model_name_to_filter.get((app_label, model_name)) + if filter_name: + qs = Establishment.objects.filter(**{filter_name: instance}) + for product in qs: + registry.update(product) + + +@receiver(post_save) +def update_news(sender, **kwargs): + from news.models import News + app_label = sender._meta.app_label + model_name = sender._meta.model_name + instance = kwargs['instance'] + app_label_model_name_to_filter = { + ('location','country'): 'country', + ('news','newstype'): 'news_type', + ('tag', 'tag'): 'tags', + } + filter_name = app_label_model_name_to_filter.get((app_label, model_name)) + if filter_name: + qs = News.objects.filter(**{filter_name: instance}) + for product in qs: + registry.update(product) + + +@receiver(post_save) +def update_product(sender, **kwargs): + from product.models import Product + app_label = sender._meta.app_label + model_name = sender._meta.model_name + instance = kwargs['instance'] + app_label_model_name_to_filter = { + ('product','productstandard'): 'standards', + ('product', 'producttype'): 'product_type', + ('tag','tag'): 'tags', + ('location', 'wineregion'): 'wine_region', + ('location', 'winesubregion'): 'wine_sub_region', + ('location', 'winevillage'): 'wine_village', + ('establishment', 'establishment'): 'establishment', + } + filter_name = app_label_model_name_to_filter.get((app_label, model_name)) + if filter_name: + qs = Product.objects.filter(**{filter_name: instance}) + for product in qs: + registry.update(product) diff --git a/project/settings/base.py b/project/settings/base.py index 4a2d0584..70427ef3 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -66,7 +66,7 @@ PROJECT_APPS = [ 'partner.apps.PartnerConfig', 'product.apps.ProductConfig', 'recipe.apps.RecipeConfig', - # 'search_indexes.apps.SearchIndexesConfig', + 'search_indexes.apps.SearchIndexesConfig', 'translation.apps.TranslationConfig', 'configuration.apps.ConfigurationConfig', 'timetable.apps.TimetableConfig', @@ -79,8 +79,8 @@ PROJECT_APPS = [ EXTERNAL_APPS = [ 'corsheaders', - # 'django_elasticsearch_dsl', - # 'django_elasticsearch_dsl_drf', + 'django_elasticsearch_dsl', + 'django_elasticsearch_dsl_drf', 'django_filters', 'drf_yasg', 'fcm_django', From 9f03da9bef1a0e1729e6a8bbb41a2cdbb73fd729 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 23:03:17 +0300 Subject: [PATCH 20/22] fix for tag migration --- apps/tag/management/commands/add_tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index 3c43a14d..fd3443ec 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -65,7 +65,7 @@ class Command(BaseCommand): return meta_type def get_label(self, text): - sp = text.key.split('_') + sp = text.split('_') label = ' '.join([sp[0].capitalize()] + sp[1:]) return label From 82c79091a3393ac30a636c6846f8f9db40e18162 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 23:24:22 +0300 Subject: [PATCH 21/22] add some change for add translations for tags --- .../commands/add_tags_translation.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/tag/management/commands/add_tags_translation.py b/apps/tag/management/commands/add_tags_translation.py index a0f19e4c..485a7f11 100644 --- a/apps/tag/management/commands/add_tags_translation.py +++ b/apps/tag/management/commands/add_tags_translation.py @@ -1,19 +1,17 @@ from django.core.management.base import BaseCommand -from establishment.models import Establishment, EstablishmentType +from tag.models import Tag from transfer import models as legacy -from tag.models import Tag, TagCategory +from tqdm import tqdm class Command(BaseCommand): help = 'Add tags translation from old db to new db' - def handle(self, *args, **kwargs): - translation = legacy.MetadatumAliases.objects.all() - # Humanisation for default values - + def humanization_tag(self): + """Humanisation for default values""" tags = Tag.objects.all() - for tag in tags: + for tag in tqdm(tags): value = tag.label for k, v in value.items(): if isinstance(v, str) and '_' in v: @@ -22,10 +20,13 @@ class Command(BaseCommand): tag.label[k] = v tag.save() - for trans in translation: + def handle(self, *args, **kwargs): + translation = legacy.MetadatumAliases.objects.all() + # self.humanization_tag() + for trans in tqdm(translation): tag = Tag.objects.filter(value=trans.value).first() if tag: tag.label.update( {trans.locale: trans.meta_alias} ) - tag.save() \ No newline at end of file + tag.save() From 09db5c95409749fb3f54c4e0d0cdc3d73aeb5fd3 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 23:26:50 +0300 Subject: [PATCH 22/22] minor changes for tags translation command --- apps/tag/management/commands/add_tags_translation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/tag/management/commands/add_tags_translation.py b/apps/tag/management/commands/add_tags_translation.py index 485a7f11..a168d35e 100644 --- a/apps/tag/management/commands/add_tags_translation.py +++ b/apps/tag/management/commands/add_tags_translation.py @@ -8,8 +8,9 @@ from tqdm import tqdm class Command(BaseCommand): help = 'Add tags translation from old db to new db' - def humanization_tag(self): - """Humanisation for default values""" + @staticmethod + def humanisation_tag(self): + """Humanisation for default values.""" tags = Tag.objects.all() for tag in tqdm(tags): value = tag.label @@ -21,8 +22,9 @@ class Command(BaseCommand): tag.save() def handle(self, *args, **kwargs): + """Translation for existed tags.""" translation = legacy.MetadatumAliases.objects.all() - # self.humanization_tag() + # self.humanisation_tag() for trans in tqdm(translation): tag = Tag.objects.filter(value=trans.value).first() if tag: