From 729a77866db44ee03ed3079ed96d0d806268a8f3 Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Thu, 3 Oct 2019 17:19:30 +0300 Subject: [PATCH 1/2] update elasticsearch index --- apps/establishment/models.py | 12 ++- .../search_indexes/documents/establishment.py | 9 +- apps/search_indexes/serializers.py | 96 +++++++++++-------- apps/search_indexes/views.py | 8 +- 4 files changed, 80 insertions(+), 45 deletions(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 589c2e22..5c2a0ff0 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -11,6 +11,7 @@ from django.db import models from django.db.models import When, Case, F, ExpressionWrapper, Subquery, Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from elasticsearch_dsl import Q from phonenumber_field.modelfields import PhoneNumberField from collection.models import Collection @@ -98,6 +99,16 @@ class EstablishmentQuerySet(models.QuerySet): else: return self.none() + def es_search(self, value, locale=None): + """Search text via ElasticSearch.""" + from search_indexes.documents import EstablishmentDocument + search = EstablishmentDocument.search().filter( + Q('match', name=value) | + Q('match', **{f'description.{locale}': value}) + ).execute() + ids = [result.meta.id for result in search] + return self.filter(id__in=ids) + def by_country_code(self, code): """Return establishments by country code""" return self.filter(address__city__country__code=code) @@ -330,7 +341,6 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): raise ValidationError('Establishment type of subtype does not match') self.establishment_subtypes.add(establishment_subtype) - @property def vintage_year(self): last_review = self.reviews.by_status(Review.READY).last() diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 16321723..a053e2c7 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -14,6 +14,7 @@ EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1) class EstablishmentDocument(Document): """Establishment document.""" + preview_image = fields.KeywordField(attr='preview_image_url') description = fields.ObjectField(attr='description_indexing', properties=OBJECT_FIELD_PROPERTIES) establishment_type = fields.ObjectField( @@ -50,7 +51,8 @@ class EstablishmentDocument(Document): fields={'raw': fields.KeywordField()} ), 'number': fields.IntegerField(), - 'location': fields.GeoPointField(attr='location_field_indexing'), + 'coordinates': fields.GeoPointField(attr='location_field_indexing'), + # todo: remove if not used 'city': fields.ObjectField( properties={ 'id': fields.IntegerField(), @@ -82,9 +84,10 @@ class EstablishmentDocument(Document): fields = ( 'id', 'name', - 'toque_number', + 'name_translated', 'price_level', - 'preview_image_url', + 'toque_number', + 'public_mark', 'slug', ) diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index 1d8e3ca3..fc08b471 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -40,13 +40,36 @@ class NewsDocumentSerializer(DocumentSerializer): return get_translated_value(obj.description) -# todo: country_name_translated +class TagsDocumentSerializer(serializers.Serializer): + """Tags serializer for ES Document.""" + + id = serializers.IntegerField() + label_translated = serializers.SerializerMethodField() + + def get_label_translated(self, obj): + return get_translated_value(obj.label) + + +class AddressDocumentSerializer(serializers.Serializer): + """Address serializer for ES Document.""" + + id = serializers.IntegerField() + street_name_1 = serializers.CharField() + street_name_2 = serializers.CharField() + number = serializers.IntegerField() + postal_code = serializers.CharField() + latitude = serializers.FloatField(allow_null=True, source='coordinates.lat') + longitude = serializers.FloatField(allow_null=True, source='coordinates.lon') + geo_lon = serializers.FloatField(allow_null=True, source='coordinates.lon') + geo_lat = serializers.FloatField(allow_null=True, source='coordinates.lat') + + class EstablishmentDocumentSerializer(DocumentSerializer): """Establishment document serializer.""" - description_translated = serializers.SerializerMethodField(allow_null=True) + address = AddressDocumentSerializer() + tags = TagsDocumentSerializer(many=True) - preview_image = serializers.URLField(source='preview_image_url') class Meta: """Meta class.""" @@ -54,43 +77,40 @@ class EstablishmentDocumentSerializer(DocumentSerializer): fields = ( 'id', 'name', - 'public_mark', - 'toque_number', + 'name_translated', 'price_level', - 'description_translated', - 'tags', - 'address', - 'collections', - 'establishment_type', - 'establishment_subtypes', - 'preview_image', + 'toque_number', + 'public_mark', 'slug', + 'preview_image', + 'address', + 'tags', + # 'collections', + # 'establishment_type', + # 'establishment_subtypes', ) - @staticmethod - def get_description_translated(obj): - return get_translated_value(obj.description) - def to_representation(self, instance): - ret = super().to_representation(instance) - dict_merge = lambda a, b: a.update(b) or a - - ret['tags'] = map(lambda tag: dict_merge(tag, {'label_translated': get_translated_value(tag.pop('label'))}), - ret['tags']) - ret['establishment_subtypes'] = map( - lambda subtype: dict_merge(subtype, {'name_translated': get_translated_value(subtype.pop('name'))}), - ret['establishment_subtypes']) - if ret.get('establishment_type'): - ret['establishment_type']['name_translated'] = get_translated_value(ret['establishment_type'].pop('name')) - if ret.get('address'): - ret['address']['city']['country']['name_translated'] = get_translated_value( - ret['address']['city']['country'].pop('name')) - location = ret['address'].pop('location') - if location: - ret['address']['geo_lon'] = location['lon'] - ret['address']['geo_lat'] = location['lat'] - - ret['type'] = ret.pop('establishment_type') - ret['subtypes'] = ret.pop('establishment_subtypes') - - return ret \ No newline at end of file + # def to_representation(self, instance): + # ret = super().to_representation(instance) + # dict_merge = lambda a, b: a.update(b) or a + # + # ret['tags'] = map(lambda tag: dict_merge(tag, {'label_translated': get_translated_value(tag.pop('label'))}), + # ret['tags']) + # ret['establishment_subtypes'] = map( + # lambda subtype: dict_merge(subtype, {'name_translated': get_translated_value(subtype.pop('name'))}), + # ret['establishment_subtypes']) + # if ret.get('establishment_type'): + # ret['establishment_type']['name_translated'] = get_translated_value(ret['establishment_type'].pop('name')) + # if ret.get('address'): + # ret['address']['city']['country']['name_translated'] = get_translated_value( + # ret['address']['city']['country'].pop('name')) + # location = ret['address'].pop('location') + # if location: + # ret['address']['geo_lon'] = location['lon'] + # ret['address']['geo_lat'] = location['lat'] + # + # ret['type'] = ret.pop('establishment_type') + # ret['subtypes'] = ret.pop('establishment_subtypes') + # + # return ret \ No newline at end of file diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index a69caf1f..bd2c91d3 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -1,8 +1,10 @@ """Search indexes app views.""" from rest_framework import permissions from django_elasticsearch_dsl_drf import constants -from django_elasticsearch_dsl_drf.filter_backends import (FilteringFilterBackend, - GeoSpatialFilteringFilterBackend) +from django_elasticsearch_dsl_drf.filter_backends import ( + FilteringFilterBackend, + GeoSpatialFilteringFilterBackend +) from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet from utils.pagination import ProjectPageNumberPagination @@ -108,7 +110,7 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): geo_spatial_filter_fields = { 'location': { - 'field': 'address.location', + 'field': 'address.coordinates', 'lookups': [ constants.LOOKUP_FILTER_GEO_BOUNDING_BOX, ] From 50a9622f78ade337152587dc490d4bc5fbb4d91e Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Fri, 4 Oct 2019 12:05:13 +0300 Subject: [PATCH 2/2] News search --- apps/news/models.py | 1 - apps/search_indexes/documents/news.py | 49 +++++++++++-------- apps/search_indexes/serializers.py | 69 +++++++++++++-------------- apps/search_indexes/signals.py | 56 ++++++++++++++++++++-- apps/search_indexes/views.py | 19 +++++++- 5 files changed, 132 insertions(+), 62 deletions(-) diff --git a/apps/news/models.py b/apps/news/models.py index efa0b566..ab3356bf 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -5,7 +5,6 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework.reverse import reverse from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin -from random import sample as random_sample class NewsType(models.Model): diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py index 45e50c42..6e0974d8 100644 --- a/apps/search_indexes/documents/news.py +++ b/apps/search_indexes/documents/news.py @@ -13,18 +13,26 @@ NewsIndex.settings(number_of_shards=1, number_of_replicas=1) class NewsDocument(Document): """News document.""" - news_type = fields.NestedField(properties={ - 'id': fields.IntegerField(), - 'name': fields.KeywordField() - }) - title = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) - subtitle = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) - description = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) - country = fields.NestedField(properties={ - 'id': fields.IntegerField(), - 'code': fields.KeywordField() - }) + news_type = fields.ObjectField(properties={'id': fields.IntegerField(), + 'name': fields.KeywordField()}) + title = fields.ObjectField(attr='title_indexing', + properties=OBJECT_FIELD_PROPERTIES) + subtitle = fields.ObjectField(attr='subtitle_indexing', + properties=OBJECT_FIELD_PROPERTIES) + description = fields.ObjectField(attr='description_indexing', + properties=OBJECT_FIELD_PROPERTIES) + country = fields.ObjectField(properties={'id': fields.IntegerField(), + 'code': fields.KeywordField()}) web_url = fields.KeywordField(attr='web_url') + tags = fields.ObjectField( + properties={ + 'id': fields.IntegerField(attr='metadata.id'), + 'label': fields.ObjectField(attr='metadata.label_indexing', + properties=OBJECT_FIELD_PROPERTIES), + 'category': fields.ObjectField(attr='metadata.category', + properties={'id': fields.IntegerField()}) + }, + multi=True) class Django: @@ -32,20 +40,19 @@ class NewsDocument(Document): fields = ( 'id', 'playlist', + 'start', + 'end', + 'slug', + 'state', + 'is_highlighted', + 'image_url', + 'preview_image_url', + 'template', ) related_models = [models.NewsType] def get_queryset(self): - return super().get_queryset().published() - - def prepare_title(self, instance): - return instance.title - - def prepare_subtitle(self, instance): - return instance.subtitle - - def prepare_description(self, instance): - return instance.description + return super().get_queryset().published().with_base_related() def get_instances_from_related(self, related_instance): """If related_models is set, define how to retrieve the Car instance(s) from the related model. diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index fc08b471..18a1e240 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -1,45 +1,11 @@ """Search indexes serializers.""" from rest_framework import serializers from django_elasticsearch_dsl_drf.serializers import DocumentSerializer +from news.serializers import NewsTypeSerializer from search_indexes.documents import EstablishmentDocument, NewsDocument from search_indexes.utils import get_translated_value -class NewsDocumentSerializer(DocumentSerializer): - """News document serializer.""" - - title_translated = serializers.SerializerMethodField(allow_null=True) - subtitle_translated = serializers.SerializerMethodField(allow_null=True) - description_translated = serializers.SerializerMethodField(allow_null=True) - - class Meta: - """Meta class.""" - - document = NewsDocument - fields = ( - 'id', - 'title', - 'subtitle', - 'description', - 'web_url', - 'title_translated', - 'subtitle_translated', - 'description_translated', - ) - - @staticmethod - def get_title_translated(obj): - return get_translated_value(obj.title) - - @staticmethod - def get_subtitle_translated(obj): - return get_translated_value(obj.subtitle) - - @staticmethod - def get_description_translated(obj): - return get_translated_value(obj.description) - - class TagsDocumentSerializer(serializers.Serializer): """Tags serializer for ES Document.""" @@ -64,6 +30,39 @@ class AddressDocumentSerializer(serializers.Serializer): geo_lat = serializers.FloatField(allow_null=True, source='coordinates.lat') +class NewsDocumentSerializer(DocumentSerializer): + """News document serializer.""" + + title_translated = serializers.SerializerMethodField(allow_null=True) + subtitle_translated = serializers.SerializerMethodField(allow_null=True) + news_type = NewsTypeSerializer() + tags = TagsDocumentSerializer(many=True) + + class Meta: + """Meta class.""" + + document = NewsDocument + fields = ( + 'id', + 'title_translated', + 'subtitle_translated', + 'is_highlighted', + 'image_url', + 'preview_image_url', + 'news_type', + 'tags', + 'slug', + ) + + @staticmethod + def get_title_translated(obj): + return get_translated_value(obj.title) + + @staticmethod + def get_subtitle_translated(obj): + return get_translated_value(obj.subtitle) + + class EstablishmentDocumentSerializer(DocumentSerializer): """Establishment document serializer.""" diff --git a/apps/search_indexes/signals.py b/apps/search_indexes/signals.py index 2c04b6c6..f7520b57 100644 --- a/apps/search_indexes/signals.py +++ b/apps/search_indexes/signals.py @@ -1,5 +1,5 @@ """Search indexes app signals.""" -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_save from django.dispatch import receiver from django_elasticsearch_dsl.registries import registry @@ -17,17 +17,65 @@ def update_document(sender, **kwargs): address__city__country=instance) for establishment in establishments: registry.update(establishment) - if model_name == 'city': establishments = Establishment.objects.filter( address__city=instance) for establishment in establishments: registry.update(establishment) - if model_name == 'address': establishments = Establishment.objects.filter( address=instance) for establishment in establishments: registry.update(establishment) -# todo: delete document + if app_label == 'establishment': + if model_name == 'establishmenttype': + establishments = Establishment.objects.filter( + establishment_type=instance) + for establishment in establishments: + registry.update(establishment) + if model_name == 'establishmentsubtype': + establishments = Establishment.objects.filter( + establishment_subtypes=instance) + for establishment in establishments: + registry.update(establishment) + + if app_label == 'main': + if model_name == 'metadata': + establishments = Establishment.objects.filter(tags__metadata=instance) + for establishment in establishments: + registry.update(establishment) + if model_name == 'metadatacontent': + establishments = Establishment.objects.filter(tags=instance) + for establishment in establishments: + registry.update(establishment) + + +@receiver(post_save) +def update_news(sender, **kwargs): + from news.models import News + app_label = sender._meta.app_label + model_name = sender._meta.model_name + instance = kwargs['instance'] + + if app_label == 'location': + if model_name == 'country': + qs = News.objects.filter(country=instance) + for news in qs: + registry.update(news) + + if app_label == 'news': + if model_name == 'newstype': + qs = News.objects.filter(news_type=instance) + for news in qs: + registry.update(news) + + if app_label == 'main': + if model_name == 'metadata': + qs = News.objects.filter(tags__metadata=instance) + for news in qs: + registry.update(news) + if model_name == 'metadatacontent': + qs = News.objects.filter(tags=instance) + for news in qs: + registry.update(news) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index bd2c91d3..57228c58 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -24,6 +24,7 @@ class NewsDocumentViewSet(BaseDocumentViewSet): filter_backends = [ filters.CustomSearchFilterBackend, + FilteringFilterBackend, ] search_fields = ( @@ -37,6 +38,16 @@ class NewsDocumentViewSet(BaseDocumentViewSet): 'description', ) + filter_fields = { + 'tag': { + 'field': 'tags.id', + 'lookups': [ + constants.LOOKUP_QUERY_IN, + ] + }, + 'slug': 'slug', + } + class EstablishmentDocumentViewSet(BaseDocumentViewSet): """Establishment document ViewSet.""" @@ -62,7 +73,13 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): 'description', ) filter_fields = { - 'tag': 'tags.id', + 'slug': 'slug', + 'tag': { + 'field': 'tags.id', + 'lookups': [ + constants.LOOKUP_QUERY_IN, + ] + }, 'toque_number': { 'field': 'toque_number', 'lookups': [