Merge branch 'develop' into feature/gm-73

apps/establishment/models.py

@@ -317,6 +317,11 @@ class Establishment(ProjectBaseMixin, ImageMixin, TranslatedFieldsMixin):
     def best_price_carte(self):
         return 200
 
+    @property
+    def tags_indexing(self):
+        return [{'id': tag.metadata.id,
+                'label': tag.metadata.label} for tag in self.tags.all()]
+
 
 class Position(BaseAttributes, TranslatedFieldsMixin):
     """Position model."""

apps/news/models.py

@@ -1,6 +1,7 @@
 """News app models."""
 from django.db import models
 from django.utils.translation import gettext_lazy as _
+from rest_framework.reverse import reverse
 
 from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin
 
@@ -77,3 +78,6 @@ class News(BaseAttributes, TranslatedFieldsMixin):
     def __str__(self):
         return f'news: {self.id}'
 
+    @property
+    def web_url(self):
+        return reverse('web:news:rud', kwargs={'pk': self.pk})

apps/notification/serializers/common.py

@@ -9,14 +9,13 @@ class SubscribeSerializer(serializers.ModelSerializer):
     """Subscribe serializer."""
 
     email = serializers.EmailField(required=False, source='send_to')
-    state_display = serializers.CharField(source='get_state_display', read_only=True)
 
     class Meta:
         """Meta class."""
 
         model = models.Subscriber
-        fields = ('email', 'state', 'state_display')
+        fields = ('email', 'state',)
-        read_only_fields = ('state', 'state_display')
+        read_only_fields = ('state',)
 
     def validate(self, attrs):
         """Validate attrs."""

apps/search_indexes/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+from django.utils.translation import ugettext_lazy as _
+
+
+class SearchIndexesConfig(AppConfig):
+    name = 'search_indexes'
+    verbose_name = _('Search indexes')

apps/search_indexes/documents/__init__.py

@@ -0,0 +1,9 @@
+from search_indexes.documents.establishment import EstablishmentDocument
+from search_indexes.documents.news import NewsDocument
+
+
+# todo: make signal to update documents on related fields
+__all__ = [
+    'EstablishmentDocument',
+    'NewsDocument',
+]

apps/search_indexes/documents/establishment.py

@@ -0,0 +1,40 @@
+"""Establishment app documents."""
+from django.conf import settings
+from django_elasticsearch_dsl import Document, Index, fields
+from search_indexes.utils import OBJECT_FIELD_PROPERTIES
+from establishment import models
+
+
+EstablishmentIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__,
+                                                                  'establishment'))
+EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1)
+
+
+@EstablishmentIndex.doc_type
+class EstablishmentDocument(Document):
+    """Establishment document."""
+
+    description = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES)
+    tags = fields.ObjectField(
+        properties={
+            'id': fields.IntegerField(attr='id'),
+            'label': fields.ObjectField(attr='label')
+        },
+        multi=True)
+
+    class Django:
+
+        model = models.Establishment
+        fields = (
+            'id',
+            'name',
+            'public_mark',
+            'toque_number',
+            'price_level',
+        )
+
+    def prepare_description(self, instance):
+        return instance.description
+
+    def prepare_tags(self, instance):
+        return instance.tags_indexing

apps/search_indexes/documents/news.py

@@ -0,0 +1,48 @@
+"""News app documents."""
+from django.conf import settings
+from django_elasticsearch_dsl import Document, Index, fields
+from search_indexes.utils import OBJECT_FIELD_PROPERTIES
+from news import models
+
+
+NewsIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, 'news'))
+NewsIndex.settings(number_of_shards=1, number_of_replicas=1)
+
+
+@NewsIndex.doc_type
+class NewsDocument(Document):
+    """News document."""
+
+    news_type = fields.NestedField(properties={
+        'id': fields.IntegerField(),
+        'name': fields.KeywordField()
+    })
+    title = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES)
+    subtitle = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES)
+    description = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES)
+    country = fields.NestedField(properties={
+        'id': fields.IntegerField(),
+        'code': fields.KeywordField()
+    })
+    web_url = fields.KeywordField(attr='web_url')
+
+    class Django:
+
+        model = models.News
+        fields = (
+            'id',
+            'playlist',
+        )
+        related_models = [models.NewsType]
+
+    def get_queryset(self):
+        return super().get_queryset().published()
+
+    def prepare_title(self, instance):
+        return instance.title
+
+    def prepare_subtitle(self, instance):
+        return instance.subtitle
+
+    def prepare_description(self, instance):
+        return instance.description

apps/search_indexes/filters.py

@@ -0,0 +1,82 @@
+"""Search indexes filters."""
+from elasticsearch_dsl.query import Q
+from django_elasticsearch_dsl_drf.filter_backends import SearchFilterBackend
+from utils.models import get_current_language
+
+
+class CustomSearchFilterBackend(SearchFilterBackend):
+    """Custom SearchFilterBackend."""
+
+    @staticmethod
+    def get_field_name(view, field):
+        field_name = field
+        if hasattr(view,  'search_fields') and hasattr(view, 'translated_search_fields'):
+            if field in view.translated_search_fields:
+                field_name = f'{field}.{get_current_language()}'
+        return field_name
+
+    def construct_search(self, request, view):
+        """Construct search.
+
+        We have to deal with two types of structures:
+
+        Type 1:
+
+        >>> search_fields = (
+        >>>     'title',
+        >>>     'description',
+        >>>     'summary',
+        >>> )
+
+        Type 2:
+
+        >>> search_fields = {
+        >>>     'title': {'boost': 2},
+        >>>     'description': None,
+        >>>     'summary': None,
+        >>> }
+
+        :param request: Django REST framework request.
+        :param queryset: Base queryset.
+        :param view: View.
+        :type request: rest_framework.request.Request
+        :type queryset: elasticsearch_dsl.search.Search
+        :type view: rest_framework.viewsets.ReadOnlyModelViewSet
+        :return: Updated queryset.
+        :rtype: elasticsearch_dsl.search.Search
+        """
+        query_params = self.get_search_query_params(request)
+        __queries = []
+        for search_term in query_params:
+            __values = self.split_lookup_name(search_term, 1)
+            __len_values = len(__values)
+            if __len_values > 1:
+                field, value = __values
+                if field in view.search_fields:
+                    # Initial kwargs for the match query
+                    field_kwargs = {self.get_field_name(view, field): {'query': value}}
+                    # In case if we deal with structure 2
+                    if isinstance(view.search_fields, dict):
+                        extra_field_kwargs = view.search_fields[field]
+                        if extra_field_kwargs:
+                            field_kwargs[self.get_field_name(view, field)].update(extra_field_kwargs)
+                    # The match query
+                    __queries.append(
+                        Q("match", **field_kwargs)
+                    )
+            else:
+                for field in view.search_fields:
+                    # Initial kwargs for the match query
+                    field_kwargs = {self.get_field_name(view, field): {'query': search_term}}
+
+                    # In case if we deal with structure 2
+                    if isinstance(view.search_fields, dict):
+                        extra_field_kwargs = view.search_fields[field]
+                        if extra_field_kwargs:
+                            field_kwargs[self.get_field_name(view, field)].update(extra_field_kwargs)
+
+                    # The match query
+                    __queries.append(
+                        Q("match", **field_kwargs)
+                    )
+        return __queries

apps/search_indexes/serializers.py

@@ -0,0 +1,65 @@
+"""Search indexes serializers."""
+from rest_framework import serializers
+from django_elasticsearch_dsl_drf.serializers import DocumentSerializer
+from search_indexes.documents import EstablishmentDocument, NewsDocument
+from search_indexes.utils import get_translated_value
+
+
+class NewsDocumentSerializer(DocumentSerializer):
+    """News document serializer."""
+
+    title_translated = serializers.SerializerMethodField(allow_null=True)
+    subtitle_translated = serializers.SerializerMethodField(allow_null=True)
+    description_translated = serializers.SerializerMethodField(allow_null=True)
+
+    class Meta:
+        """Meta class."""
+
+        document = NewsDocument
+        fields = (
+            'id',
+            'title',
+            'subtitle',
+            'description',
+            'web_url',
+            'title_translated',
+            'subtitle_translated',
+            'description_translated',
+        )
+
+    @staticmethod
+    def get_title_translated(obj):
+        return get_translated_value(obj.title)
+
+    @staticmethod
+    def get_subtitle_translated(obj):
+        return get_translated_value(obj.subtitle)
+
+    @staticmethod
+    def get_description_translated(obj):
+        return get_translated_value(obj.description)
+
+
+class EstablishmentDocumentSerializer(DocumentSerializer):
+    """Establishment document serializer."""
+
+    description_translated = serializers.SerializerMethodField(allow_null=True)
+
+    class Meta:
+        """Meta class."""
+
+        document = EstablishmentDocument
+        fields = (
+            'id',
+            'name',
+            'description',
+            'public_mark',
+            'toque_number',
+            'price_level',
+            'description_translated',
+            'tags',
+        )
+
+    @staticmethod
+    def get_description_translated(obj):
+        return get_translated_value(obj.description)

apps/search_indexes/urls.py

@@ -0,0 +1,10 @@
+"""Search indexes app urlconf."""
+from rest_framework import routers
+from search_indexes import views
+
+
+router = routers.SimpleRouter()
+router.register(r'news', views.NewsDocumentViewSet, basename='news')
+router.register(r'establishments', views.EstablishmentDocumentViewSet, basename='establishment')
+
+urlpatterns = router.urls

apps/search_indexes/utils.py

@@ -0,0 +1,19 @@
+"""Search indexes utils."""
+from django_elasticsearch_dsl import fields
+from utils.models import get_current_language
+
+
+# object field properties
+OBJECT_FIELD_PROPERTIES = {
+    'en-GB': fields.TextField(analyzer='english'),
+    'ru-RU': fields.TextField(analyzer='russian'),
+}
+
+
+# todo: refactor serializer
+def get_translated_value(value):
+    if value is None:
+        return None
+    elif not isinstance(value, dict):
+        field_dict = value.to_dict()
+    return field_dict.get(get_current_language())

apps/search_indexes/views.py

@@ -0,0 +1,81 @@
+"""Search indexes app views."""
+from rest_framework import permissions
+from django_elasticsearch_dsl_drf import constants
+from django_elasticsearch_dsl_drf.filter_backends import FilteringFilterBackend
+from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet
+from django_elasticsearch_dsl_drf.pagination import PageNumberPagination
+from search_indexes import serializers, filters
+from search_indexes.documents import EstablishmentDocument, NewsDocument
+
+
+class NewsDocumentViewSet(BaseDocumentViewSet):
+    """News document ViewSet."""
+
+    document = NewsDocument
+    lookup_field = 'id'
+    pagination_class = PageNumberPagination
+    permission_classes = (permissions.AllowAny,)
+    serializer_class = serializers.NewsDocumentSerializer
+    ordering = ('id',)
+
+    filter_backends = [
+        filters.CustomSearchFilterBackend,
+    ]
+
+    search_fields = (
+        'title',
+        'subtitle',
+        'description',
+    )
+    translated_search_fields = (
+        'title',
+        'subtitle',
+        'description',
+    )
+
+
+class EstablishmentDocumentViewSet(BaseDocumentViewSet):
+    """Establishment document ViewSet."""
+
+    document = EstablishmentDocument
+    lookup_field = 'id'
+    pagination_class = PageNumberPagination
+    permission_classes = (permissions.AllowAny,)
+    serializer_class = serializers.EstablishmentDocumentSerializer
+    ordering = ('id',)
+
+    filter_backends = [
+        FilteringFilterBackend,
+        filters.CustomSearchFilterBackend,
+    ]
+
+    search_fields = (
+        'name',
+        'description',
+    )
+    translated_search_fields = (
+        'description',
+    )
+    filter_fields = {
+        'tag': 'tags.id',
+        'toque_number': {
+            'field': 'toque_number',
+            'lookups': [
+                constants.LOOKUP_FILTER_RANGE,
+                constants.LOOKUP_QUERY_GT,
+                constants.LOOKUP_QUERY_GTE,
+                constants.LOOKUP_QUERY_LT,
+                constants.LOOKUP_QUERY_LTE,
+            ]
+        },
+        'price_level': {
+            'field': 'price_level',
+            'lookups': [
+                constants.LOOKUP_FILTER_RANGE,
+                constants.LOOKUP_QUERY_GT,
+                constants.LOOKUP_QUERY_GTE,
+                constants.LOOKUP_QUERY_LT,
+                constants.LOOKUP_QUERY_LTE,
+            ]
+        },
+    }

docker-compose.yml

@@ -16,6 +16,20 @@ services:
       - db-net
     volumes:
       - gm-db:/var/lib/postgresql/data/
+  elasticsearch:
+    image: elasticsearch:7.3.1
+    volumes:
+      - gm-esdata:/usr/share/elasticsearch/data
+    hostname: elasticsearch
+    ports:
+      - 9200:9200
+      - 9300:9300
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    networks:
+      - app-net
   # RabbitMQ
   rabbitmq:
     image: rabbitmq:latest
@@ -68,6 +82,7 @@ services:
       - rabbitmq
       - worker
       - worker_beat
+      - elasticsearch
     networks:
       - app-net
       - db-net
@@ -87,3 +102,5 @@ volumes:
 
   gm-media:
     name: gm-media
+
+  gm-esdata:

project/settings/__init__.py

@@ -13,5 +13,8 @@ elif configuration == 'development':
 elif configuration == 'production':
     # production server settings
     from .production import *
+elif configuration == 'stage':
+    # production server settings
+    from .stage import *
 else:
     from .base import *

project/settings/base.py

@@ -63,6 +63,7 @@ PROJECT_APPS = [
     'news.apps.NewsConfig',
     'notification.apps.NotificationConfig',
     'partner.apps.PartnerConfig',
+    'search_indexes.apps.SearchIndexesConfig',
     'translation.apps.TranslationConfig',
     'configuration.apps.ConfigurationConfig',
     'timetable.apps.TimetableConfig',
@@ -72,6 +73,9 @@ PROJECT_APPS = [
 ]
 
 EXTERNAL_APPS = [
+    'corsheaders',
+    'django_elasticsearch_dsl',
+    'django_elasticsearch_dsl_drf',
     'django_filters',
     'drf_yasg',
     'fcm_django',
@@ -86,7 +90,6 @@ EXTERNAL_APPS = [
     'rest_framework_simplejwt.token_blacklist',
     'solo',
     'phonenumber_field',
-    'corsheaders',
 ]
 
 
@@ -377,7 +380,7 @@ CONFIRM_EMAIL_TEMPLATE = 'authorization/confirm_email.html'
 
 
 # COOKIES
-COOKIES_MAX_AGE = 86400  # 24 hours
+COOKIES_MAX_AGE = 2628000  # 30 days
 SESSION_COOKIE_SAMESITE = None
 
 

project/settings/development.py

@@ -1,5 +1,7 @@
 """Development settings."""
 from .base import *
+import sentry_sdk
+from sentry_sdk.integrations.django import DjangoIntegration
 
 ALLOWED_HOSTS = ['gm.id-east.ru', '95.213.204.126']
 
@@ -11,3 +13,23 @@ SCHEMA_URI = 'http'
 DEFAULT_SUBDOMAIN = 'www'
 SITE_DOMAIN_URI = 'id-east.ru'
 DOMAIN_URI = 'gm.id-east.ru'
+
+
+# ELASTICSEARCH SETTINGS
+ELASTICSEARCH_DSL = {
+    'default': {
+        'hosts': 'localhost:9200'
+    }
+}
+
+
+ELASTICSEARCH_INDEX_NAMES = {
+    'search_indexes.documents.news': 'development_news',
+    'search_indexes.documents.establishment': 'development_establishment',
+}
+
+
+sentry_sdk.init(
+    dsn="https://35d9bb789677410ab84a822831c6314f@sentry.io/1729093",
+    integrations=[DjangoIntegration()]
+)

project/settings/local.py

@@ -54,3 +54,16 @@ LOGGING = {
         },
     }
 }
+
+# ELASTICSEARCH SETTINGS
+ELASTICSEARCH_DSL = {
+    'default': {
+        'hosts': 'elasticsearch:9200'
+    }
+}
+
+
+ELASTICSEARCH_INDEX_NAMES = {
+    'search_indexes.documents.news': 'local_news',
+    'search_indexes.documents.establishment': 'local_establishment',
+}

project/settings/stage.py

@@ -11,3 +11,17 @@ SCHEMA_URI = 'https'
 DEFAULT_SUBDOMAIN = 'www'
 SITE_DOMAIN_URI = 'id-east.ru'
 DOMAIN_URI = 'gm-stage.id-east.ru'
+
+
+# ELASTICSEARCH SETTINGS
+ELASTICSEARCH_DSL = {
+    'default': {
+        'hosts': 'localhost:9200'
+    }
+}
+
+
+ELASTICSEARCH_INDEX_NAMES = {
+    'search_indexes.documents.news': 'stage_news',
+    'search_indexes.documents.establishment': 'stage_establishment',
+}

project/urls/__init__.py

@@ -55,6 +55,7 @@ api_urlpatterns = [
     path('web/', include(('project.urls.web', 'web'), namespace='web')),
     path('back/', include(('project.urls.back', 'back'), namespace='back')),
     path('mobile/', include(('project.urls.mobile', 'mobile'), namespace='mobile')),
+    path('search/', include('search_indexes.urls')),
 ]
 
 urlpatterns = [

requirements/base.txt

@@ -29,3 +29,7 @@ django-cors-headers==3.0.2
 
 # JWT
 djangorestframework-simplejwt==4.3.0
+
+django-elasticsearch-dsl>=7.0.0,<8.0.0
+django-elasticsearch-dsl-drf==0.20.2
+sentry-sdk==0.11.2