diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 2e9b27ed..84fc3e02 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -317,6 +317,11 @@ class Establishment(ProjectBaseMixin, ImageMixin, TranslatedFieldsMixin): def best_price_carte(self): return 200 + @property + def tags_indexing(self): + return [{'id': tag.metadata.id, + 'label': tag.metadata.label} for tag in self.tags.all()] + class Position(BaseAttributes, TranslatedFieldsMixin): """Position model.""" diff --git a/apps/news/models.py b/apps/news/models.py index df92341d..3be34261 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -1,6 +1,7 @@ """News app models.""" from django.db import models from django.utils.translation import gettext_lazy as _ +from rest_framework.reverse import reverse from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin @@ -77,3 +78,6 @@ class News(BaseAttributes, TranslatedFieldsMixin): def __str__(self): return f'news: {self.id}' + @property + def web_url(self): + return reverse('web:news:rud', kwargs={'pk': self.pk}) diff --git a/apps/notification/serializers/common.py b/apps/notification/serializers/common.py index a4d85d1f..f74e4c19 100644 --- a/apps/notification/serializers/common.py +++ b/apps/notification/serializers/common.py @@ -9,14 +9,13 @@ class SubscribeSerializer(serializers.ModelSerializer): """Subscribe serializer.""" email = serializers.EmailField(required=False, source='send_to') - state_display = serializers.CharField(source='get_state_display', read_only=True) class Meta: """Meta class.""" model = models.Subscriber - fields = ('email', 'state', 'state_display') - read_only_fields = ('state', 'state_display') + fields = ('email', 'state',) + read_only_fields = ('state',) def validate(self, attrs): """Validate attrs.""" diff --git a/apps/search_indexes/__init__.py b/apps/search_indexes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/search_indexes/apps.py b/apps/search_indexes/apps.py new file mode 100644 index 00000000..9834b705 --- /dev/null +++ b/apps/search_indexes/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig +from django.utils.translation import ugettext_lazy as _ + + +class SearchIndexesConfig(AppConfig): + name = 'search_indexes' + verbose_name = _('Search indexes') diff --git a/apps/search_indexes/documents/__init__.py b/apps/search_indexes/documents/__init__.py new file mode 100644 index 00000000..796bf289 --- /dev/null +++ b/apps/search_indexes/documents/__init__.py @@ -0,0 +1,9 @@ +from search_indexes.documents.establishment import EstablishmentDocument +from search_indexes.documents.news import NewsDocument + + +# todo: make signal to update documents on related fields +__all__ = [ + 'EstablishmentDocument', + 'NewsDocument', +] \ No newline at end of file diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py new file mode 100644 index 00000000..a12e23c8 --- /dev/null +++ b/apps/search_indexes/documents/establishment.py @@ -0,0 +1,40 @@ +"""Establishment app documents.""" +from django.conf import settings +from django_elasticsearch_dsl import Document, Index, fields +from search_indexes.utils import OBJECT_FIELD_PROPERTIES +from establishment import models + + +EstablishmentIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, + 'establishment')) +EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1) + + +@EstablishmentIndex.doc_type +class EstablishmentDocument(Document): + """Establishment document.""" + + description = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) + tags = fields.ObjectField( + properties={ + 'id': fields.IntegerField(attr='id'), + 'label': fields.ObjectField(attr='label') + }, + multi=True) + + class Django: + + model = models.Establishment + fields = ( + 'id', + 'name', + 'public_mark', + 'toque_number', + 'price_level', + ) + + def prepare_description(self, instance): + return instance.description + + def prepare_tags(self, instance): + return instance.tags_indexing diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py new file mode 100644 index 00000000..bf67a0f6 --- /dev/null +++ b/apps/search_indexes/documents/news.py @@ -0,0 +1,48 @@ +"""News app documents.""" +from django.conf import settings +from django_elasticsearch_dsl import Document, Index, fields +from search_indexes.utils import OBJECT_FIELD_PROPERTIES +from news import models + + +NewsIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, 'news')) +NewsIndex.settings(number_of_shards=1, number_of_replicas=1) + + +@NewsIndex.doc_type +class NewsDocument(Document): + """News document.""" + + news_type = fields.NestedField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField() + }) + title = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) + subtitle = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) + description = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) + country = fields.NestedField(properties={ + 'id': fields.IntegerField(), + 'code': fields.KeywordField() + }) + web_url = fields.KeywordField(attr='web_url') + + class Django: + + model = models.News + fields = ( + 'id', + 'playlist', + ) + related_models = [models.NewsType] + + def get_queryset(self): + return super().get_queryset().published() + + def prepare_title(self, instance): + return instance.title + + def prepare_subtitle(self, instance): + return instance.subtitle + + def prepare_description(self, instance): + return instance.description diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py new file mode 100644 index 00000000..f9ae0ef1 --- /dev/null +++ b/apps/search_indexes/filters.py @@ -0,0 +1,82 @@ +"""Search indexes filters.""" +from elasticsearch_dsl.query import Q +from django_elasticsearch_dsl_drf.filter_backends import SearchFilterBackend +from utils.models import get_current_language + + +class CustomSearchFilterBackend(SearchFilterBackend): + """Custom SearchFilterBackend.""" + + @staticmethod + def get_field_name(view, field): + field_name = field + if hasattr(view, 'search_fields') and hasattr(view, 'translated_search_fields'): + if field in view.translated_search_fields: + field_name = f'{field}.{get_current_language()}' + return field_name + + def construct_search(self, request, view): + """Construct search. + + We have to deal with two types of structures: + + Type 1: + + >>> search_fields = ( + >>> 'title', + >>> 'description', + >>> 'summary', + >>> ) + + Type 2: + + >>> search_fields = { + >>> 'title': {'boost': 2}, + >>> 'description': None, + >>> 'summary': None, + >>> } + + :param request: Django REST framework request. + :param queryset: Base queryset. + :param view: View. + :type request: rest_framework.request.Request + :type queryset: elasticsearch_dsl.search.Search + :type view: rest_framework.viewsets.ReadOnlyModelViewSet + :return: Updated queryset. + :rtype: elasticsearch_dsl.search.Search + """ + query_params = self.get_search_query_params(request) + __queries = [] + for search_term in query_params: + __values = self.split_lookup_name(search_term, 1) + __len_values = len(__values) + if __len_values > 1: + field, value = __values + if field in view.search_fields: + # Initial kwargs for the match query + field_kwargs = {self.get_field_name(view, field): {'query': value}} + # In case if we deal with structure 2 + if isinstance(view.search_fields, dict): + extra_field_kwargs = view.search_fields[field] + if extra_field_kwargs: + field_kwargs[self.get_field_name(view, field)].update(extra_field_kwargs) + # The match query + __queries.append( + Q("match", **field_kwargs) + ) + else: + for field in view.search_fields: + # Initial kwargs for the match query + field_kwargs = {self.get_field_name(view, field): {'query': search_term}} + + # In case if we deal with structure 2 + if isinstance(view.search_fields, dict): + extra_field_kwargs = view.search_fields[field] + if extra_field_kwargs: + field_kwargs[self.get_field_name(view, field)].update(extra_field_kwargs) + + # The match query + __queries.append( + Q("match", **field_kwargs) + ) + return __queries \ No newline at end of file diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py new file mode 100644 index 00000000..f4c27c9b --- /dev/null +++ b/apps/search_indexes/serializers.py @@ -0,0 +1,65 @@ +"""Search indexes serializers.""" +from rest_framework import serializers +from django_elasticsearch_dsl_drf.serializers import DocumentSerializer +from search_indexes.documents import EstablishmentDocument, NewsDocument +from search_indexes.utils import get_translated_value + + +class NewsDocumentSerializer(DocumentSerializer): + """News document serializer.""" + + title_translated = serializers.SerializerMethodField(allow_null=True) + subtitle_translated = serializers.SerializerMethodField(allow_null=True) + description_translated = serializers.SerializerMethodField(allow_null=True) + + class Meta: + """Meta class.""" + + document = NewsDocument + fields = ( + 'id', + 'title', + 'subtitle', + 'description', + 'web_url', + 'title_translated', + 'subtitle_translated', + 'description_translated', + ) + + @staticmethod + def get_title_translated(obj): + return get_translated_value(obj.title) + + @staticmethod + def get_subtitle_translated(obj): + return get_translated_value(obj.subtitle) + + @staticmethod + def get_description_translated(obj): + return get_translated_value(obj.description) + + +class EstablishmentDocumentSerializer(DocumentSerializer): + """Establishment document serializer.""" + + description_translated = serializers.SerializerMethodField(allow_null=True) + + class Meta: + """Meta class.""" + + document = EstablishmentDocument + fields = ( + 'id', + 'name', + 'description', + 'public_mark', + 'toque_number', + 'price_level', + 'description_translated', + 'tags', + ) + + @staticmethod + def get_description_translated(obj): + return get_translated_value(obj.description) diff --git a/apps/search_indexes/urls.py b/apps/search_indexes/urls.py new file mode 100644 index 00000000..60b05fb5 --- /dev/null +++ b/apps/search_indexes/urls.py @@ -0,0 +1,10 @@ +"""Search indexes app urlconf.""" +from rest_framework import routers +from search_indexes import views + + +router = routers.SimpleRouter() +router.register(r'news', views.NewsDocumentViewSet, basename='news') +router.register(r'establishments', views.EstablishmentDocumentViewSet, basename='establishment') + +urlpatterns = router.urls diff --git a/apps/search_indexes/utils.py b/apps/search_indexes/utils.py new file mode 100644 index 00000000..a3cfac87 --- /dev/null +++ b/apps/search_indexes/utils.py @@ -0,0 +1,19 @@ +"""Search indexes utils.""" +from django_elasticsearch_dsl import fields +from utils.models import get_current_language + + +# object field properties +OBJECT_FIELD_PROPERTIES = { + 'en-GB': fields.TextField(analyzer='english'), + 'ru-RU': fields.TextField(analyzer='russian'), +} + + +# todo: refactor serializer +def get_translated_value(value): + if value is None: + return None + elif not isinstance(value, dict): + field_dict = value.to_dict() + return field_dict.get(get_current_language()) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py new file mode 100644 index 00000000..5dbaf8fd --- /dev/null +++ b/apps/search_indexes/views.py @@ -0,0 +1,81 @@ +"""Search indexes app views.""" +from rest_framework import permissions +from django_elasticsearch_dsl_drf import constants +from django_elasticsearch_dsl_drf.filter_backends import FilteringFilterBackend +from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet +from django_elasticsearch_dsl_drf.pagination import PageNumberPagination +from search_indexes import serializers, filters +from search_indexes.documents import EstablishmentDocument, NewsDocument + + +class NewsDocumentViewSet(BaseDocumentViewSet): + """News document ViewSet.""" + + document = NewsDocument + lookup_field = 'id' + pagination_class = PageNumberPagination + permission_classes = (permissions.AllowAny,) + serializer_class = serializers.NewsDocumentSerializer + ordering = ('id',) + + filter_backends = [ + filters.CustomSearchFilterBackend, + ] + + search_fields = ( + 'title', + 'subtitle', + 'description', + ) + translated_search_fields = ( + 'title', + 'subtitle', + 'description', + ) + + +class EstablishmentDocumentViewSet(BaseDocumentViewSet): + """Establishment document ViewSet.""" + + document = EstablishmentDocument + lookup_field = 'id' + pagination_class = PageNumberPagination + permission_classes = (permissions.AllowAny,) + serializer_class = serializers.EstablishmentDocumentSerializer + ordering = ('id',) + + filter_backends = [ + FilteringFilterBackend, + filters.CustomSearchFilterBackend, + ] + + search_fields = ( + 'name', + 'description', + ) + translated_search_fields = ( + 'description', + ) + filter_fields = { + 'tag': 'tags.id', + 'toque_number': { + 'field': 'toque_number', + 'lookups': [ + constants.LOOKUP_FILTER_RANGE, + constants.LOOKUP_QUERY_GT, + constants.LOOKUP_QUERY_GTE, + constants.LOOKUP_QUERY_LT, + constants.LOOKUP_QUERY_LTE, + ] + }, + 'price_level': { + 'field': 'price_level', + 'lookups': [ + constants.LOOKUP_FILTER_RANGE, + constants.LOOKUP_QUERY_GT, + constants.LOOKUP_QUERY_GTE, + constants.LOOKUP_QUERY_LT, + constants.LOOKUP_QUERY_LTE, + ] + }, + } diff --git a/docker-compose.yml b/docker-compose.yml index 0ab73e8a..3fb05f98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,20 @@ services: - db-net volumes: - gm-db:/var/lib/postgresql/data/ + elasticsearch: + image: elasticsearch:7.3.1 + volumes: + - gm-esdata:/usr/share/elasticsearch/data + hostname: elasticsearch + ports: + - 9200:9200 + - 9300:9300 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - discovery.type=single-node + - xpack.security.enabled=false + networks: + - app-net # RabbitMQ rabbitmq: image: rabbitmq:latest @@ -68,6 +82,7 @@ services: - rabbitmq - worker - worker_beat + - elasticsearch networks: - app-net - db-net @@ -87,3 +102,5 @@ volumes: gm-media: name: gm-media + + gm-esdata: \ No newline at end of file diff --git a/project/settings/__init__.py b/project/settings/__init__.py index 6eaed852..5995ff1a 100644 --- a/project/settings/__init__.py +++ b/project/settings/__init__.py @@ -13,5 +13,8 @@ elif configuration == 'development': elif configuration == 'production': # production server settings from .production import * +elif configuration == 'stage': + # production server settings + from .stage import * else: from .base import * diff --git a/project/settings/base.py b/project/settings/base.py index b5c69ccf..71fbf147 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -63,6 +63,7 @@ PROJECT_APPS = [ 'news.apps.NewsConfig', 'notification.apps.NotificationConfig', 'partner.apps.PartnerConfig', + 'search_indexes.apps.SearchIndexesConfig', 'translation.apps.TranslationConfig', 'configuration.apps.ConfigurationConfig', 'timetable.apps.TimetableConfig', @@ -72,6 +73,9 @@ PROJECT_APPS = [ ] EXTERNAL_APPS = [ + 'corsheaders', + 'django_elasticsearch_dsl', + 'django_elasticsearch_dsl_drf', 'django_filters', 'drf_yasg', 'fcm_django', @@ -86,7 +90,6 @@ EXTERNAL_APPS = [ 'rest_framework_simplejwt.token_blacklist', 'solo', 'phonenumber_field', - 'corsheaders', ] @@ -377,7 +380,7 @@ CONFIRM_EMAIL_TEMPLATE = 'authorization/confirm_email.html' # COOKIES -COOKIES_MAX_AGE = 86400 # 24 hours +COOKIES_MAX_AGE = 2628000 # 30 days SESSION_COOKIE_SAMESITE = None diff --git a/project/settings/development.py b/project/settings/development.py index 7ecc6aa2..7c47ff5b 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -1,5 +1,7 @@ """Development settings.""" from .base import * +import sentry_sdk +from sentry_sdk.integrations.django import DjangoIntegration ALLOWED_HOSTS = ['gm.id-east.ru', '95.213.204.126'] @@ -11,3 +13,23 @@ SCHEMA_URI = 'http' DEFAULT_SUBDOMAIN = 'www' SITE_DOMAIN_URI = 'id-east.ru' DOMAIN_URI = 'gm.id-east.ru' + + +# ELASTICSEARCH SETTINGS +ELASTICSEARCH_DSL = { + 'default': { + 'hosts': 'localhost:9200' + } +} + + +ELASTICSEARCH_INDEX_NAMES = { + 'search_indexes.documents.news': 'development_news', + 'search_indexes.documents.establishment': 'development_establishment', +} + + +sentry_sdk.init( + dsn="https://35d9bb789677410ab84a822831c6314f@sentry.io/1729093", + integrations=[DjangoIntegration()] +) \ No newline at end of file diff --git a/project/settings/local.py b/project/settings/local.py index 1c73a857..febe4dd4 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -54,3 +54,16 @@ LOGGING = { }, } } + +# ELASTICSEARCH SETTINGS +ELASTICSEARCH_DSL = { + 'default': { + 'hosts': 'elasticsearch:9200' + } +} + + +ELASTICSEARCH_INDEX_NAMES = { + 'search_indexes.documents.news': 'local_news', + 'search_indexes.documents.establishment': 'local_establishment', +} \ No newline at end of file diff --git a/project/settings/stage.py b/project/settings/stage.py index bbb2f245..59fc78ed 100644 --- a/project/settings/stage.py +++ b/project/settings/stage.py @@ -11,3 +11,17 @@ SCHEMA_URI = 'https' DEFAULT_SUBDOMAIN = 'www' SITE_DOMAIN_URI = 'id-east.ru' DOMAIN_URI = 'gm-stage.id-east.ru' + + +# ELASTICSEARCH SETTINGS +ELASTICSEARCH_DSL = { + 'default': { + 'hosts': 'localhost:9200' + } +} + + +ELASTICSEARCH_INDEX_NAMES = { + 'search_indexes.documents.news': 'stage_news', + 'search_indexes.documents.establishment': 'stage_establishment', +} diff --git a/project/urls/__init__.py b/project/urls/__init__.py index eb9b623e..f1a89695 100644 --- a/project/urls/__init__.py +++ b/project/urls/__init__.py @@ -55,6 +55,7 @@ api_urlpatterns = [ path('web/', include(('project.urls.web', 'web'), namespace='web')), path('back/', include(('project.urls.back', 'back'), namespace='back')), path('mobile/', include(('project.urls.mobile', 'mobile'), namespace='mobile')), + path('search/', include('search_indexes.urls')), ] urlpatterns = [ diff --git a/requirements/base.txt b/requirements/base.txt index 1c70e7c2..25749c4b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -28,4 +28,8 @@ django-extensions==2.2.1 django-cors-headers==3.0.2 # JWT -djangorestframework-simplejwt==4.3.0 \ No newline at end of file +djangorestframework-simplejwt==4.3.0 + +django-elasticsearch-dsl>=7.0.0,<8.0.0 +django-elasticsearch-dsl-drf==0.20.2 +sentry-sdk==0.11.2 \ No newline at end of file