From 35b6a66c8541548a5cd899a0ba2def440ede7295 Mon Sep 17 00:00:00 2001
From: evgeniy-st <stoyushko.evgeniy@gmail.com>
Date: Thu, 21 Nov 2019 18:12:52 +0300
Subject: [PATCH] In favorites

---
 apps/establishment/models.py                  |  6 ++-
 apps/establishment/views/web.py               | 23 +++------
 apps/news/models.py                           |  6 ++-
 apps/news/views.py                            | 16 ++----
 apps/product/models.py                        |  5 +-
 apps/product/views/common.py                  | 21 ++------
 apps/search_indexes/documents/__init__.py     |  3 +-
 .../search_indexes/documents/establishment.py |  1 +
 apps/search_indexes/documents/news.py         |  3 +-
 apps/search_indexes/documents/product.py      |  1 +
 apps/search_indexes/serializers.py            | 24 +++++++--
 apps/search_indexes/tasks.py                  | 50 +++++++++++++++++++
 apps/utils/models.py                          |  8 +++
 apps/utils/views.py                           | 39 ++++++++++++++-
 docker-compose.yml                            |  2 +-
 project/settings/base.py                      | 11 ++++
 requirements/base.txt                         |  1 +
 run_celery_beat.sh                            |  2 +-
 18 files changed, 161 insertions(+), 61 deletions(-)
 create mode 100644 apps/search_indexes/tasks.py

diff --git a/apps/establishment/models.py b/apps/establishment/models.py
index 3aee6c31..a6737850 100644
--- a/apps/establishment/models.py
+++ b/apps/establishment/models.py
@@ -25,7 +25,8 @@ from main.models import Award, Currency
 from review.models import Review
 from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin,
                           TranslatedFieldsMixin, BaseAttributes, GalleryModelMixin,
-                          IntermediateGalleryModelMixin, HasTagsMixin)
+                          IntermediateGalleryModelMixin, HasTagsMixin,
+                          FavoritesMixin)
 
 
 # todo: establishment type&subtypes check
@@ -319,7 +320,8 @@ class EstablishmentQuerySet(models.QuerySet):
         return self.exclude(address__city__country__in=countries)
 
 
-class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin, HasTagsMixin):
+class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin,
+                    TranslatedFieldsMixin, HasTagsMixin, FavoritesMixin):
     """Establishment model."""
 
     # todo: delete image URL fields after moving on gallery
diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py
index 20e8f81a..4f1fe07c 100644
--- a/apps/establishment/views/web.py
+++ b/apps/establishment/views/web.py
@@ -5,12 +5,11 @@ from django.shortcuts import get_object_or_404
 from rest_framework import generics, permissions
 
 from comment import models as comment_models
-from establishment import filters
-from establishment import models, serializers
+from comment.serializers import CommentRUDSerializer
+from establishment import filters, models, serializers
 from main import methods
 from utils.pagination import EstablishmentPortionPagination
-from utils.permissions import IsCountryAdmin
-from comment.serializers import CommentRUDSerializer
+from utils.views import FavoritesCreateDestroyMixinView
 
 
 class EstablishmentMixinView:
@@ -134,21 +133,11 @@ class EstablishmentCommentRUDView(generics.RetrieveUpdateDestroyAPIView):
         return comment_obj
 
 
-class EstablishmentFavoritesCreateDestroyView(generics.CreateAPIView, generics.DestroyAPIView):
+class EstablishmentFavoritesCreateDestroyView(FavoritesCreateDestroyMixinView):
     """View for create/destroy establishment from favorites."""
-    serializer_class = serializers.EstablishmentFavoritesCreateSerializer
-    lookup_field = 'slug'
 
-    def get_object(self):
-        """
-        Returns the object the view is displaying.
-        """
-        establishment = get_object_or_404(models.Establishment,
-                                          slug=self.kwargs['slug'])
-        favorites = get_object_or_404(establishment.favorites.filter(user=self.request.user))
-        # May raise a permission denied
-        self.check_object_permissions(self.request, favorites)
-        return favorites
+    _model = models.Establishment
+    serializer_class = serializers.EstablishmentFavoritesCreateSerializer
 
 
 class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIView):
diff --git a/apps/news/models.py b/apps/news/models.py
index d0e79c64..8a86e688 100644
--- a/apps/news/models.py
+++ b/apps/news/models.py
@@ -8,7 +8,8 @@ from rest_framework.reverse import reverse
 
 from rating.models import Rating, ViewCount
 from utils.models import (BaseAttributes, TJSONField, TranslatedFieldsMixin, HasTagsMixin,
-                          ProjectBaseMixin, GalleryModelMixin, IntermediateGalleryModelMixin)
+                          ProjectBaseMixin, GalleryModelMixin, IntermediateGalleryModelMixin,
+                          FavoritesMixin)
 from utils.querysets import TranslationQuerysetMixin
 from django.conf import settings
 
@@ -126,7 +127,8 @@ class NewsQuerySet(TranslationQuerysetMixin):
         )
 
 
-class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixin):
+class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixin,
+           FavoritesMixin):
     """News model."""
 
     STR_FIELD_NAME = 'title'
diff --git a/apps/news/views.py b/apps/news/views.py
index 3e841246..a215947a 100644
--- a/apps/news/views.py
+++ b/apps/news/views.py
@@ -6,7 +6,7 @@ from rest_framework import generics, permissions
 from news import filters, models, serializers
 from rating.tasks import add_rating
 from utils.permissions import IsCountryAdmin, IsContentPageManager
-from utils.views import CreateDestroyGalleryViewMixin
+from utils.views import CreateDestroyGalleryViewMixin, FavoritesCreateDestroyMixinView
 from utils.serializers import ImageBaseSerializer
 
 
@@ -150,18 +150,8 @@ class NewsBackOfficeRUDView(NewsBackOfficeMixinView,
         return self.retrieve(request, *args, **kwargs)
 
 
-class NewsFavoritesCreateDestroyView(generics.CreateAPIView, generics.DestroyAPIView):
+class NewsFavoritesCreateDestroyView(FavoritesCreateDestroyMixinView):
     """View for create/destroy news from favorites."""
 
+    _model = models.News
     serializer_class = serializers.NewsFavoritesCreateSerializer
-    lookup_field = 'slug'
-
-    def get_object(self):
-        """
-        Returns the object the view is displaying.
-        """
-        news = get_object_or_404(models.News, slug=self.kwargs['slug'])
-        favorites = get_object_or_404(news.favorites.filter(user=self.request.user))
-        # May raise a permission denied
-        self.check_object_permissions(self.request, favorites)
-        return favorites
diff --git a/apps/product/models.py b/apps/product/models.py
index 6baaf43e..896eb211 100644
--- a/apps/product/models.py
+++ b/apps/product/models.py
@@ -8,7 +8,7 @@ from django.utils.translation import gettext_lazy as _
 from django.core.validators import MaxValueValidator, MinValueValidator
 
 from utils.models import (BaseAttributes, ProjectBaseMixin, HasTagsMixin,
-                          TranslatedFieldsMixin, TJSONField,
+                          TranslatedFieldsMixin, TJSONField, FavoritesMixin,
                           GalleryModelMixin, IntermediateGalleryModelMixin)
 
 
@@ -131,7 +131,8 @@ class ProductQuerySet(models.QuerySet):
         )
 
 
-class Product(GalleryModelMixin, TranslatedFieldsMixin, BaseAttributes, HasTagsMixin):
+class Product(GalleryModelMixin, TranslatedFieldsMixin, BaseAttributes,
+              HasTagsMixin, FavoritesMixin):
     """Product models."""
 
     EARLIEST_VINTAGE_YEAR = 1700
diff --git a/apps/product/views/common.py b/apps/product/views/common.py
index 8b857ddb..f984a87b 100644
--- a/apps/product/views/common.py
+++ b/apps/product/views/common.py
@@ -3,9 +3,9 @@ from rest_framework import generics, permissions
 from django.shortcuts import get_object_or_404
 from product.models import Product
 from comment.models import Comment
-from product import serializers
-from product import filters
+from product import filters, serializers
 from comment.serializers import CommentRUDSerializer
+from utils.views import FavoritesCreateDestroyMixinView
 
 
 class ProductBaseView(generics.GenericAPIView):
@@ -37,22 +37,11 @@ class ProductDetailView(ProductBaseView, generics.RetrieveAPIView):
     serializer_class = serializers.ProductDetailSerializer
 
 
-class CreateFavoriteProductView(generics.CreateAPIView,
-                                generics.DestroyAPIView):
+class CreateFavoriteProductView(FavoritesCreateDestroyMixinView):
     """View for create/destroy product in favorites."""
+
+    _model = Product
     serializer_class = serializers.ProductFavoritesCreateSerializer
-    lookup_field = 'slug'
-
-    def get_object(self):
-        """
-        Returns the object the view is displaying.
-        """
-        product = get_object_or_404(Product, slug=self.kwargs['slug'])
-        favorites = get_object_or_404(product.favorites.filter(user=self.request.user))
-
-        # May raise a permission denied
-        self.check_object_permissions(self.request, favorites)
-        return favorites
 
 
 class ProductCommentCreateView(generics.CreateAPIView):
diff --git a/apps/search_indexes/documents/__init__.py b/apps/search_indexes/documents/__init__.py
index 70d17330..c357f29e 100644
--- a/apps/search_indexes/documents/__init__.py
+++ b/apps/search_indexes/documents/__init__.py
@@ -1,11 +1,12 @@
 from search_indexes.documents.establishment import EstablishmentDocument
 from search_indexes.documents.news import NewsDocument
 from search_indexes.documents.product import ProductDocument
-
+from search_indexes.tasks import es_update
 
 # todo: make signal to update documents on related fields
 __all__ = [
     'EstablishmentDocument',
     'NewsDocument',
     'ProductDocument',
+    'es_update',
 ]
\ No newline at end of file
diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py
index 8b4e5c3c..45923182 100644
--- a/apps/search_indexes/documents/establishment.py
+++ b/apps/search_indexes/documents/establishment.py
@@ -113,6 +113,7 @@ class EstablishmentDocument(Document):
             ),
         },
     )
+    favorites_for_users = fields.ListField(field=fields.IntegerField())
 
     class Django:
 
diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py
index e39036d3..1fa2f9d9 100644
--- a/apps/search_indexes/documents/news.py
+++ b/apps/search_indexes/documents/news.py
@@ -41,6 +41,7 @@ class NewsDocument(Document):
                                         properties=OBJECT_FIELD_PROPERTIES),
         },
         multi=True)
+    favorites_for_users = fields.ListField(field=fields.IntegerField())
 
     class Django:
 
@@ -57,7 +58,7 @@ class NewsDocument(Document):
         related_models = [models.NewsType]
 
     def get_queryset(self):
-        return super().get_queryset().published().with_base_related().sort_by_start()
+        return super().get_queryset().published().with_base_related()
 
     def get_instances_from_related(self, related_instance):
         """If related_models is set, define how to retrieve the Car instance(s) from the related model.
diff --git a/apps/search_indexes/documents/product.py b/apps/search_indexes/documents/product.py
index 1a092dac..61c4eeeb 100644
--- a/apps/search_indexes/documents/product.py
+++ b/apps/search_indexes/documents/product.py
@@ -148,6 +148,7 @@ class ProductDocument(Document):
     name = fields.TextField(attr='display_name', analyzer='english')
     name_ru = fields.TextField(attr='display_name', analyzer='russian')
     name_fr = fields.TextField(attr='display_name', analyzer='french')
+    favorites_for_users = fields.ListField(field=fields.IntegerField())
 
     class Django:
         model = models.Product
diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py
index cac4e336..84782f0c 100644
--- a/apps/search_indexes/serializers.py
+++ b/apps/search_indexes/serializers.py
@@ -167,7 +167,25 @@ class ScheduleDocumentSerializer(serializers.Serializer):
     closed_at = serializers.CharField()
 
 
-class NewsDocumentSerializer(DocumentSerializer):
+class InFavoritesMixin(DocumentSerializer):
+    """Append in_favorites field."""
+
+    in_favorites = serializers.SerializerMethodField()
+
+    def get_in_favorites(self, obj):
+        request = self.context['request']
+        user = request.user
+        if user.is_authenticated:
+            return user.id in obj.favorites_for_users
+        return False
+
+    class Meta:
+        """Meta class."""
+
+        _abstract = True
+
+
+class NewsDocumentSerializer(InFavoritesMixin, DocumentSerializer):
     """News document serializer."""
 
     title_translated = serializers.SerializerMethodField(allow_null=True)
@@ -200,7 +218,7 @@ class NewsDocumentSerializer(DocumentSerializer):
         return get_translated_value(obj.subtitle)
 
 
-class EstablishmentDocumentSerializer(DocumentSerializer):
+class EstablishmentDocumentSerializer(InFavoritesMixin, DocumentSerializer):
     """Establishment document serializer."""
 
     establishment_type = EstablishmentTypeSerializer()
@@ -236,7 +254,7 @@ class EstablishmentDocumentSerializer(DocumentSerializer):
         )
 
 
-class ProductDocumentSerializer(DocumentSerializer):
+class ProductDocumentSerializer(InFavoritesMixin, DocumentSerializer):
     """Product document serializer"""
 
     tags = TagsDocumentSerializer(many=True, source='related_tags')
diff --git a/apps/search_indexes/tasks.py b/apps/search_indexes/tasks.py
new file mode 100644
index 00000000..4935814d
--- /dev/null
+++ b/apps/search_indexes/tasks.py
@@ -0,0 +1,50 @@
+"""SearchIndex tasks."""
+import logging
+from django.db import models
+from celery.schedules import crontab
+from celery.task import periodic_task
+from django_elasticsearch_dsl.registries import registry
+from django_redis import get_redis_connection
+from establishment.models import Establishment
+from news.models import News
+from product.models import Product
+
+
+logger = logging.getLogger(__name__)
+
+
+@periodic_task(run_every=crontab(minute=1))
+def update_index():
+    """Updates ES index."""
+    try:
+        cn = get_redis_connection('es_queue')
+        for model in [Establishment, News, Product]:
+            model_name = model.__name__.lower()
+            while True:
+                ids  = cn.spop(model_name, 500)
+                if not ids:
+                    break
+                qs = model.objects.filter(id__in=ids)
+                try:
+                    doc = registry.get_documents([model]).pop()
+                except KeyError:
+                    pass
+                else:
+                    doc().update(qs)
+    except Exception as ex:
+        logger.error(f'Updating index failed: {ex}')
+
+
+def es_update(obj):
+    """Adds object to set of objects for indexing."""
+    try:
+        cn = get_redis_connection('es_queue')
+        allowed_models = [Establishment, News, Product]
+        if isinstance(obj, models.QuerySet) and obj.model in allowed_models:
+            key = obj.model.__name__.lower()
+            cn.sadd(key, *obj.values_list('id', flat=True))
+        elif isinstance(obj, models.Model) and obj.__class__ in allowed_models:
+            key = obj.__class__.__name__.lower()
+            cn.sadd(key, obj.id)
+    except Exception as ex:
+        logger.warning(f'Send obj to ES failed: {ex}')
diff --git a/apps/utils/models.py b/apps/utils/models.py
index f86093af..8c186f2f 100644
--- a/apps/utils/models.py
+++ b/apps/utils/models.py
@@ -5,6 +5,7 @@ from os.path import exists
 from django.conf import settings
 from django.contrib.auth.tokens import PasswordResetTokenGenerator
 from django.contrib.gis.db import models
+from django.contrib.postgres.aggregates import ArrayAgg
 from django.contrib.postgres.fields import JSONField
 from django.contrib.postgres.fields.jsonb import KeyTextTransform
 from django.utils import timezone
@@ -435,4 +436,11 @@ class HasTagsMixin(models.Model):
         abstract = True
 
 
+class FavoritesMixin:
+    """Append favorites_for_user property."""
+
+    @property
+    def favorites_for_users(self):
+        return self.favorites.aggregate(arr=ArrayAgg('user_id')).get('arr')
+
 timezone.datetime.now().date().isoformat()
\ No newline at end of file
diff --git a/apps/utils/views.py b/apps/utils/views.py
index d3d09079..0f5f74cd 100644
--- a/apps/utils/views.py
+++ b/apps/utils/views.py
@@ -2,11 +2,12 @@ from collections import namedtuple
 
 from django.conf import settings
 from django.db.transaction import on_commit
-from rest_framework import generics
-from rest_framework import status
+from django.shortcuts import get_object_or_404
+from rest_framework import generics, status
 from rest_framework.response import Response
 
 from gallery.tasks import delete_image
+from search_indexes.documents import es_update
 
 
 # JWT
@@ -121,3 +122,37 @@ class CreateDestroyGalleryViewMixin(generics.CreateAPIView,
         # Delete an instances of Gallery model
         gallery_obj.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class FavoritesCreateDestroyMixinView(generics.CreateAPIView,
+                                      generics.DestroyAPIView):
+    """Favorites Create Destroy mixin."""
+
+    _model = None
+    serializer_class = None
+    lookup_field = 'slug'
+
+    def get_base_object(self):
+        return get_object_or_404(self._model, slug=self.kwargs['slug'])
+
+    def get_object(self):
+        """
+        Returns the object the view is displaying.
+        """
+        obj = self.get_base_object()
+        favorites = get_object_or_404(obj.favorites.filter(user=self.request.user))
+        # May raise a permission denied
+        self.check_object_permissions(self.request, favorites)
+        return favorites
+
+    def es_update_base_object(self):
+        es_update(self.get_base_object())
+
+    def perform_create(self, serializer):
+        serializer.save()
+        self.es_update_base_object()
+
+    def perform_destroy(self, instance):
+        instance.delete()
+        self.es_update_base_object()
+
diff --git a/docker-compose.yml b/docker-compose.yml
index 48fea8eb..12217d4e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -30,7 +30,7 @@ services:
 
   # Redis
   redis:
-    image: redis:2.8.23
+    image: redis:latest
 
   # Celery
   worker:
diff --git a/project/settings/base.py b/project/settings/base.py
index 9c365b52..792ad281 100644
--- a/project/settings/base.py
+++ b/project/settings/base.py
@@ -250,6 +250,17 @@ AUTHENTICATION_BACKENDS = (
     'django.contrib.auth.backends.ModelBackend',
 )
 
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+    },
+    'es_queue': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': 'redis://redis:6379/2'
+    }
+}
+
 # Override default OAuth2 namespace
 DRFSO2_URL_NAMESPACE = 'auth'
 SOCIAL_AUTH_URL_NAMESPACE = 'auth'
diff --git a/requirements/base.txt b/requirements/base.txt
index 8ce99c84..e02e8ac5 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -54,5 +54,6 @@ PyYAML==5.1.2
 
 # temp solution
 redis==3.2.0
+django_redis==4.10.0  # used byes indexing cache
 kombu==4.6.6
 celery==4.3.0
diff --git a/run_celery_beat.sh b/run_celery_beat.sh
index 75de842d..e683ef9e 100755
--- a/run_celery_beat.sh
+++ b/run_celery_beat.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 sleep 5
 
-celery -A project worker -B -l info
\ No newline at end of file
+celery -A project beat -l info
\ No newline at end of file