From a1f06fd0b9328bae804a8efe5e40e4ec92535236 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Thu, 19 Dec 2019 19:07:23 +0300 Subject: [PATCH 01/32] panels executor --- apps/main/admin.py | 13 ++++++ apps/main/models.py | 88 +++++++++++++++++++++++++++++++++++++++- apps/main/serializers.py | 17 ++++++++ apps/main/urls/back.py | 4 +- apps/main/views/back.py | 17 +++++++- apps/transfer/models.py | 1 + apps/utils/exceptions.py | 8 ++++ apps/utils/methods.py | 9 ++++ 8 files changed, 152 insertions(+), 5 deletions(-) diff --git a/apps/main/admin.py b/apps/main/admin.py index 315d1c2b..6a12541d 100644 --- a/apps/main/admin.py +++ b/apps/main/admin.py @@ -54,3 +54,16 @@ class PageAdmin(admin.ModelAdmin): list_display = ('id', '__str__', 'advertisement') list_filter = ('advertisement__url', 'source') date_hierarchy = 'created' + + +@admin.register(models.Footer) +class FooterAdmin(admin.ModelAdmin): + """Footer admin.""" + list_display = ('id', 'site', ) + + +@admin.register(models.Panel) +class PanelAdmin(admin.ModelAdmin): + """Panel admin.""" + list_display = ('id', 'created', ) + raw_id_fields = ('user', ) \ No newline at end of file diff --git a/apps/main/models.py b/apps/main/models.py index b39a6037..80fc5a1a 100644 --- a/apps/main/models.py +++ b/apps/main/models.py @@ -6,14 +6,18 @@ from django.contrib.contenttypes import fields as generic from django.contrib.contenttypes.models import ContentType from django.contrib.postgres.fields import JSONField from django.core.validators import EMPTY_VALUES +from django.db import connections, connection from django.db import models from django.db.models import Q from django.utils.translation import gettext_lazy as _ +from rest_framework import exceptions from configuration.models import TranslationSettings from location.models import Country from main import methods from review.models import Review +from utils.exceptions import UnprocessableEntityError +from utils.methods import dictfetchall from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin, TranslatedFieldsMixin, PlatformMixin) @@ -402,5 +406,85 @@ class Panel(ProjectBaseMixin): def __str__(self): return self.name - def execute_query(self): - pass + def execute_query(self, request): + """Execute query""" + raw = self.query + page = int(request.query_params.get('page', 0)) + page_size = int(request.query_params.get('page_size', 10)) + + if raw: + data = { + "count": 0, + "next": 2, + "previous": None, + "columns": None, + "results": [] + + } + with connections['default'].cursor() as cursor: + count = self._raw_count(raw) + start = page*page_size + cursor.execute(*self.set_limits(start, page_size)) + data["count"] = count + data["next"] = self.get_next_page(count, page, page_size) + data["previous"] = self.get_previous_page(count, page) + data["results"] = dictfetchall(cursor) + data["columns"] = self._raw_columns(cursor) + return data + + def get_next_page(self, count, page, page_size): + max_page = count/page_size-1 + if not 0 <= page <= max_page: + raise exceptions.NotFound('Invalid page.') + if max_page > page: + return page + 1 + return None + + def get_previous_page(self, count, page): + if page > 0: + return page - 1 + return None + + @staticmethod + def _raw_execute(row): + with connections['default'].cursor() as cursor: + try: + cursor.execute(row) + return cursor.execute(row) + except Exception as er: + # TODO: log + raise UnprocessableEntityError() + + def _raw_count(self, subquery): + if ';' in subquery: + subquery = subquery.replace(';', '') + _count_query = f"""SELECT count(*) from ({subquery}) as t;""" + # cursor = self._raw_execute(_count_query) + with connections['default'].cursor() as cursor: + cursor.execute(_count_query) + row = cursor.fetchone() + return row[0] + + @staticmethod + def _raw_columns(cursor): + columns = [col[0] for col in cursor.description] + return columns + + def _raw_page(self, raw, request): + page = request.query_params.get('page', 0) + page_size = request.query_params.get('page_size', 0) + raw = f"""{raw} LIMIT {page_size} OFFSET {page}""" + return raw + + def set_limits(self, start, limit, params=tuple()): + limit_offset = '' + new_params = tuple() + if start > 0: + new_params += (start,) + limit_offset = ' OFFSET %s' + if limit is not None: + new_params = (limit,) + new_params + limit_offset = ' LIMIT %s' + limit_offset + params = params + new_params + query = self.query + limit_offset + return query, params diff --git a/apps/main/serializers.py b/apps/main/serializers.py index 39c51845..0f86daa2 100644 --- a/apps/main/serializers.py +++ b/apps/main/serializers.py @@ -296,3 +296,20 @@ class PanelSerializer(serializers.ModelSerializer): 'user', 'user_id' ] + + +class PanelExecuteSerializer(serializers.ModelSerializer): + """Panel execute serializer.""" + class Meta: + model = models.Panel + fields = [ + 'id', + 'name', + 'display', + 'description', + 'query', + 'created', + 'modified', + 'user', + 'user_id' + ] diff --git a/apps/main/urls/back.py b/apps/main/urls/back.py index 26afd1a6..a2049a42 100644 --- a/apps/main/urls/back.py +++ b/apps/main/urls/back.py @@ -23,8 +23,8 @@ urlpatterns = [ path('page-types/', views.PageTypeListCreateView.as_view(), name='page-types-list-create'), path('panels/', views.PanelsListCreateView.as_view(), name='panels'), - path('panels//', views.PanelsListCreateView.as_view(), name='panels-rud'), - # path('panels//execute/', views.PanelsView.as_view(), name='panels-execute') + path('panels//', views.PanelsRUDView.as_view(), name='panels-rud'), + path('panels//execute/', views.PanelsExecuteView.as_view(), name='panels-execute') ] diff --git a/apps/main/views/back.py b/apps/main/views/back.py index 0a2b7377..dd898a88 100644 --- a/apps/main/views/back.py +++ b/apps/main/views/back.py @@ -1,11 +1,14 @@ from django.contrib.contenttypes.models import ContentType from django_filters.rest_framework import DjangoFilterBackend from rest_framework import generics, permissions +from rest_framework.generics import get_object_or_404 +from rest_framework.response import Response from main import serializers from main.filters import AwardFilter from main.models import Award, Footer, PageType, Panel from main.views import SiteSettingsView, SiteListView +from utils.pagination import TestPagination class AwardLstView(generics.ListCreateAPIView): @@ -106,4 +109,16 @@ class PanelsRUDView(generics.RetrieveUpdateDestroyAPIView): permissions.IsAdminUser, ) serializer_class = serializers.PanelSerializer - queryset = Panel.objects.all() \ No newline at end of file + queryset = Panel.objects.all() + + +class PanelsExecuteView(generics.ListAPIView): + """Custom panels view.""" + permission_classes = ( + permissions.IsAdminUser, + ) + queryset = Panel.objects.all() + + def list(self, request, *args, **kwargs): + panel = get_object_or_404(Panel, id=self.kwargs['pk']) + return Response(panel.execute_query(request)) diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 8fbb7e97..420e41bc 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -1239,6 +1239,7 @@ class OwnershipAffs(MigrateMixin): managed = False db_table = 'ownership_affs' + class Panels(MigrateMixin): using = 'legacy' diff --git a/apps/utils/exceptions.py b/apps/utils/exceptions.py index c82ff023..08ab433e 100644 --- a/apps/utils/exceptions.py +++ b/apps/utils/exceptions.py @@ -171,3 +171,11 @@ class RemovedBindingObjectNotFound(serializers.ValidationError): """The exception must be thrown if the object not found.""" default_detail = _('Removed binding object not found.') + + +class UnprocessableEntityError(exceptions.APIException): + """ + The exception should be thrown when executing data on server rise error. + """ + status_code = status.HTTP_422_UNPROCESSABLE_ENTITY + default_detail = _('Unprocessable entity valid.') diff --git a/apps/utils/methods.py b/apps/utils/methods.py index 227bd1ee..ef1d6d82 100644 --- a/apps/utils/methods.py +++ b/apps/utils/methods.py @@ -132,3 +132,12 @@ def namedtuplefetchall(cursor): desc = cursor.description nt_result = namedtuple('Result', [col[0] for col in desc]) return [nt_result(*row) for row in cursor.fetchall()] + + +def dictfetchall(cursor): + "Return all rows from a cursor as a dict" + columns = [col[0] for col in cursor.description] + return [ + dict(zip(columns, row)) + for row in cursor.fetchall() + ] \ No newline at end of file From 2b117ba0f5d18c6210b85551ad33e33ea13270da Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 01:23:21 +0300 Subject: [PATCH 02/32] fix typo --- apps/main/views/back.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/main/views/back.py b/apps/main/views/back.py index dd898a88..98398f17 100644 --- a/apps/main/views/back.py +++ b/apps/main/views/back.py @@ -8,7 +8,6 @@ from main import serializers from main.filters import AwardFilter from main.models import Award, Footer, PageType, Panel from main.views import SiteSettingsView, SiteListView -from utils.pagination import TestPagination class AwardLstView(generics.ListCreateAPIView): From f69d7d6e71d5c5150e2ee8bb982ad41f525e433d Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Fri, 20 Dec 2019 10:01:22 +0300 Subject: [PATCH 03/32] add chenge to panel admin --- apps/main/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/main/admin.py b/apps/main/admin.py index 6009f404..f4cd6970 100644 --- a/apps/main/admin.py +++ b/apps/main/admin.py @@ -70,5 +70,5 @@ class FooterLinkAdmin(admin.ModelAdmin): @admin.register(models.Panel) class PanelAdmin(admin.ModelAdmin): """Panel admin.""" - list_display = ('id', 'created', ) - raw_id_fields = ('user', ) \ No newline at end of file + list_display = ('id', 'name', 'user', 'created', ) + raw_id_fields = ('user', ) From 71fb3249b0a4b2be9e2a66d219db836580f343fb Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Fri, 20 Dec 2019 10:02:59 +0300 Subject: [PATCH 04/32] add chenge to panel admin --- apps/main/admin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/main/admin.py b/apps/main/admin.py index f4cd6970..8275e742 100644 --- a/apps/main/admin.py +++ b/apps/main/admin.py @@ -72,3 +72,5 @@ class PanelAdmin(admin.ModelAdmin): """Panel admin.""" list_display = ('id', 'name', 'user', 'created', ) raw_id_fields = ('user', ) + list_display_links = ('id', 'name', ) + From 36834835cd155cd80778df4442e7b67499ed5abb Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 20 Dec 2019 11:05:02 +0300 Subject: [PATCH 05/32] added filters to location views --- apps/location/filters.py | 31 +++++++++++++++++++++++++++++++ apps/location/models.py | 22 ++++++++++++++++++++++ apps/location/views/back.py | 7 ++----- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/apps/location/filters.py b/apps/location/filters.py index 5e95db44..c50b9964 100644 --- a/apps/location/filters.py +++ b/apps/location/filters.py @@ -22,3 +22,34 @@ class CityBackFilter(filters.FilterSet): if value not in EMPTY_VALUES: return queryset.search_by_name(value) return queryset + + +class RegionFilter(filters.FilterSet): + """Region filter set.""" + + country_id = filters.CharFilter() + sub_regions_by_region_id = filters.CharFilter(method='by_region') + without_parent_region = filters.BooleanFilter(method='by_parent_region') + + class Meta: + """Meta class.""" + model = models.Region + fields = ( + 'country_id', + 'sub_regions_by_region_id', + 'without_parent_region', + ) + + def by_region(self, queryset, name, value): + """Search regions by sub region id.""" + if value not in EMPTY_VALUES: + return queryset.sub_regions_by_region_id(value) + + def by_parent_region(self, queryset, name, value): + """ + Search if region instance has a parent region.. + If True then show only Regions + Otherwise show only Sub regions. + """ + if value not in EMPTY_VALUES: + return queryset.without_parent_region(value) diff --git a/apps/location/models.py b/apps/location/models.py index dc7834c9..be26e2c7 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -70,6 +70,26 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): return str_name +class RegionQuerySet(models.QuerySet): + """QuerySet for model Region.""" + + def without_parent_region(self, switcher: bool = True): + """Filter regions by parent region.""" + return self.filter(parent_region__isnull=switcher) + + def by_region_id(self, region_id): + """Filter regions by region id.""" + return self.filter(id=region_id) + + def by_sub_region_id(self, sub_region_id): + """Filter sub regions by sub region id.""" + return self.filter(parent_region_id=sub_region_id) + + def sub_regions_by_region_id(self, region_id): + """Filter regions by sub region id.""" + return self.filter(parent_region_id=region_id) + + class Region(models.Model): """Region model.""" @@ -82,6 +102,8 @@ class Region(models.Model): Country, verbose_name=_('country'), on_delete=models.CASCADE) old_id = models.IntegerField(null=True, blank=True, default=None) + objects = RegionQuerySet.as_manager() + class Meta: """Meta class.""" diff --git a/apps/location/views/back.py b/apps/location/views/back.py index 5dcb55bf..e306bc6e 100644 --- a/apps/location/views/back.py +++ b/apps/location/views/back.py @@ -1,5 +1,4 @@ """Location app views.""" -from django_filters.rest_framework import DjangoFilterBackend from rest_framework import generics from location import models, serializers @@ -9,6 +8,7 @@ from utils.views import CreateDestroyGalleryViewMixin from rest_framework.permissions import IsAuthenticatedOrReadOnly from django.shortcuts import get_object_or_404 from utils.serializers import ImageBaseSerializer +from location.filters import RegionFilter from location import filters @@ -109,11 +109,8 @@ class RegionListCreateView(common.RegionViewMixin, generics.ListCreateAPIView): pagination_class = None serializer_class = serializers.RegionSerializer permission_classes = [IsAuthenticatedOrReadOnly | IsCountryAdmin] - filter_backends = (DjangoFilterBackend,) ordering_fields = '__all__' - filterset_fields = ( - 'country', - ) + filter_class = RegionFilter class RegionRUDView(common.RegionViewMixin, generics.RetrieveUpdateDestroyAPIView): From 821526d6316f7093553153747ea6e8b994755d54 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 20 Dec 2019 12:36:11 +0300 Subject: [PATCH 06/32] remove incorrect check for establishment, set schedule attr property as blank=True, refactored timetable model --- .../migrations/0068_auto_20191220_0914.py | 18 +++++++++++++++ apps/establishment/models.py | 8 +------ apps/timetable/models.py | 22 ++++++++++++++----- 3 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 apps/establishment/migrations/0068_auto_20191220_0914.py diff --git a/apps/establishment/migrations/0068_auto_20191220_0914.py b/apps/establishment/migrations/0068_auto_20191220_0914.py new file mode 100644 index 00000000..d3c4f9ab --- /dev/null +++ b/apps/establishment/migrations/0068_auto_20191220_0914.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-12-20 09:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0067_auto_20191122_1244'), + ] + + operations = [ + migrations.AlterField( + model_name='establishment', + name='schedule', + field=models.ManyToManyField(blank=True, related_name='schedule', to='timetable.Timetable', verbose_name='Establishment schedule'), + ), + ] diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 8162aab1..8aaf345b 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -483,7 +483,7 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, booking = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Booking URL')) is_publish = models.BooleanField(default=False, verbose_name=_('Publish status')) - schedule = models.ManyToManyField(to='timetable.Timetable', + schedule = models.ManyToManyField(to='timetable.Timetable', blank=True, verbose_name=_('Establishment schedule'), related_name='schedule') # holidays_from = models.DateTimeField(verbose_name=_('Holidays from'), @@ -532,12 +532,6 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, def __str__(self): return f'id:{self.id}-{self.name}' - def clean_fields(self, exclude=None): - super().clean_fields(exclude) - if self.purchased_products.filter(product_type__index_name='souvenir').exists(): - raise ValidationError( - _('Only souvenirs.')) - def delete(self, using=None, keep_parents=False): """Overridden delete method""" # Delete all related companies diff --git a/apps/timetable/models.py b/apps/timetable/models.py index 90a6ae38..07e52807 100644 --- a/apps/timetable/models.py +++ b/apps/timetable/models.py @@ -35,6 +35,22 @@ class Timetable(ProjectBaseMixin): opening_at = models.TimeField(verbose_name=_('Opening time'), null=True) closed_at = models.TimeField(verbose_name=_('Closed time'), null=True) + class Meta: + """Meta class.""" + verbose_name = _('Timetable') + verbose_name_plural = _('Timetables') + ordering = ['weekday'] + + def __str__(self): + """Overridden str dunder.""" + return f'{self.get_weekday_display()} ' \ + f'(closed_at - {self.closed_at_str}, ' \ + f'opening_at - {self.opening_at_str}, ' \ + f'opening_time - {self.opening_time}, ' \ + f'ending_time - {self.ending_time}, ' \ + f'works_at_noon - {self.works_at_noon}, ' \ + f'works_at_afternoon: {self.works_at_afternoon})' + @property def closed_at_str(self): return str(self.closed_at) if self.closed_at else None @@ -58,9 +74,3 @@ class Timetable(ProjectBaseMixin): @property def works_at_afternoon(self): return bool(self.ending_time and self.ending_time > self.NOON) - - class Meta: - """Meta class.""" - verbose_name = _('Timetable') - verbose_name_plural = _('Timetables') - ordering = ['weekday'] From 35b7846ca198bcf0372771ba267fdf1a36589c8c Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 20 Dec 2019 12:57:48 +0300 Subject: [PATCH 07/32] refactored methods for getting similar establishments --- apps/establishment/models.py | 86 +++++++++++++++------------------ apps/establishment/views/web.py | 44 +++++++++++++---- 2 files changed, 73 insertions(+), 57 deletions(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 8aaf345b..2442d449 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -221,15 +221,16 @@ class EstablishmentQuerySet(models.QuerySet): Return filtered QuerySet by base filters. Filters including: 1 Filter by type (and subtype) establishment. - 2 Filter by published Review. - 3 With annotated distance. + 2 With annotated distance. + 3 By country """ filters = { - 'reviews__status': Review.READY, 'establishment_type': establishment.establishment_type, + 'address__city__country': establishment.address.city.country } if establishment.establishment_subtypes.exists(): filters.update({'establishment_subtypes__in': establishment.establishment_subtypes.all()}) + return self.exclude(id=establishment.id) \ .filter(**filters) \ .annotate_distance(point=establishment.location) @@ -248,28 +249,25 @@ class EstablishmentQuerySet(models.QuerySet): .values('id') ) - def similar_restaurants(self, slug): + def similar_restaurants(self, restaurant): """ Return QuerySet with objects that similar to Restaurant. - :param slug: str restaurant slug + :param restaurant: Establishment instance. """ - restaurant_qs = self.filter(slug=slug) - if restaurant_qs.exists(): - restaurant = restaurant_qs.first() - ids_by_subquery = self.similar_base_subquery( - establishment=restaurant, - filters={ - 'public_mark__gte': 10, - 'establishment_gallery__is_main': True, - } - ) - return self.filter(id__in=ids_by_subquery) \ - .annotate_intermediate_public_mark() \ - .annotate_mark_similarity(mark=restaurant.public_mark) \ - .order_by('mark_similarity') \ - .distinct('mark_similarity', 'id') - else: - return self.none() + + ids_by_subquery = self.similar_base_subquery( + establishment=restaurant, + filters={ + 'reviews__status': Review.READY, + 'public_mark__gte': 10, + 'establishment_gallery__is_main': True, + } + ) + return self.filter(id__in=ids_by_subquery) \ + .annotate_intermediate_public_mark() \ + .annotate_mark_similarity(mark=restaurant.public_mark) \ + .order_by('mark_similarity') \ + .distinct('mark_similarity', 'id') def same_subtype(self, establishment): """Annotate flag same subtype.""" @@ -282,21 +280,17 @@ class EstablishmentQuerySet(models.QuerySet): output_field=models.BooleanField(default=False) )) - def similar_artisans_producers(self, slug): + def similar_artisans_producers(self, establishment): """ Return QuerySet with objects that similar to Artisan/Producer(s). - :param slug: str artisan/producer slug + :param establishment: Establishment instance """ - establishment_qs = self.filter(slug=slug) - if establishment_qs.exists(): - establishment = establishment_qs.first() - return self.similar_base(establishment) \ - .same_subtype(establishment) \ - .order_by(F('same_subtype').desc(), - F('distance').asc()) \ - .distinct('same_subtype', 'distance', 'id') - else: - return self.none() + return self.similar_base(establishment) \ + .same_subtype(establishment) \ + .has_published_reviews() \ + .order_by(F('same_subtype').desc(), + F('distance').asc()) \ + .distinct('same_subtype', 'distance', 'id') def by_wine_region(self, wine_region): """ @@ -312,23 +306,19 @@ class EstablishmentQuerySet(models.QuerySet): """ return self.filter(wine_origin__wine_sub_region=wine_sub_region).distinct() - def similar_wineries(self, slug: str): + def similar_wineries(self, winery): """ Return QuerySet with objects that similar to Winery. - :param establishment_slug: str Establishment slug + :param winery: Establishment instance """ - winery_qs = self.filter(slug=slug) - if winery_qs.exists(): - winery = winery_qs.first() - return self.similar_base(winery) \ - .order_by(F('wine_origins__wine_region').asc(), - F('wine_origins__wine_sub_region').asc()) \ - .annotate_distance(point=winery.location) \ - .order_by('distance') \ - .distinct('distance', 'wine_origins__wine_region', - 'wine_origins__wine_sub_region', 'id') - else: - return self.none() + return self.similar_base(winery) \ + .order_by(F('wine_origins__wine_region').asc(), + F('wine_origins__wine_sub_region').asc(), + F('distance').asc()) \ + .distinct('wine_origins__wine_region', + 'wine_origins__wine_sub_region', + 'distance', + 'id') def last_reviewed(self, point: Point): """ diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 7eba8607..d94be1d5 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -46,6 +46,19 @@ class EstablishmentSimilarView(EstablishmentListView): serializer_class = serializers.EstablishmentSimilarSerializer pagination_class = PortionPagination + def get_base_object(self): + """ + Return base establishment instance for a getting list of similar establishments. + """ + establishment = get_object_or_404(models.Establishment.objects.all(), + slug=self.kwargs.get('slug')) + return establishment + + def get_queryset(self): + """Overridden get_queryset method.""" + return EstablishmentMixinView.get_queryset(self) \ + .has_location() + class EstablishmentRetrieveView(EstablishmentMixinView, generics.RetrieveAPIView): """Resource for getting a establishment.""" @@ -88,9 +101,14 @@ class RestaurantSimilarListView(EstablishmentSimilarView): def get_queryset(self): """Overridden get_queryset method""" - return EstablishmentMixinView.get_queryset(self) \ - .has_location() \ - .similar_restaurants(slug=self.kwargs.get('slug')) + qs = super(RestaurantSimilarListView, self).get_queryset() + base_establishment = self.get_base_object() + + if base_establishment: + return qs.similar_restaurants(base_establishment) + else: + return EstablishmentMixinView.get_queryset(self) \ + .none() class WinerySimilarListView(EstablishmentSimilarView): @@ -98,9 +116,13 @@ class WinerySimilarListView(EstablishmentSimilarView): def get_queryset(self): """Overridden get_queryset method""" - return EstablishmentMixinView.get_queryset(self) \ - .has_location() \ - .similar_wineries(slug=self.kwargs.get('slug')) + qs = EstablishmentSimilarView.get_queryset(self) + base_establishment = self.get_base_object() + + if base_establishment: + return qs.similar_wineries(base_establishment) + else: + return qs.none() class ArtisanProducerSimilarListView(EstablishmentSimilarView): @@ -108,9 +130,13 @@ class ArtisanProducerSimilarListView(EstablishmentSimilarView): def get_queryset(self): """Overridden get_queryset method""" - return EstablishmentMixinView.get_queryset(self) \ - .has_location() \ - .similar_artisans_producers(slug=self.kwargs.get('slug')) + qs = super(ArtisanProducerSimilarListView, self).get_queryset() + base_establishment = self.get_base_object() + + if base_establishment: + return qs.similar_artisans_producers(base_establishment) + else: + return qs.none() class EstablishmentTypeListView(generics.ListAPIView): From 7a5897e0bbed5a732ca1de2fa2b9710b76b0f4c4 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 20 Dec 2019 13:02:28 +0300 Subject: [PATCH 08/32] Add slash Add docs --- apps/account/urls/back.py | 2 +- apps/account/views/back.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/account/urls/back.py b/apps/account/urls/back.py index cf1d114e..a46b39bf 100644 --- a/apps/account/urls/back.py +++ b/apps/account/urls/back.py @@ -10,5 +10,5 @@ urlpatterns = [ path('user-role/', views.UserRoleLstView.as_view(), name='user-role-list-create'), path('user/', views.UserLstView.as_view(), name='user-create-list'), path('user//', views.UserRUDView.as_view(), name='user-rud'), - path('user//csv', views.get_user_csv, name='user-csv'), + path('user//csv/', views.get_user_csv, name='user-csv'), ] diff --git a/apps/account/views/back.py b/apps/account/views/back.py index 92dca84d..ded254dc 100644 --- a/apps/account/views/back.py +++ b/apps/account/views/back.py @@ -52,6 +52,7 @@ class UserRUDView(generics.RetrieveUpdateDestroyAPIView): def get_user_csv(request, id): + """User CSV file download""" # fields = ["id", "uuid", "nickname", "locale", "country_code", "city", "role", "consent_purpose", "consent_at", # "last_seen_at", "created_at", "updated_at", "email", "is_admin", "ezuser_id", "ez_user_id", # "encrypted_password", "reset_password_token", "reset_password_sent_at", "remember_created_at", From 80759c24e7a6832d8db342489b4f0905127dce19 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 20 Dec 2019 13:19:56 +0300 Subject: [PATCH 09/32] Fix number --- .../migrations/0032_auto_20191220_1019.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 apps/location/migrations/0032_auto_20191220_1019.py diff --git a/apps/location/migrations/0032_auto_20191220_1019.py b/apps/location/migrations/0032_auto_20191220_1019.py new file mode 100644 index 00000000..c4a1ba62 --- /dev/null +++ b/apps/location/migrations/0032_auto_20191220_1019.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-12-20 10:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0031_establishmentwineoriginaddress_wineoriginaddress'), + ] + + operations = [ + migrations.AlterField( + model_name='address', + name='number', + field=models.IntegerField(blank=True, default=0, verbose_name='number'), + ), + ] From 769a71b5eb41691b617593fbcebb4f292b177872 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 20 Dec 2019 13:52:20 +0300 Subject: [PATCH 10/32] Fix --- apps/location/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/location/models.py b/apps/location/models.py index be26e2c7..13607b12 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -185,7 +185,7 @@ class Address(models.Model): _('street name 1'), max_length=500, blank=True, default='') street_name_2 = models.CharField( _('street name 2'), max_length=500, blank=True, default='') - number = models.IntegerField(_('number')) + number = models.IntegerField(_('number'), blank=True, default=0) postal_code = models.CharField( _('postal code'), max_length=10, blank=True, default='', help_text=_('Ex.: 350018')) From 8e20424b78899379095fb9ddb854e863a561e999 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 14:32:01 +0300 Subject: [PATCH 11/32] bo creation & modification news info --- apps/news/serializers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/news/serializers.py b/apps/news/serializers.py index 6841b954..602e49a6 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -189,8 +189,12 @@ class NewsBackOfficeBaseSerializer(NewsBaseSerializer): 'must_of_the_week', 'publication_date', 'publication_time', + 'created', + 'modified', ) extra_kwargs = { + 'created': {'read_only': True}, + 'modified': {'read_only': True}, 'duplication_date': {'read_only': True}, 'locale_to_description_is_active': {'allow_null': False}, 'must_of_the_week': {'read_only': True}, From dbc928a2f5b4d41ba35da835a62552897b1320e6 Mon Sep 17 00:00:00 2001 From: dormantman Date: Fri, 20 Dec 2019 15:35:17 +0300 Subject: [PATCH 12/32] Fix collections back url --- apps/collection/urls/back.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/collection/urls/back.py b/apps/collection/urls/back.py index 6db5bde2..fc92f503 100644 --- a/apps/collection/urls/back.py +++ b/apps/collection/urls/back.py @@ -7,7 +7,7 @@ from collection.views import back as views app_name = 'collection' router = SimpleRouter() -router.register(r'collections', views.CollectionBackOfficeViewSet) +router.register(r'', views.CollectionBackOfficeViewSet) urlpatterns = [ path('guides/', views.GuideListCreateView.as_view(), From 0993f0a4516dbeee97d7970f483b83067c74965f Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 16:04:23 +0300 Subject: [PATCH 13/32] Revert "Fix collections back url" This reverts commit dbc928a --- apps/collection/urls/back.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/collection/urls/back.py b/apps/collection/urls/back.py index fc92f503..6db5bde2 100644 --- a/apps/collection/urls/back.py +++ b/apps/collection/urls/back.py @@ -7,7 +7,7 @@ from collection.views import back as views app_name = 'collection' router = SimpleRouter() -router.register(r'', views.CollectionBackOfficeViewSet) +router.register(r'collections', views.CollectionBackOfficeViewSet) urlpatterns = [ path('guides/', views.GuideListCreateView.as_view(), From 2499c0406da7e60808df2a451dc001f35097bd54 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 20 Dec 2019 16:19:04 +0300 Subject: [PATCH 14/32] news transfer serializer --- .../news/management/commands/add_news_tags.py | 3 +- apps/news/transfer_data.py | 24 +- apps/transfer/serializers/news.py | 213 +++++++++++------- 3 files changed, 144 insertions(+), 96 deletions(-) diff --git a/apps/news/management/commands/add_news_tags.py b/apps/news/management/commands/add_news_tags.py index 0ca2cb88..b4c5f8eb 100644 --- a/apps/news/management/commands/add_news_tags.py +++ b/apps/news/management/commands/add_news_tags.py @@ -6,7 +6,8 @@ from transfer.models import PageMetadata, Pages, PageTexts class Command(BaseCommand): - help = 'Remove old news from new bd' + help = 'Remove old news from new bd'\ + # TODO: изменить перенос тэгов по old_id новостей (они теперь от page) def handle(self, *args, **kwargs): count = 0 diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 33aeedfd..8596bcf7 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -24,21 +24,29 @@ class GroupConcat(Aggregate): def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name='News') - tag_cat, _ = TagCategory.objects.get_or_create(index_name='tag') - news_type.tag_categories.add(tag_cat) + tag_cat_tag, _ = TagCategory.objects.get_or_create(index_name='tag') + tag_cat_category, _ = TagCategory.objects.get_or_create(index_name='category') + news_type.tag_categories.add(tag_cat_tag) + news_type.tag_categories.add(tag_cat_category) news_type.save() queryset = PageTexts.objects.filter( page__type='News', ).annotate( - tag_cat_id=Value(tag_cat.id, output_field=IntegerField()), + page__id=F('page__id'), news_type_id=Value(news_type.id, output_field=IntegerField()), - country_code=F('page__site__country_code_2'), - news_title=F('page__root_title'), - image=F('page__attachment_suffix_url'), - template=F('page__template'), + page__created_at=F('page__created_at'), + page__account_id=F('page__account_id'), + page__state=F('page__state'), + page__template=F('page__template'), + page__site__country_code_2=F('page__site__country_code_2'), + page__root_title=F('page__root_title'), + page__attachment_suffix_url=F('page__attachment_suffix_url'), + page__published_at=F('page__published_at'), + tags=GroupConcat('page__tags__id'), - account_id=F('page__account_id'), + tag_cat_tag_id=Value(tag_cat_tag.id, output_field=IntegerField()), + tag_cat_category_id=Value(tag_cat_category.id, output_field=IntegerField()), ) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 4ef03184..15634ae7 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -11,63 +11,133 @@ from account.models import User class NewsSerializer(serializers.Serializer): - id = serializers.IntegerField() - account_id = serializers.IntegerField(allow_null=True) - tag_cat_id = serializers.IntegerField() - news_type_id = serializers.IntegerField() - news_title = serializers.CharField() - title = serializers.CharField() - summary = serializers.CharField(allow_null=True, allow_blank=True) - body = serializers.CharField(allow_null=True) - created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') - slug = serializers.CharField() - state = serializers.CharField() - template = serializers.CharField() - country_code = serializers.CharField(allow_null=True) + # old_id = page__id id -done + # news_type = 'News' создали или получили в трансфере -done + # title = {"en-GB":"some text"} из locale и title -done + # backoffice_title = page__root_title -done + # subtitle = {"en-GB":"some text"} из locale и summary -done + # description = {"en-GB":"some text"} из locale и body -done + # locale_to_description_is_active = {"en-GB": true, "fr-FR": false} из locale и true -done + # publication_date = DateField из page published_at -done ??? проверить + # publication_time = DateField из page published_at -done ??? проверить + # slugs = {"en-GB":"some slug"} из locale и slug -done + # state = page__state -done + # template = page__template -done + # country = по page__site__country_code_2 -done + # tags = по page__tags__id -progress -!!! + # gallery = в методе make_gallery из page__attachment_suffix_url -done + # created_by = page__account_id -done + # modified_by = page__account_id -done + # created = page created_at -done + locale = serializers.CharField() - image = serializers.CharField() + page__id = serializers.IntegerField() + news_type_id = serializers.IntegerField() + page__created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + page__account_id = serializers.IntegerField(allow_null=True) + page__state = serializers.CharField() + page__template = serializers.CharField() + page__site__country_code_2 = serializers.CharField(allow_null=True) + slug = serializers.CharField() + body = serializers.CharField(allow_null=True) + title = serializers.CharField() + page__root_title = serializers.CharField() + summary = serializers.CharField(allow_null=True, allow_blank=True) + page__attachment_suffix_url = serializers.CharField() + page__published_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S', allow_null=True) + tags = serializers.CharField(allow_null=True) + tag_cat_tag_id = serializers.IntegerField() + tag_cat_category_id = serializers.IntegerField() - def create(self, validated_data): - + def create(self, data): + account = self.get_account(data) payload = { - 'old_id': validated_data['id'], - 'news_type_id': validated_data['news_type_id'], - 'title': {validated_data['locale']: validated_data['news_title']}, - 'subtitle': self.get_subtitle(validated_data), - 'description': self.get_description(validated_data), - 'start': validated_data['created_at'], - 'slug': generate_unique_slug(News, validated_data['slug']), - 'state': self.get_state(validated_data), - 'template': self.get_template(validated_data), - 'country': self.get_country(validated_data), - 'created_by': self.get_account(validated_data), - 'modified_by': self.get_account(validated_data), + 'old_id': data['page__id'], + 'news_type_id': data['news_type_id'], + 'created': data['page__created_at'], + 'created_by': account, + 'modified_by': account, + 'state': self.get_state(data), + 'template': self.get_template(data), + 'country': self.get_country(data), + 'slug': {data['locale']: data['slug']}, + 'description': self.get_description(data), + 'title': {data['locale']: data['title']}, + 'backoffice_title': data['page__root_title'], + 'subtitle': self.get_subtitle(data), + 'locale_to_description_is_active': {data['locale']: True}, + 'publication_date': self.get_publication_date(data), + 'publication_time': self.get_publication_time(data), } - obj = News.objects.create(**payload) - tags = self.get_tags(validated_data) + obj, _ = News.objects.update_or_create( + old_id=data['old_id'], + defaults=payload, + ) + + tags = self.get_tags(data) for tag in tags: obj.tags.add(tag) obj.save() - self.make_gallery(validated_data, obj) + self.make_gallery(data, obj) return obj @staticmethod - def make_gallery(data, obj): - if not data['image'] or data['image'] == 'default/missing.png': - return + def get_publication_date(data): + published_at = data.get('page__published_at') + if published_at: + return published_at.date() + return None - img = Image.objects.create( - image=data['image'], - title=data['news_title'], - ) - NewsGallery.objects.create( - news=obj, - image=img, - is_main=True, - ) + @staticmethod + def get_publication_time(data): + published_at = data.get('page__published_at') + if published_at: + return published_at.time() + return None + + @staticmethod + def get_account(data): + return User.objects.filter(old_id=data['page__account_id']).first() + + @staticmethod + def get_state(data): + states = { + 'new': News.WAITING, + 'published': News.PUBLISHED, + 'hidden': News.HIDDEN, + 'published_exclusive': News.PUBLISHED_EXCLUSIVE, + 'scheduled_exclusively': News.WAITING, + } + return states.get(data['page__state'], News.WAITING) + + @staticmethod + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + 'newspaper': News.NEWSPAPER, + } + return templates.get(data['page__template'], News.MAIN) + + @staticmethod + def get_country(data): + return Country.objects.filter(code__iexact=data['page__site__country_code_2']).first() + + @staticmethod + def get_description(data): + if data['body']: + content = parse_legacy_news_content(data['body']) + return {data['locale']: content} + return None + + @staticmethod + def get_subtitle(data): + if data.get('summary'): + return {data['locale']: data['summary']} + return None @staticmethod def get_tags(data): @@ -78,7 +148,6 @@ class NewsSerializer(serializers.Serializer): meta_ids = (int(_id) for _id in data['tags'].split(',')) tags = PageMetadata.objects.filter( id__in=meta_ids, - key='tag', value__isnull=False, ) for old_tag in tags: @@ -90,48 +159,18 @@ class NewsSerializer(serializers.Serializer): return results @staticmethod - def get_description(data): - if data['body']: - content = parse_legacy_news_content(data['body']) - return {data['locale']: content} - return None + def make_gallery(data, obj): + if not data['page__attachment_suffix_url'] or data['page__attachment_suffix_url'] == 'default/missing.png': + return - @staticmethod - def get_state(data): - states = { - 'new': News.WAITING, - 'published': News.PUBLISHED, - 'hidden': News.HIDDEN, - 'published_exclusive': News.PUBLISHED_EXCLUSIVE, - 'scheduled_exclusively': News.WAITING, - } - return states.get(data['state'], News.WAITING) + img, _ = Image.objects.get_or_create( + image=data['page__attachment_suffix_url'], + title=data['page__root_title'], + created=data['page__created_at'] + ) - @staticmethod - def get_template(data): - templates = { - 'main': News.MAIN, - 'main.pdf.erb': News.MAIN_PDF_ERB, - } - return templates.get(data['template'], News.MAIN) - - @staticmethod - def get_country(data): - return Country.objects.filter(code__iexact=data['country_code']).first() - - @staticmethod - def get_title(data): - return {data['locale']: data['title']} - - @staticmethod - def get_subtitle(data): - if data.get('summary'): - content = {data['locale']: data['summary']} - else: - content = {data['locale']: data['title']} - return content - - @staticmethod - def get_account(data): - """Get account""" - return User.objects.filter(old_id=data['account_id']).first() + gal, _ = NewsGallery.objects.get_or_create( + news=obj, + image=img, + is_main=True, + ) From 0420fafabf8c8612c5943cc8b921bb0fee022ab7 Mon Sep 17 00:00:00 2001 From: dormantman Date: Fri, 20 Dec 2019 16:52:25 +0300 Subject: [PATCH 15/32] Added types to collection objects --- apps/collection/models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/collection/models.py b/apps/collection/models.py index 90837cd7..61e59788 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -118,22 +118,23 @@ class Collection(ProjectBaseMixin, CollectionDateMixin, instances = getattr(self, f'{related_object}') if instances.exists(): for instance in instances.all(): - raw_object = (instance.id, instance.slug) if hasattr(instance, 'slug') else ( - instance.id, None - ) + raw_object = (instance.id, instance.establishment_type.index_name, + instance.slug) if \ + hasattr(instance, 'slug') else (instance.id, None) raw_objects.append(raw_object) # parse slugs related_objects = [] object_names = set() re_pattern = r'[\w]+' - for object_id, raw_name, in raw_objects: + for object_id, object_type, raw_name, in raw_objects: result = re.findall(re_pattern, raw_name) if result: name = ' '.join(result).capitalize() if name not in object_names: related_objects.append({ 'id': object_id, + 'establishment_type': object_type, 'name': name }) object_names.add(name) From 2ca31cedfbb840ad044736e0d493b3f2b351014f Mon Sep 17 00:00:00 2001 From: dormantman Date: Fri, 20 Dec 2019 16:59:14 +0300 Subject: [PATCH 16/32] Added none condition to response --- apps/collection/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/collection/models.py b/apps/collection/models.py index 61e59788..e3ca63be 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -120,7 +120,7 @@ class Collection(ProjectBaseMixin, CollectionDateMixin, for instance in instances.all(): raw_object = (instance.id, instance.establishment_type.index_name, instance.slug) if \ - hasattr(instance, 'slug') else (instance.id, None) + hasattr(instance, 'slug') else (instance.id, None, None) raw_objects.append(raw_object) # parse slugs From 72e5d723dba8f8f6e80b2feea8817b12160bda75 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 20 Dec 2019 17:26:55 +0300 Subject: [PATCH 17/32] added default image for type/subtype --- .../commands/fill_artisan_default_image.py | 68 +++++++++++++++++++ .../migrations/0069_auto_20191220_1007.py | 25 +++++++ apps/establishment/models.py | 8 +++ apps/establishment/serializers/common.py | 7 +- apps/gallery/models.py | 4 +- .../migrations/0022_auto_20191220_1007.py | 25 +++++++ apps/product/models.py | 8 +++ apps/product/serializers/common.py | 6 ++ 8 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 apps/establishment/management/commands/fill_artisan_default_image.py create mode 100644 apps/establishment/migrations/0069_auto_20191220_1007.py create mode 100644 apps/product/migrations/0022_auto_20191220_1007.py diff --git a/apps/establishment/management/commands/fill_artisan_default_image.py b/apps/establishment/management/commands/fill_artisan_default_image.py new file mode 100644 index 00000000..1231ba76 --- /dev/null +++ b/apps/establishment/management/commands/fill_artisan_default_image.py @@ -0,0 +1,68 @@ +import boto3 +from django.conf import settings +from django.core.management.base import BaseCommand + +from establishment.models import EstablishmentSubType +from gallery.models import Image + + +class Command(BaseCommand): + help = """ + Fill establishment type by index names. + Steps: + 1 Upload default images into s3 bucket + 2 Run command ./manage.py fill_artisan_default_image + """ + + def add_arguments(self, parser): + parser.add_argument( + '--template_image_folder_name', + help='Template image folder in Amazon S3 bucket' + ) + + def handle(self, *args, **kwargs): + not_updated = 0 + template_image_folder_name = kwargs.get('template_image_folder_name') + if (template_image_folder_name and + hasattr(settings, 'AWS_ACCESS_KEY_ID') and + hasattr(settings, 'AWS_SECRET_ACCESS_KEY') and + hasattr(settings, 'AWS_STORAGE_BUCKET_NAME')): + to_update = [] + s3 = boto3.resource('s3') + s3_bucket = s3.Bucket(settings.AWS_STORAGE_BUCKET_NAME) + + for object_summary in s3_bucket.objects.filter(Prefix=f'media/{template_image_folder_name}/'): + uri_path = object_summary.key + filename = uri_path.split('/')[-1:][0] + if filename: + artisan_index_slice = filename.split('.')[:-1][0] \ + .split('_')[2:] + if len(artisan_index_slice) > 1: + artisan_index_name = '_'.join(artisan_index_slice) + else: + artisan_index_name = artisan_index_slice[0] + + attachment_suffix_url = f'{template_image_folder_name}/{filename}' + + # check artisan in db + artisan_qs = EstablishmentSubType.objects.filter(index_name__iexact=artisan_index_name, + establishment_type__index_name__iexact='artisan') + if artisan_qs.exists(): + artisan = artisan_qs.first() + image, created = Image.objects.get_or_create(image=attachment_suffix_url, + defaults={ + 'image': attachment_suffix_url, + 'orientation': Image.HORIZONTAL, + 'title': f'{artisan.__str__()} ' + f'{artisan.id} - ' + f'{attachment_suffix_url}'}) + if created: + # update artisan instance + artisan.default_image = image + to_update.append(artisan) + else: + not_updated += 1 + + EstablishmentSubType.objects.bulk_update(to_update, ['default_image', ]) + self.stdout.write(self.style.WARNING(f'Updated {len(to_update)} objects.')) + self.stdout.write(self.style.WARNING(f'Not updated {not_updated} objects.')) \ No newline at end of file diff --git a/apps/establishment/migrations/0069_auto_20191220_1007.py b/apps/establishment/migrations/0069_auto_20191220_1007.py new file mode 100644 index 00000000..6225c592 --- /dev/null +++ b/apps/establishment/migrations/0069_auto_20191220_1007.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.7 on 2019-12-20 10:07 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0007_auto_20191211_1528'), + ('establishment', '0068_auto_20191220_0914'), + ] + + operations = [ + migrations.AddField( + model_name='establishmentsubtype', + name='default_image', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='establishment_sub_types', to='gallery.Image', verbose_name='default image'), + ), + migrations.AddField( + model_name='establishmenttype', + name='default_image', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='establishment_types', to='gallery.Image', verbose_name='default image'), + ), + ] diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 2442d449..b4655b24 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -51,6 +51,10 @@ class EstablishmentType(TranslatedFieldsMixin, ProjectBaseMixin): tag_categories = models.ManyToManyField('tag.TagCategory', related_name='establishment_types', verbose_name=_('Tag')) + default_image = models.ForeignKey('gallery.Image', on_delete=models.SET_NULL, + related_name='establishment_types', + blank=True, null=True, default=None, + verbose_name='default image') class Meta: """Meta class.""" @@ -85,6 +89,10 @@ class EstablishmentSubType(ProjectBaseMixin, TranslatedFieldsMixin): tag_categories = models.ManyToManyField('tag.TagCategory', related_name='establishment_subtypes', verbose_name=_('Tag')) + default_image = models.ForeignKey('gallery.Image', on_delete=models.SET_NULL, + related_name='establishment_sub_types', + blank=True, null=True, default=None, + verbose_name='default image') objects = EstablishmentSubTypeManager() diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 19b4b764..1e7a153a 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -97,6 +97,8 @@ class MenuRUDSerializers(ProjectModelSerializer): class EstablishmentTypeBaseSerializer(serializers.ModelSerializer): """Serializer for EstablishmentType model.""" name_translated = TranslatedField() + default_image_url = serializers.ImageField(source='default_image.image', + allow_null=True) class Meta: """Meta class.""" @@ -107,6 +109,7 @@ class EstablishmentTypeBaseSerializer(serializers.ModelSerializer): 'name_translated', 'use_subtypes', 'index_name', + 'default_image_url', ] extra_kwargs = { 'name': {'write_only': True}, @@ -129,8 +132,9 @@ class EstablishmentTypeGeoSerializer(EstablishmentTypeBaseSerializer): class EstablishmentSubTypeBaseSerializer(serializers.ModelSerializer): """Serializer for EstablishmentSubType models.""" - name_translated = TranslatedField() + default_image_url = serializers.ImageField(source='default_image.image', + allow_null=True) class Meta: """Meta class.""" @@ -141,6 +145,7 @@ class EstablishmentSubTypeBaseSerializer(serializers.ModelSerializer): 'name_translated', 'establishment_type', 'index_name', + 'default_image_url', ] extra_kwargs = { 'name': {'write_only': True}, diff --git a/apps/gallery/models.py b/apps/gallery/models.py index 22c5b5e7..0cc8c60e 100644 --- a/apps/gallery/models.py +++ b/apps/gallery/models.py @@ -1,6 +1,6 @@ from django.db import models from django.utils.translation import gettext_lazy as _ -from sorl.thumbnail import delete +from sorl import thumbnail from sorl.thumbnail.fields import ImageField as SORLImageField from utils.methods import image_path @@ -47,7 +47,7 @@ class Image(ProjectBaseMixin, SORLImageMixin, PlatformMixin): """ try: # Delete from remote storage - delete(file_=self.image.file, delete_file=completely) + thumbnail.delete(file_=self.image.file, delete_file=completely) except FileNotFoundError: pass finally: diff --git a/apps/product/migrations/0022_auto_20191220_1007.py b/apps/product/migrations/0022_auto_20191220_1007.py new file mode 100644 index 00000000..c99b0e37 --- /dev/null +++ b/apps/product/migrations/0022_auto_20191220_1007.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2.7 on 2019-12-20 10:07 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0007_auto_20191211_1528'), + ('product', '0021_auto_20191212_0926'), + ] + + operations = [ + migrations.AddField( + model_name='productsubtype', + name='default_image', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='product_sub_types', to='gallery.Image', verbose_name='default image'), + ), + migrations.AddField( + model_name='producttype', + name='default_image', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='product_types', to='gallery.Image', verbose_name='default image'), + ), + ] diff --git a/apps/product/models.py b/apps/product/models.py index 7aeacdf2..3541122b 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -37,6 +37,10 @@ class ProductType(TranslatedFieldsMixin, ProjectBaseMixin): tag_categories = models.ManyToManyField('tag.TagCategory', related_name='product_types', verbose_name=_('Tag categories')) + default_image = models.ForeignKey('gallery.Image', on_delete=models.SET_NULL, + related_name='product_types', + blank=True, null=True, default=None, + verbose_name='default image') class Meta: """Meta class.""" @@ -62,6 +66,10 @@ class ProductSubType(TranslatedFieldsMixin, ProjectBaseMixin): verbose_name=_('Name'), help_text='{"en-GB":"some text"}') index_name = models.CharField(max_length=50, unique=True, db_index=True, verbose_name=_('Index name')) + default_image = models.ForeignKey('gallery.Image', on_delete=models.SET_NULL, + related_name='product_sub_types', + blank=True, null=True, default=None, + verbose_name='default image') class Meta: """Meta class.""" diff --git a/apps/product/serializers/common.py b/apps/product/serializers/common.py index 86344a36..f3678448 100644 --- a/apps/product/serializers/common.py +++ b/apps/product/serializers/common.py @@ -34,6 +34,8 @@ class ProductSubTypeBaseSerializer(serializers.ModelSerializer): name_translated = TranslatedField() index_name_display = serializers.CharField(source='get_index_name_display', read_only=True) + default_image_url = serializers.ImageField(source='default_image.image', + allow_null=True) class Meta: model = models.ProductSubType @@ -41,12 +43,15 @@ class ProductSubTypeBaseSerializer(serializers.ModelSerializer): 'id', 'name_translated', 'index_name_display', + 'default_image_url', ] class ProductTypeBaseSerializer(serializers.ModelSerializer): """ProductType base serializer""" name_translated = TranslatedField() + default_image_url = serializers.ImageField(source='default_image.image', + allow_null=True) class Meta: model = models.ProductType @@ -54,6 +59,7 @@ class ProductTypeBaseSerializer(serializers.ModelSerializer): 'id', 'name_translated', 'index_name', + 'default_image_url', ] From d1699e001efb7a81b9708e1bf82e0ced2eb70ee0 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 18:01:46 +0300 Subject: [PATCH 18/32] tag models translations in interface dictionary --- apps/tag/filters.py | 2 +- .../tag/migrations/0016_auto_20191220_1224.py | 46 +++++++++++++++++++ apps/tag/models.py | 12 +++-- apps/utils/models.py | 20 ++++++++ 4 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 apps/tag/migrations/0016_auto_20191220_1224.py diff --git a/apps/tag/filters.py b/apps/tag/filters.py index 46470ca1..29b623a9 100644 --- a/apps/tag/filters.py +++ b/apps/tag/filters.py @@ -51,7 +51,7 @@ class TagCategoryFilterSet(TagsBaseFilterSet): # todo: filter by establishment type def by_establishment_type(self, queryset, name, value): if value == EstablishmentType.ARTISAN: - qs = models.TagCategory.objects.filter(index_name='shop_category') + qs = models.TagCategory.objects.with_base_related().filter(index_name='shop_category') else: qs = queryset.by_establishment_type(value) return qs diff --git a/apps/tag/migrations/0016_auto_20191220_1224.py b/apps/tag/migrations/0016_auto_20191220_1224.py new file mode 100644 index 00000000..3d70ecb3 --- /dev/null +++ b/apps/tag/migrations/0016_auto_20191220_1224.py @@ -0,0 +1,46 @@ +# Generated by Django 2.2.7 on 2019-12-20 12:24 + +from django.db import migrations, models +import django.db.models.deletion + + +def fill_translations(apps, schemaeditor): + Tag = apps.get_model('tag', 'Tag') + TagCategory = apps.get_model('tag', 'TagCategory') + SiteInterfaceDictionary = apps.get_model('translation', 'SiteInterfaceDictionary') + + for tag_category in TagCategory.objects.all(): + if tag_category.label: + t = SiteInterfaceDictionary(text=tag_category.label) + t.save() + tag_category.translation = t + tag_category.save() + + for tag in Tag.objects.all(): + if tag.label: + t = SiteInterfaceDictionary(text=tag.label) + t.save() + tag.translation = t + tag.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('translation', '0007_language_is_active'), + ('tag', '0015_auto_20191118_1210'), + ] + + operations = [ + migrations.AddField( + model_name='tag', + name='translation', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tag', to='translation.SiteInterfaceDictionary', verbose_name='Translation'), + ), + migrations.AddField( + model_name='tagcategory', + name='translation', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tag_category', to='translation.SiteInterfaceDictionary', verbose_name='Translation'), + ), + migrations.RunPython(fill_translations, migrations.RunPython.noop) + ] diff --git a/apps/tag/models.py b/apps/tag/models.py index b718d83c..973cc326 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -5,7 +5,7 @@ from django.utils.translation import gettext_lazy as _ from configuration.models import TranslationSettings from location.models import Country -from utils.models import TJSONField, TranslatedFieldsMixin +from utils.models import TJSONField, TagsTranslationModelMixin class TagQuerySet(models.QuerySet): @@ -29,7 +29,7 @@ class TagQuerySet(models.QuerySet): return self.filter(category__establishment_types__index_name=index_name) -class Tag(TranslatedFieldsMixin, models.Model): +class Tag(models.Model, TagsTranslationModelMixin): """Tag model.""" label = TJSONField(blank=True, null=True, default=None, @@ -48,6 +48,8 @@ class Tag(TranslatedFieldsMixin, models.Model): old_id_meta_product = models.PositiveIntegerField(_('old id metadata product'), blank=True, null=True, default=None) + translation = models.ForeignKey('translation.SiteInterfaceDictionary', on_delete=models.SET_NULL, + null=True, related_name='tag', verbose_name=_('Translation')) objects = TagQuerySet.as_manager() @@ -88,7 +90,7 @@ class TagCategoryQuerySet(models.QuerySet): def with_base_related(self): """Select related objects.""" - return self.prefetch_related('tags') + return self.prefetch_related('tags', 'tags__translation').select_related('translation') def with_extended_related(self): """Select related objects.""" @@ -119,7 +121,7 @@ class TagCategoryQuerySet(models.QuerySet): return self.exclude(tags__isnull=switcher) -class TagCategory(TranslatedFieldsMixin, models.Model): +class TagCategory(models.Model, TagsTranslationModelMixin): """Tag base category model.""" STRING = 'string' @@ -151,6 +153,8 @@ class TagCategory(TranslatedFieldsMixin, models.Model): value_type = models.CharField(_('value type'), max_length=255, choices=VALUE_TYPE_CHOICES, default=LIST, ) old_id = models.IntegerField(blank=True, null=True) + translation = models.OneToOneField('translation.SiteInterfaceDictionary', on_delete=models.SET_NULL, + null=True, related_name='tag_category', verbose_name=_('Translation')) objects = TagCategoryQuerySet.as_manager() diff --git a/apps/utils/models.py b/apps/utils/models.py index 07891330..910b8473 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -88,6 +88,18 @@ def translate_field(self, field_name, toggle_field_name=None): return None return translate +def translate_by_relation(self): + def translate(self): + field = self.translation.text if self.translation else None + if not isinstance(field, dict): + return None + try: + return field.get(to_locale(get_language()), + field.get(get_default_locale(), + next(iter(field.values())))) + except StopIteration: + return None + return translate # todo: refactor this class IndexJSON: @@ -135,6 +147,14 @@ class TranslatedFieldsMixin: return value if value else super(TranslatedFieldsMixin, self).__str__() +class TagsTranslationModelMixin: + def __init__(self, *args, **kwargs): + super(TagsTranslationModelMixin, self).__init__(*args, **kwargs) + setattr(self.__class__, 'label_translated', + property(translate_by_relation(self))) + setattr(self.__class__, 'label_indexing', + property(lambda self: self.translation.text)) + class OAuthProjectMixin: """OAuth2 mixin for project GM""" From 660c8e791b5c8bb3a4f2d9d0b34618585d26c4b1 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 19:08:53 +0300 Subject: [PATCH 19/32] tags translation --- apps/establishment/models.py | 2 +- apps/news/models.py | 2 +- apps/product/models.py | 2 +- apps/tag/models.py | 14 ++++++-- apps/tag/serializers.py | 64 +++++++++++++++++++----------------- apps/utils/models.py | 24 ++------------ project/settings/local.py | 10 +++--- 7 files changed, 55 insertions(+), 63 deletions(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index b4655b24..cde60618 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -113,7 +113,7 @@ class EstablishmentQuerySet(models.QuerySet): def with_base_related(self): """Return qs with related objects.""" return self.select_related('address', 'establishment_type'). \ - prefetch_related('tags') + prefetch_related('tags', 'tags__translation') def with_schedule(self): """Return qs with related schedule.""" diff --git a/apps/news/models.py b/apps/news/models.py index ab65ed88..64b22ffb 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -74,7 +74,7 @@ class NewsQuerySet(TranslationQuerysetMixin): def with_base_related(self): """Return qs with related objects.""" - return self.select_related('news_type', 'country').prefetch_related('tags') + return self.select_related('news_type', 'country').prefetch_related('tags', 'tags__translation') def with_extended_related(self): """Return qs with related objects.""" diff --git a/apps/product/models.py b/apps/product/models.py index 3541122b..2c2081c9 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -91,7 +91,7 @@ class ProductQuerySet(models.QuerySet): def with_base_related(self): return self.select_related('product_type', 'establishment') \ - .prefetch_related('product_type__subtypes') + .prefetch_related('product_type__subtypes', 'tags', 'tags__translation') def with_extended_related(self): """Returns qs with almost all related objects.""" diff --git a/apps/tag/models.py b/apps/tag/models.py index 973cc326..7685cfd0 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -5,7 +5,7 @@ from django.utils.translation import gettext_lazy as _ from configuration.models import TranslationSettings from location.models import Country -from utils.models import TJSONField, TagsTranslationModelMixin +from utils.models import TJSONField class TagQuerySet(models.QuerySet): @@ -29,7 +29,7 @@ class TagQuerySet(models.QuerySet): return self.filter(category__establishment_types__index_name=index_name) -class Tag(models.Model, TagsTranslationModelMixin): +class Tag(models.Model): """Tag model.""" label = TJSONField(blank=True, null=True, default=None, @@ -51,6 +51,10 @@ class Tag(models.Model, TagsTranslationModelMixin): translation = models.ForeignKey('translation.SiteInterfaceDictionary', on_delete=models.SET_NULL, null=True, related_name='tag', verbose_name=_('Translation')) + @property + def label_indexing(self): + return self.translation.text + objects = TagQuerySet.as_manager() class Meta: @@ -121,7 +125,7 @@ class TagCategoryQuerySet(models.QuerySet): return self.exclude(tags__isnull=switcher) -class TagCategory(models.Model, TagsTranslationModelMixin): +class TagCategory(models.Model): """Tag base category model.""" STRING = 'string' @@ -156,6 +160,10 @@ class TagCategory(models.Model, TagsTranslationModelMixin): translation = models.OneToOneField('translation.SiteInterfaceDictionary', on_delete=models.SET_NULL, null=True, related_name='tag_category', verbose_name=_('Translation')) + @property + def label_indexing(self): + return self.translation.text + objects = TagCategoryQuerySet.as_manager() class Meta: diff --git a/apps/tag/serializers.py b/apps/tag/serializers.py index b5e5a267..2155de73 100644 --- a/apps/tag/serializers.py +++ b/apps/tag/serializers.py @@ -2,15 +2,25 @@ from rest_framework import serializers from rest_framework.fields import SerializerMethodField -from establishment.models import Establishment -from establishment.models import EstablishmentType +from establishment.models import Establishment, EstablishmentType from news.models import News from news.models import NewsType from tag import models -from utils.exceptions import BindingObjectNotFound -from utils.exceptions import ObjectAlreadyAdded -from utils.exceptions import RemovedBindingObjectNotFound +from utils.exceptions import BindingObjectNotFound, ObjectAlreadyAdded, RemovedBindingObjectNotFound from utils.serializers import TranslatedField +from utils.models import get_default_locale, get_language, to_locale + + +def translate_obj(obj): + if not obj.translation or not isinstance(obj.translation.text, dict): + return None + try: + field = obj.translation.text + return field.get(to_locale(get_language()), + field.get(get_default_locale(), + next(iter(field.values())))) + except StopIteration: + return None class TagBaseSerializer(serializers.ModelSerializer): @@ -19,8 +29,11 @@ class TagBaseSerializer(serializers.ModelSerializer): def get_extra_kwargs(self): return super().get_extra_kwargs() - label_translated = TranslatedField() index_name = serializers.CharField(source='value', read_only=True, allow_null=True) + label_translated = serializers.SerializerMethodField(read_only=True, allow_null=True) + + def get_label_translated(self, obj): + return translate_obj(obj) class Meta: """Meta class.""" @@ -47,8 +60,10 @@ class TagBackOfficeSerializer(TagBaseSerializer): class TagCategoryProductSerializer(serializers.ModelSerializer): """SHORT Serializer for TagCategory""" + label_translated = serializers.SerializerMethodField(read_only=True, allow_null=True) - label_translated = TranslatedField() + def get_label_translated(self, obj): + return translate_obj(obj) class Meta: """Meta class.""" @@ -56,7 +71,6 @@ class TagCategoryProductSerializer(serializers.ModelSerializer): model = models.TagCategory fields = ( 'id', - 'label_translated', 'index_name', ) @@ -64,8 +78,8 @@ class TagCategoryProductSerializer(serializers.ModelSerializer): class TagCategoryBaseSerializer(serializers.ModelSerializer): """Serializer for model TagCategory.""" - label_translated = TranslatedField() - tags = SerializerMethodField() + tags = TagBaseSerializer(many=True, allow_null=True) + label_translated = serializers.SerializerMethodField(read_only=True, allow_null=True) class Meta: """Meta class.""" @@ -78,33 +92,17 @@ class TagCategoryBaseSerializer(serializers.ModelSerializer): 'tags', ) - def get_tags(self, obj): - query_params = dict(self.context['request'].query_params) - - if len(query_params) > 1: - return [] - - params = {} - if 'establishment_type' in query_params: - params = { - 'establishments__isnull': False, - } - elif 'product_type' in query_params: - params = { - 'products__isnull': False, - } - - tags = obj.tags.filter(**params).distinct() - return TagBaseSerializer(instance=tags, many=True, read_only=True).data + def get_label_translated(self, obj): + return translate_obj(obj) class FiltersTagCategoryBaseSerializer(serializers.ModelSerializer): """Serializer for model TagCategory.""" - label_translated = TranslatedField() filters = SerializerMethodField() param_name = SerializerMethodField() type = SerializerMethodField() + label_translated = serializers.SerializerMethodField(allow_null=True, read_only=True) class Meta: """Meta class.""" @@ -127,6 +125,9 @@ class FiltersTagCategoryBaseSerializer(serializers.ModelSerializer): return 'wine_colors_id__in' return 'tags_id__in' + def get_label_translated(self, obj): + return translate_obj(obj) + def get_fields(self, *args, **kwargs): fields = super(FiltersTagCategoryBaseSerializer, self).get_fields() @@ -157,10 +158,13 @@ class FiltersTagCategoryBaseSerializer(serializers.ModelSerializer): class TagCategoryShortSerializer(serializers.ModelSerializer): """Serializer for model TagCategory.""" - label_translated = TranslatedField() + label_translated = serializers.SerializerMethodField(allow_null=True, read_only=True) value_type_display = serializers.CharField(source='get_value_type_display', read_only=True) + def get_label_translated(self, obj): + return translate_obj(obj) + class Meta(TagCategoryBaseSerializer.Meta): """Meta class.""" fields = [ diff --git a/apps/utils/models.py b/apps/utils/models.py index 910b8473..42e35bb1 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -88,19 +88,6 @@ def translate_field(self, field_name, toggle_field_name=None): return None return translate -def translate_by_relation(self): - def translate(self): - field = self.translation.text if self.translation else None - if not isinstance(field, dict): - return None - try: - return field.get(to_locale(get_language()), - field.get(get_default_locale(), - next(iter(field.values())))) - except StopIteration: - return None - return translate - # todo: refactor this class IndexJSON: @@ -147,14 +134,6 @@ class TranslatedFieldsMixin: return value if value else super(TranslatedFieldsMixin, self).__str__() -class TagsTranslationModelMixin: - def __init__(self, *args, **kwargs): - super(TagsTranslationModelMixin, self).__init__(*args, **kwargs) - setattr(self.__class__, 'label_translated', - property(translate_by_relation(self))) - setattr(self.__class__, 'label_indexing', - property(lambda self: self.translation.text)) - class OAuthProjectMixin: """OAuth2 mixin for project GM""" @@ -463,7 +442,8 @@ class HasTagsMixin(models.Model): @property def visible_tags(self): - return self.tags.filter(category__public=True).prefetch_related('category')\ + return self.tags.filter(category__public=True).prefetch_related('category', + 'translation', 'category__translation')\ .exclude(category__value_type='bool') class Meta: diff --git a/project/settings/local.py b/project/settings/local.py index d9c7cab8..b101d78e 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -86,11 +86,11 @@ LOGGING = { 'py.warnings': { 'handlers': ['console'], }, - # 'django.db.backends': { - # 'handlers': ['console', ], - # 'level': 'DEBUG', - # 'propagate': False, - # }, + 'django.db.backends': { + 'handlers': ['console', ], + 'level': 'DEBUG', + 'propagate': False, + }, } } From dd07769b9b72b58c7da77fc607d568f3e0b3625b Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 19:25:14 +0300 Subject: [PATCH 20/32] remove label fields from tags --- .../tag/migrations/0017_auto_20191220_1623.py | 21 +++++++++++++++++++ apps/tag/models.py | 7 ------- 2 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 apps/tag/migrations/0017_auto_20191220_1623.py diff --git a/apps/tag/migrations/0017_auto_20191220_1623.py b/apps/tag/migrations/0017_auto_20191220_1623.py new file mode 100644 index 00000000..f36f0a55 --- /dev/null +++ b/apps/tag/migrations/0017_auto_20191220_1623.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.7 on 2019-12-20 16:23 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0016_auto_20191220_1224'), + ] + + operations = [ + migrations.RemoveField( + model_name='tag', + name='label', + ), + migrations.RemoveField( + model_name='tagcategory', + name='label', + ), + ] diff --git a/apps/tag/models.py b/apps/tag/models.py index 7685cfd0..e250e474 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -32,9 +32,6 @@ class TagQuerySet(models.QuerySet): class Tag(models.Model): """Tag model.""" - label = TJSONField(blank=True, null=True, default=None, - verbose_name=_('label'), - help_text='{"en-GB":"some text"}') value = models.CharField(_('indexing name'), max_length=255, blank=True, db_index=True, null=True, default=None) category = models.ForeignKey('TagCategory', on_delete=models.CASCADE, @@ -143,10 +140,6 @@ class TagCategory(models.Model): (PERCENTAGE, _('percentage')), (BOOLEAN, _('boolean')), ) - - label = TJSONField(blank=True, null=True, default=None, - verbose_name=_('label'), - help_text='{"en-GB":"some text"}') country = models.ForeignKey('location.Country', on_delete=models.SET_NULL, null=True, default=None) From ee8d133248fdf26358df5ac88525999514cab65d Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 20:11:58 +0300 Subject: [PATCH 21/32] Example for tags creation --- apps/transfer/serializers/tag.py | 6 ++++- apps/translation/models.py | 40 +++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/apps/transfer/serializers/tag.py b/apps/transfer/serializers/tag.py index c47ffafc..4f8ae862 100644 --- a/apps/transfer/serializers/tag.py +++ b/apps/transfer/serializers/tag.py @@ -3,6 +3,7 @@ from django.utils.text import slugify from rest_framework import serializers from tag.models import Tag +from translation.models import SiteInterfaceDictionary from transfer.mixins import TransferSerializerMixin from transfer.models import Cepages @@ -36,8 +37,11 @@ class AssemblageTagSerializer(TransferSerializerMixin): def create(self, validated_data): qs = self.Meta.model.objects.filter(**validated_data) category = validated_data.get('category') + translations = validated_data.pop('label') if not qs.exists() and category: - return super().create(validated_data) + instance = super().create(validated_data) + SiteInterfaceDictionary.objects.update_or_create_for_tag(instance, translations) + return instance def get_tag_value(self, cepage, percent): if cepage and percent: diff --git a/apps/translation/models.py b/apps/translation/models.py index 1d9695fe..7b64dce0 100644 --- a/apps/translation/models.py +++ b/apps/translation/models.py @@ -2,9 +2,9 @@ from django.contrib.postgres.fields import JSONField from django.db import models from django.utils.translation import gettext_lazy as _ +from django.apps import apps from utils.models import ProjectBaseMixin, LocaleManagerMixin - class LanguageQuerySet(models.QuerySet): """QuerySet for model Language""" @@ -50,6 +50,44 @@ class Language(models.Model): class SiteInterfaceDictionaryManager(LocaleManagerMixin): """Extended manager for SiteInterfaceDictionary model.""" + def update_or_create_for_tag(self, tag, translations: dict): + Tag = apps.get_model('tag', 'Tag') + """Creates or updates translation for EXISTING in DB Tag""" + if not tag.pk or not isinstance(tag, Tag): + raise NotImplementedError + if tag.translation: + tag.translation.text = translations + tag.translation.page = 'tag' + tag.translation.keywords = f'tag-{tag.pk}' + else: + trans = SiteInterfaceDictionary({ + 'text': translations, + 'page': 'tag', + 'keywords': f'tag-{tag.pk}' + }) + trans.save() + tag.translation = trans + tag.save() + + def update_or_create_for_tag_category(self, tag_category, translations: dict): + """Creates or updates translation for EXISTING in DB TagCategory""" + TagCategory = apps.get_model('tag', 'TagCategory') + if not tag_category.pk or not isinstance(tag_category, TagCategory): + raise NotImplementedError + if tag_category.translation: + tag_category.translation.text = translations + tag_category.translation.page = 'tag' + tag_category.translation.keywords = f'tag_category-{tag_category.pk}' + else: + trans = SiteInterfaceDictionary({ + 'text': translations, + 'page': 'tag', + 'keywords': f'tag_category-{tag_category.pk}' + }) + trans.save() + tag_category.translation = trans + tag_category.save() + class SiteInterfaceDictionary(ProjectBaseMixin): """Site interface dictionary model.""" From 0f0a3f957796d471e3431b5560576cf1fcc49ae2 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 21:10:28 +0300 Subject: [PATCH 22/32] hardcoded tags --- apps/tag/filters.py | 4 ++-- project/settings/base.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/tag/filters.py b/apps/tag/filters.py index 29b623a9..5b0ba43e 100644 --- a/apps/tag/filters.py +++ b/apps/tag/filters.py @@ -73,10 +73,10 @@ class TagsFilterSet(TagsBaseFilterSet): def by_establishment_type(self, queryset, name, value): if value == EstablishmentType.ARTISAN: - qs = models.Tag.objects.by_category_index_name('shop_category') + qs = models.Tag.objects.filter(index_name__in=settings.ARTISANS_CHOSEN_TAGS) if self.request.country_code and self.request.country_code not in settings.INTERNATIONAL_COUNTRY_CODES: qs = qs.filter(establishments__address__city__country__code=self.request.country_code).distinct('id') - return qs.exclude(establishments__isnull=True)[0:8] + return qs.exclude(establishments__isnull=True) return queryset.by_establishment_type(value) # TMP TODO remove it later diff --git a/project/settings/base.py b/project/settings/base.py index 2dfe1ba1..a7d3274f 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -516,8 +516,12 @@ PHONENUMBER_DEFAULT_REGION = "FR" FALLBACK_LOCALE = 'en-GB' -ESTABLISHMENT_CHOSEN_TAGS = ['gastronomic', 'en_vogue', 'terrace', 'streetfood', 'business', 'bar_cocktail', 'brunch', 'pop'] +ESTABLISHMENT_CHOSEN_TAGS = ['gastronomic', 'terrace', 'streetfood', 'business', 'bar_cocktail', 'brunch', 'pop'] NEWS_CHOSEN_TAGS = ['eat', 'drink', 'cook', 'style', 'international', 'event', 'partnership'] +ARTISANS_CHOSEN_TAGS = ['butchery', 'bakery', 'patisserie', 'cheese_shop', 'fish_shop', 'ice-cream_maker', + 'wine_merchant', 'coffe_shop'] +RECIPES_CHOSEN_TAGS = ['cook', 'eat', 'drink'] + INTERNATIONAL_COUNTRY_CODES = ['www', 'main', 'next'] THUMBNAIL_ENGINE = 'utils.thumbnail_engine.GMEngine' From 1771b02afdea601a94dda6a2f48e701a173f974e Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 21:57:46 +0300 Subject: [PATCH 23/32] fix tags indexing --- apps/tag/models.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/tag/models.py b/apps/tag/models.py index e250e474..28ff1183 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -5,7 +5,8 @@ from django.utils.translation import gettext_lazy as _ from configuration.models import TranslationSettings from location.models import Country -from utils.models import TJSONField +from elasticsearch_dsl.utils import AttrDict +from search_indexes.utils import OBJECT_FIELD_PROPERTIES class TagQuerySet(models.QuerySet): @@ -50,7 +51,9 @@ class Tag(models.Model): @property def label_indexing(self): - return self.translation.text + base_dict = self.translation.text if self.translation and isinstance(self.translation.text, dict) else {} + dict_to_index = {locale: base_dict.get(locale) for locale in OBJECT_FIELD_PROPERTIES} + return AttrDict(dict_to_index) objects = TagQuerySet.as_manager() @@ -155,7 +158,9 @@ class TagCategory(models.Model): @property def label_indexing(self): - return self.translation.text + base_dict = self.translation.text if self.translation and isinstance(self.translation.text, dict) else {} + dict_to_index = {locale: base_dict.get(locale) for locale in OBJECT_FIELD_PROPERTIES} + return AttrDict(dict_to_index) objects = TagCategoryQuerySet.as_manager() From 35673f9ad02ad633835be895663f3f9506284a53 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 21:59:49 +0300 Subject: [PATCH 24/32] fix tags indexing #2 --- apps/tag/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/tag/models.py b/apps/tag/models.py index 28ff1183..a3acbc8e 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -159,7 +159,7 @@ class TagCategory(models.Model): @property def label_indexing(self): base_dict = self.translation.text if self.translation and isinstance(self.translation.text, dict) else {} - dict_to_index = {locale: base_dict.get(locale) for locale in OBJECT_FIELD_PROPERTIES} + dict_to_index = {locale: base_dict.get(locale) for locale in OBJECT_FIELD_PROPERTIES.keys()} return AttrDict(dict_to_index) objects = TagCategoryQuerySet.as_manager() From d97972a3ef3d4f8c20420c74a4a50f9bb1133c14 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 22:07:19 +0300 Subject: [PATCH 25/32] fix tags indexing (final) --- apps/search_indexes/documents/establishment.py | 2 ++ apps/tag/models.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 8ae26097..6f9f1abf 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -143,6 +143,8 @@ class EstablishmentDocument(Document): 'weekday_display': fields.KeywordField(attr='get_weekday_display'), 'closed_at': fields.KeywordField(attr='closed_at_str'), 'opening_at': fields.KeywordField(attr='opening_at_str'), + # 'closed_at_datetime': fields.DateField(attr='closed_at'), + # 'opening_at_datetime': fields.DateField(attr='opening_at'), } )) address = fields.ObjectField( diff --git a/apps/tag/models.py b/apps/tag/models.py index a3acbc8e..4d0ab43a 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -5,8 +5,7 @@ from django.utils.translation import gettext_lazy as _ from configuration.models import TranslationSettings from location.models import Country -from elasticsearch_dsl.utils import AttrDict -from search_indexes.utils import OBJECT_FIELD_PROPERTIES +from utils.models import IndexJSON class TagQuerySet(models.QuerySet): @@ -52,8 +51,10 @@ class Tag(models.Model): @property def label_indexing(self): base_dict = self.translation.text if self.translation and isinstance(self.translation.text, dict) else {} - dict_to_index = {locale: base_dict.get(locale) for locale in OBJECT_FIELD_PROPERTIES} - return AttrDict(dict_to_index) + index = IndexJSON() + for k, v in base_dict.items(): + setattr(index, k, v) + return index objects = TagQuerySet.as_manager() @@ -159,8 +160,10 @@ class TagCategory(models.Model): @property def label_indexing(self): base_dict = self.translation.text if self.translation and isinstance(self.translation.text, dict) else {} - dict_to_index = {locale: base_dict.get(locale) for locale in OBJECT_FIELD_PROPERTIES.keys()} - return AttrDict(dict_to_index) + index = IndexJSON() + for k, v in base_dict.items(): + setattr(index, k, v) + return index objects = TagCategoryQuerySet.as_manager() From 1247b4c5777ca3a384a05b8c87a0af1518378640 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Fri, 20 Dec 2019 22:28:57 +0300 Subject: [PATCH 26/32] indexing working time --- apps/search_indexes/documents/establishment.py | 4 ++-- apps/timetable/models.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 6f9f1abf..e9761393 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -143,8 +143,8 @@ class EstablishmentDocument(Document): 'weekday_display': fields.KeywordField(attr='get_weekday_display'), 'closed_at': fields.KeywordField(attr='closed_at_str'), 'opening_at': fields.KeywordField(attr='opening_at_str'), - # 'closed_at_datetime': fields.DateField(attr='closed_at'), - # 'opening_at_datetime': fields.DateField(attr='opening_at'), + 'closed_at_indexing': fields.DateField(), + 'opening_at_indexing': fields.DateField(), } )) address = fields.ObjectField( diff --git a/apps/timetable/models.py b/apps/timetable/models.py index 07e52807..c9295b3b 100644 --- a/apps/timetable/models.py +++ b/apps/timetable/models.py @@ -1,6 +1,6 @@ from django.db import models from django.utils.translation import gettext_lazy as _ -from datetime import time +from datetime import time, datetime from utils.models import ProjectBaseMixin @@ -59,6 +59,14 @@ class Timetable(ProjectBaseMixin): def opening_at_str(self): return str(self.opening_at) if self.opening_at else None + @property + def closed_at_indexing(self): + return datetime.combine(time=self.closed_at, date=datetime(1970, 1, 1 + self.weekday).date()) if self.closed_at else None + + @property + def opening_at_indexing(self): + return datetime.combine(time=self.opening_at, date=datetime(1970, 1, 1 + self.weekday).date()) if self.opening_at else None + @property def opening_time(self): return self.opening_at or self.lunch_start or self.dinner_start From f010c974330631e57825cd4b134358592d9f3a7d Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sun, 22 Dec 2019 20:33:51 +0300 Subject: [PATCH 27/32] add export for panel --- apps/main/models.py | 13 +++++ apps/main/tasks.py | 14 +++++ apps/main/urls/back.py | 5 +- apps/main/views/back.py | 36 ++++++++++++- apps/utils/export.py | 115 ++++++++++++++++++++++++++++++++++++++++ requirements/base.txt | 3 ++ 6 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 apps/main/tasks.py create mode 100644 apps/utils/export.py diff --git a/apps/main/models.py b/apps/main/models.py index 83109f40..d8860aee 100644 --- a/apps/main/models.py +++ b/apps/main/models.py @@ -481,6 +481,19 @@ class Panel(ProjectBaseMixin): columns = [col[0] for col in cursor.description] return columns + def get_headers(self): + with connections['default'].cursor() as cursor: + try: + cursor.execute(self.query) + except Exception as er: + raise UnprocessableEntityError() + return self._raw_columns(cursor) + + def get_data(self): + with connections['default'].cursor() as cursor: + cursor.execute(self.query) + return cursor.fetchall() + def _raw_page(self, raw, request): page = request.query_params.get('page', 0) page_size = request.query_params.get('page_size', 0) diff --git a/apps/main/tasks.py b/apps/main/tasks.py new file mode 100644 index 00000000..0231b83f --- /dev/null +++ b/apps/main/tasks.py @@ -0,0 +1,14 @@ +"""Task methods for main app.""" + +from celery import shared_task + +from account.models import User +from main.models import Panel +from utils.export import SendExport + + +@shared_task +def send_export_to_email(panel_id, user_id, file_type='csv'): + panel = Panel.objects.get(id=panel_id) + user = User.objects.get(id=user_id) + SendExport(user, panel, file_type).send() diff --git a/apps/main/urls/back.py b/apps/main/urls/back.py index a2049a42..3d39f008 100644 --- a/apps/main/urls/back.py +++ b/apps/main/urls/back.py @@ -24,8 +24,9 @@ urlpatterns = [ name='page-types-list-create'), path('panels/', views.PanelsListCreateView.as_view(), name='panels'), path('panels//', views.PanelsRUDView.as_view(), name='panels-rud'), - path('panels//execute/', views.PanelsExecuteView.as_view(), name='panels-execute') - + path('panels//execute/', views.PanelsExecuteView.as_view(), name='panels-execute'), + path('panels//csv/', views.PanelsExportCSVView.as_view(), name='panels-csv'), + path('panels//xls/', views.PanelsExecuteXLSView.as_view(), name='panels-xls') ] diff --git a/apps/main/views/back.py b/apps/main/views/back.py index 98398f17..e819b71d 100644 --- a/apps/main/views/back.py +++ b/apps/main/views/back.py @@ -1,10 +1,12 @@ from django.contrib.contenttypes.models import ContentType +from django.utils.translation import gettext_lazy as _ from django_filters.rest_framework import DjangoFilterBackend -from rest_framework import generics, permissions +from rest_framework import generics, permissions, status from rest_framework.generics import get_object_or_404 from rest_framework.response import Response from main import serializers +from main import tasks from main.filters import AwardFilter from main.models import Award, Footer, PageType, Panel from main.views import SiteSettingsView, SiteListView @@ -121,3 +123,35 @@ class PanelsExecuteView(generics.ListAPIView): def list(self, request, *args, **kwargs): panel = get_object_or_404(Panel, id=self.kwargs['pk']) return Response(panel.execute_query(request)) + + +class PanelsExportCSVView(PanelsExecuteView): + """Export panels via csv view.""" + permission_classes = (permissions.IsAdminUser,) + queryset = Panel.objects.all() + + def list(self, request, *args, **kwargs): + panel = get_object_or_404(Panel, id=self.kwargs['pk']) + # make task for celery + tasks.send_export_to_email.delay( + panel_id=panel.id, user_id=request.user.id) + return Response( + {"success": _('The file will be sent to your email.')}, + status=status.HTTP_200_OK + ) + + +class PanelsExecuteXLSView(PanelsExecuteView): + """Export panels via xlsx view.""" + permission_classes = (permissions.IsAdminUser,) + queryset = Panel.objects.all() + + def list(self, request, *args, **kwargs): + panel = get_object_or_404(Panel, id=self.kwargs['pk']) + # make task for celery + tasks.send_export_to_email.delay( + panel_id=panel.id, user_id=request.user.id, file_type='xls') + return Response( + {"success": _('The file will be sent to your email.')}, + status=status.HTTP_200_OK + ) diff --git a/apps/utils/export.py b/apps/utils/export.py new file mode 100644 index 00000000..e4756b09 --- /dev/null +++ b/apps/utils/export.py @@ -0,0 +1,115 @@ +import csv +import xlsxwriter +import logging +import os +import tempfile +from smtplib import SMTPException + +from django.conf import settings +from django.core.mail import EmailMultiAlternatives + +logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO) +logger = logging.getLogger(__name__) + + +class SendExport: + + def __init__(self, user, panel, file_type='csv'): + self.type_mapper = { + "csv": self.make_csv_file, + "xls": self.make_xls_file + } + self.file_type = file_type + self.user = user + self.panel = panel + self.email_from = settings.EMAIL_HOST_USER + self.email_subject = f'Export panel: {self.get_file_name()}' + self.email_body = 'Exported panel data' + self.get_file_method = self.type_mapper[file_type] + self.file_path = os.path.join( + settings.STATIC_ROOT, + 'email', tempfile.gettempdir(), + self.get_file_name() + ) + self.success = False + + def get_file_name(self): + name = '_'.join(self.panel.name.split(' ')) + return f'export_{name.lower()}.{self.file_type}' + + def get_data(self): + return self.panel.get_data() + + def get_headers(self): + try: + header = self.panel.get_headers() + self.success = True + return header + except Exception as err: + logger.info(f'HEADER:{err}') + + def make_csv_file(self): + file_header = self.get_headers() + if not self.success: + return + with open(self.file_path, 'w') as f: + file_writer = csv.writer(f, quotechar='"', quoting=csv.QUOTE_MINIMAL) + # Write headers to CSV file + file_writer.writerow(file_header) + for row in self.get_data(): + file_writer.writerow(row) + + def make_xls_file(self): + headings = self.get_headers() + if not self.success: + return + with xlsxwriter.Workbook(self.file_path) as workbook: + worksheet = workbook.add_worksheet() + + # Add a bold format to use to highlight cells. + bold = workbook.add_format({'bold': True}) + + # Add the worksheet data that the charts will refer to. + data = self.get_data() + + worksheet.write_row('A1', headings, bold) + for n, row in enumerate(data): + worksheet.write_row(f'A{n+2}', [str(i) for i in row]) + workbook.close() + + def send(self): + self.get_file_method() + print(f'ok: {self.file_path}') + self.send_email() + + def get_file(self): + if os.path.exists(self.file_path) and os.path.isfile(self.file_path): + with open(self.file_path, 'rb') as export_file: + return export_file + else: + logger.info('COMMUTATOR:image file not found dir: {path}') + + def send_email(self): + + msg = EmailMultiAlternatives( + subject=self.email_subject, + body=self.email_body, + from_email=self.email_from, + to=[ + self.user.email, + 'kuzmenko.da@gmail.com', + 'sinapsit@yandex.ru' + ] + ) + + # Create an inline attachment + if self.file_path and self.success: + msg.attach_file(self.file_path) + else: + msg.body = 'An error occurred while executing the request.' + + try: + msg.send() + logger.debug(f"COMMUTATOR:Email successfully sent") + except SMTPException as e: + logger.error(f"COMMUTATOR:Email connector: {e}") \ No newline at end of file diff --git a/requirements/base.txt b/requirements/base.txt index 90e5b2d5..18bb1bc5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -63,3 +63,6 @@ pycountry==19.8.18 # sql-tree django-mptt==0.9.1 + +# Export to Excel +XlsxWriter==1.2.6 \ No newline at end of file From cc7125b468c94ac63683f44fb590173cdc140e1a Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 20 Dec 2019 18:34:33 +0300 Subject: [PATCH 28/32] added image url to es --- apps/collection/models.py | 2 +- apps/establishment/models.py | 10 ++-- apps/location/models.py | 4 +- apps/news/models.py | 4 +- apps/product/models.py | 9 ++-- apps/review/models.py | 4 +- .../search_indexes/documents/establishment.py | 2 + apps/search_indexes/documents/product.py | 2 + apps/utils/models.py | 53 +++++++++++-------- 9 files changed, 51 insertions(+), 39 deletions(-) diff --git a/apps/collection/models.py b/apps/collection/models.py index e3ca63be..d273a9cd 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -11,7 +11,7 @@ from utils.models import ( URLImageMixin, ) from utils.querysets import RelatedObjectsCountMixin -from utils.models import IntermediateGalleryModelMixin, GalleryModelMixin +from utils.models import IntermediateGalleryModelMixin, GalleryMixin # Mixins diff --git a/apps/establishment/models.py b/apps/establishment/models.py index cde60618..e5514bb8 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -27,13 +27,13 @@ from main.models import Award, Currency from review.models import Review from tag.models import Tag from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin, - TranslatedFieldsMixin, BaseAttributes, GalleryModelMixin, + TranslatedFieldsMixin, BaseAttributes, GalleryMixin, IntermediateGalleryModelMixin, HasTagsMixin, - FavoritesMixin) + FavoritesMixin, TypeDefaultImageMixin) # todo: establishment type&subtypes check -class EstablishmentType(TranslatedFieldsMixin, ProjectBaseMixin): +class EstablishmentType(TypeDefaultImageMixin, TranslatedFieldsMixin, ProjectBaseMixin): """Establishment type model.""" STR_FIELD_NAME = 'name' @@ -73,7 +73,7 @@ class EstablishmentSubTypeManager(models.Manager): return obj -class EstablishmentSubType(ProjectBaseMixin, TranslatedFieldsMixin): +class EstablishmentSubType(TypeDefaultImageMixin, TranslatedFieldsMixin, ProjectBaseMixin): """Establishment type model.""" # EXAMPLE OF INDEX NAME CHOICES @@ -431,7 +431,7 @@ class EstablishmentQuerySet(models.QuerySet): ) -class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, +class Establishment(GalleryMixin, ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin, HasTagsMixin, FavoritesMixin): """Establishment model.""" diff --git a/apps/location/models.py b/apps/location/models.py index 13607b12..996cddc7 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -12,7 +12,7 @@ from typing import List from translation.models import Language from utils.models import (ProjectBaseMixin, SVGImageMixin, TJSONField, TranslatedFieldsMixin, get_current_locale, - IntermediateGalleryModelMixin, GalleryModelMixin) + IntermediateGalleryModelMixin, GalleryMixin) class CountryQuerySet(models.QuerySet): @@ -134,7 +134,7 @@ class CityQuerySet(models.QuerySet): return self.filter(country__code=code) -class City(GalleryModelMixin): +class City(GalleryMixin, models.Model): """Region model.""" name = models.CharField(_('name'), max_length=250) code = models.CharField(_('code'), max_length=250) diff --git a/apps/news/models.py b/apps/news/models.py index 64b22ffb..5e5dfcbf 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -14,7 +14,7 @@ from rest_framework.reverse import reverse from main.models import Carousel from rating.models import Rating, ViewCount from utils.models import (BaseAttributes, TJSONField, TranslatedFieldsMixin, HasTagsMixin, - ProjectBaseMixin, GalleryModelMixin, IntermediateGalleryModelMixin, + ProjectBaseMixin, GalleryMixin, IntermediateGalleryModelMixin, FavoritesMixin) from utils.querysets import TranslationQuerysetMixin from datetime import datetime @@ -140,7 +140,7 @@ class NewsQuerySet(TranslationQuerysetMixin): return self.filter(title__icontains=locale) -class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixin, +class News(GalleryMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixin, FavoritesMixin): """News model.""" diff --git a/apps/product/models.py b/apps/product/models.py index 2c2081c9..86aacd6f 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -14,10 +14,11 @@ from location.models import WineOriginAddressMixin from review.models import Review from utils.models import (BaseAttributes, ProjectBaseMixin, HasTagsMixin, TranslatedFieldsMixin, TJSONField, FavoritesMixin, - GalleryModelMixin, IntermediateGalleryModelMixin) + GalleryMixin, IntermediateGalleryModelMixin, + TypeDefaultImageMixin) -class ProductType(TranslatedFieldsMixin, ProjectBaseMixin): +class ProductType(TypeDefaultImageMixin, TranslatedFieldsMixin, ProjectBaseMixin): """ProductType model.""" STR_FIELD_NAME = 'name' @@ -49,7 +50,7 @@ class ProductType(TranslatedFieldsMixin, ProjectBaseMixin): verbose_name_plural = _('Product types') -class ProductSubType(TranslatedFieldsMixin, ProjectBaseMixin): +class ProductSubType(TypeDefaultImageMixin, TranslatedFieldsMixin, ProjectBaseMixin): """ProductSubtype model.""" STR_FIELD_NAME = 'name' @@ -203,7 +204,7 @@ class ProductQuerySet(models.QuerySet): return self.none() -class Product(GalleryModelMixin, TranslatedFieldsMixin, BaseAttributes, +class Product(GalleryMixin, TranslatedFieldsMixin, BaseAttributes, HasTagsMixin, FavoritesMixin): """Product models.""" diff --git a/apps/review/models.py b/apps/review/models.py index bb344fc5..a65ef96f 100644 --- a/apps/review/models.py +++ b/apps/review/models.py @@ -5,7 +5,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from utils.models import (BaseAttributes, TranslatedFieldsMixin, - ProjectBaseMixin, GalleryModelMixin, + ProjectBaseMixin, GalleryMixin, TJSONField, IntermediateGalleryModelMixin) @@ -93,7 +93,7 @@ class Review(BaseAttributes, TranslatedFieldsMixin): verbose_name_plural = _('Reviews') -class Inquiries(GalleryModelMixin, ProjectBaseMixin): +class Inquiries(GalleryMixin, ProjectBaseMixin): NONE = 0 DINER = 1 LUNCH = 2 diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index e9761393..e520932d 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -23,12 +23,14 @@ class EstablishmentDocument(Document): 'name': fields.ObjectField(attr='name_indexing', properties=OBJECT_FIELD_PROPERTIES), 'index_name': fields.KeywordField(attr='index_name'), + 'default_image': fields.KeywordField(attr='default_image_url'), }) establishment_subtypes = fields.ObjectField( properties={ 'id': fields.IntegerField(), 'name': fields.ObjectField(attr='name_indexing'), 'index_name': fields.KeywordField(attr='index_name'), + 'default_image': fields.KeywordField(attr='default_image_url'), }, multi=True) works_evening = fields.ListField(fields.IntegerField( diff --git a/apps/search_indexes/documents/product.py b/apps/search_indexes/documents/product.py index aa8fc999..3d6ebbd5 100644 --- a/apps/search_indexes/documents/product.py +++ b/apps/search_indexes/documents/product.py @@ -19,6 +19,7 @@ class ProductDocument(Document): 'id': fields.IntegerField(), 'name': fields.ObjectField(attr='name_indexing', properties=OBJECT_FIELD_PROPERTIES), 'index_name': fields.KeywordField(), + 'default_image': fields.KeywordField(attr='default_image_url'), }, ) subtypes = fields.ObjectField( @@ -26,6 +27,7 @@ class ProductDocument(Document): 'id': fields.IntegerField(), 'name': fields.ObjectField(attr='name_indexing', properties=OBJECT_FIELD_PROPERTIES), 'index_name': fields.KeywordField(), + 'default_image': fields.KeywordField(attr='default_image_url'), }, multi=True ) diff --git a/apps/utils/models.py b/apps/utils/models.py index 42e35bb1..5f694d95 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -364,16 +364,12 @@ class GMTokenGenerator(PasswordResetTokenGenerator): return self.get_fields(user, timestamp) -class GalleryModelMixin(models.Model): +class GalleryMixin: """Mixin for models that has gallery.""" - class Meta: - """Meta class.""" - abstract = True - @property def crop_gallery(self): - if hasattr(self, 'gallery'): + if hasattr(self, 'gallery') and hasattr(self, '_meta'): gallery = [] images = self.gallery.all() crop_parameters = [p for p in settings.SORL_THUMBNAIL_ALIASES @@ -393,22 +389,23 @@ class GalleryModelMixin(models.Model): @property def crop_main_image(self): - if hasattr(self, 'main_image') and self.main_image: - image = self.main_image - image_property = { - 'id': image.id, - 'title': image.title, - 'original_url': image.image.url, - 'orientation_display': image.get_orientation_display(), - 'auto_crop_images': {}, - } - crop_parameters = [p for p in settings.SORL_THUMBNAIL_ALIASES - if p.startswith(self._meta.model_name.lower())] - for crop in crop_parameters: - image_property['auto_crop_images'].update( - {crop: image.get_image_url(crop)} - ) - return image_property + if hasattr(self, 'main_image') and hasattr(self, '_meta'): + if self.main_image: + image = self.main_image + image_property = { + 'id': image.id, + 'title': image.title, + 'original_url': image.image.url, + 'orientation_display': image.get_orientation_display(), + 'auto_crop_images': {}, + } + crop_parameters = [p for p in settings.SORL_THUMBNAIL_ALIASES + if p.startswith(self._meta.model_name.lower())] + for crop in crop_parameters: + image_property['auto_crop_images'].update( + {crop: image.get_image_url(crop)} + ) + return image_property class IntermediateGalleryModelQuerySet(models.QuerySet): @@ -459,4 +456,14 @@ class FavoritesMixin: return self.favorites.aggregate(arr=ArrayAgg('user_id')).get('arr') -timezone.datetime.now().date().isoformat() \ No newline at end of file +timezone.datetime.now().date().isoformat() + + +class TypeDefaultImageMixin: + """Model mixin for default image.""" + + @property + def default_image_url(self): + """Return image url.""" + if hasattr(self, 'default_image') and self.default_image: + return self.default_image.image From a6bcbdddc1e10efbc9bc96729f722fcedfe0464d Mon Sep 17 00:00:00 2001 From: Anatoly Date: Mon, 23 Dec 2019 09:44:29 +0300 Subject: [PATCH 29/32] fix es image url --- apps/utils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/utils/models.py b/apps/utils/models.py index 5f694d95..de83c711 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -466,4 +466,4 @@ class TypeDefaultImageMixin: def default_image_url(self): """Return image url.""" if hasattr(self, 'default_image') and self.default_image: - return self.default_image.image + return self.default_image.image.url From 17fdc7705e47c74819f7e506591b83b44065ec3b Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 23 Dec 2019 11:53:34 +0300 Subject: [PATCH 30/32] news transfer fin --- apps/news/management/commands/add_author.py | 29 ---- .../news/management/commands/add_news_tags.py | 37 ----- .../migrations/0049_auto_20191223_0619.py | 19 +++ apps/news/models.py | 11 +- apps/news/transfer_data.py | 151 +++++++++++++++--- .../migrations/0005_auto_20191223_0850.py | 18 +++ apps/rating/models.py | 2 +- apps/recipe/models.py | 4 + apps/transfer/serializers/news.py | 72 +++------ docker-compose.mysql.yml | 2 - 10 files changed, 195 insertions(+), 150 deletions(-) delete mode 100644 apps/news/management/commands/add_author.py delete mode 100644 apps/news/management/commands/add_news_tags.py create mode 100644 apps/news/migrations/0049_auto_20191223_0619.py create mode 100644 apps/rating/migrations/0005_auto_20191223_0850.py diff --git a/apps/news/management/commands/add_author.py b/apps/news/management/commands/add_author.py deleted file mode 100644 index 0313d7b6..00000000 --- a/apps/news/management/commands/add_author.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.core.management.base import BaseCommand -from django.db.models import F -from tqdm import tqdm - -from account.models import User -from news.models import News -from transfer.models import PageTexts - - -class Command(BaseCommand): - help = 'Add author of News' - - def handle(self, *args, **kwargs): - count = 0 - news_list = News.objects.filter(created_by__isnull=True) - - for news in tqdm(news_list, desc="Find author for exist news"): - old_news = PageTexts.objects.filter(id=news.old_id).annotate( - account_id=F('page__account_id'), - ).first() - if old_news: - user = User.objects.filter(old_id=old_news.account_id).first() - if user: - news.created_by = user - news.modified_by = user - news.save() - count += 1 - - self.stdout.write(self.style.WARNING(f'Update {count} objects.')) diff --git a/apps/news/management/commands/add_news_tags.py b/apps/news/management/commands/add_news_tags.py deleted file mode 100644 index b4c5f8eb..00000000 --- a/apps/news/management/commands/add_news_tags.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.core.management.base import BaseCommand - -from news.models import News, NewsType -from tag.models import Tag, TagCategory -from transfer.models import PageMetadata, Pages, PageTexts - - -class Command(BaseCommand): - help = 'Remove old news from new bd'\ - # TODO: изменить перенос тэгов по old_id новостей (они теперь от page) - - def handle(self, *args, **kwargs): - count = 0 - news_type, _ = NewsType.objects.get_or_create(name='News') - tag_cat, _ = TagCategory.objects.get_or_create(index_name='category') - news_type.tag_categories.add(tag_cat) - news_type.save() - - old_news_tag = PageMetadata.objects.filter(key='category', page__pagetexts__isnull=False) - for old_tag in old_news_tag: - old_id_list = old_tag.page.pagetexts_set.all().values_list('id', flat=True) - - # Make Tag - new_tag, created = Tag.objects.get_or_create(category=tag_cat, value=old_tag.value) - if created: - text_value = ' '.join(new_tag.value.split('_')) - new_tag.label = {'en-GB': text_value} - new_tag.save() - for id in old_id_list: - if isinstance(id, int): - news = News.objects.filter(old_id=id).first() - if news: - news.tags.add(new_tag) - news.save() - count += 1 - - self.stdout.write(self.style.WARNING(f'Create or update {count} objects.')) diff --git a/apps/news/migrations/0049_auto_20191223_0619.py b/apps/news/migrations/0049_auto_20191223_0619.py new file mode 100644 index 00000000..4d9cf52c --- /dev/null +++ b/apps/news/migrations/0049_auto_20191223_0619.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.7 on 2019-12-23 06:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0048_remove_news_must_of_the_week'), + ] + + operations = [ + migrations.AlterField( + model_name='news', + name='views_count', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='news', to='rating.ViewCount'), + ), + ] diff --git a/apps/news/models.py b/apps/news/models.py index 64b22ffb..76348a3d 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -105,10 +105,10 @@ class NewsQuerySet(TranslationQuerysetMixin): date_now = now.date() time_now = now.time() return self.exclude(models.Q(publication_date__isnull=True) | models.Q(publication_time__isnull=True)). \ - filter(models.Q(models.Q(end__gte=now) | - models.Q(end__isnull=True)), - state__in=self.model.PUBLISHED_STATES, publication_date__lte=date_now, - publication_time__lte=time_now) + filter(models.Q(models.Q(end__gte=now) | + models.Q(end__isnull=True)), + state__in=self.model.PUBLISHED_STATES, publication_date__lte=date_now, + publication_time__lte=time_now) # todo: filter by best score # todo: filter by country? @@ -215,7 +215,8 @@ class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixi tags = models.ManyToManyField('tag.Tag', related_name='news', verbose_name=_('Tags')) gallery = models.ManyToManyField('gallery.Image', through='news.NewsGallery') - views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL) + views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL, + related_name='news') ratings = generic.GenericRelation(Rating) favorites = generic.GenericRelation(to='favorites.Favorites') carousels = generic.GenericRelation(to='main.Carousel') diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 8596bcf7..c2f621a5 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,34 +1,47 @@ from pprint import pprint -from django.db.models import Aggregate, CharField, Value from django.db.models import IntegerField, F +from django.db.models import Value +from tqdm import tqdm -from news.models import NewsType -from tag.models import TagCategory -from transfer.models import PageTexts +from gallery.models import Image +from news.models import NewsType, News +from rating.models import ViewCount +from tag.models import TagCategory, Tag +from transfer.models import PageTexts, PageCounters, PageMetadata from transfer.serializers.news import NewsSerializer -class GroupConcat(Aggregate): - function = 'GROUP_CONCAT' - template = '%(function)s(%(expressions)s)' +def add_locale(locale, data): + if isinstance(data, dict) and locale not in data: + data.update({ + locale: next(iter(data.values())) + }) + return data - def __init__(self, expression, **extra): - output_field = extra.pop('output_field', CharField()) - super().__init__(expression, output_field=output_field, **extra) - def as_postgresql(self, compiler, connection): - self.function = 'STRING_AGG' - return super().as_sql(compiler, connection) +def clear_old_news(): + """ + Clear lod news and news images + """ + images = Image.objects.filter( + news_gallery__isnull=False, + news__gallery__news__old_id__isnull=False + ) + img_num = images.count() + + news = News.objects.filter(old_id__isnull=False) + news_num = news.count() + + images.delete() + news.delete() + + print(f'Deleted {img_num} images') + print(f'Deleted {news_num} news') def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name='News') - tag_cat_tag, _ = TagCategory.objects.get_or_create(index_name='tag') - tag_cat_category, _ = TagCategory.objects.get_or_create(index_name='category') - news_type.tag_categories.add(tag_cat_tag) - news_type.tag_categories.add(tag_cat_category) - news_type.save() queryset = PageTexts.objects.filter( page__type='News', @@ -43,10 +56,6 @@ def transfer_news(): page__root_title=F('page__root_title'), page__attachment_suffix_url=F('page__attachment_suffix_url'), page__published_at=F('page__published_at'), - - tags=GroupConcat('page__tags__id'), - tag_cat_tag_id=Value(tag_cat_tag.id, output_field=IntegerField()), - tag_cat_category_id=Value(tag_cat_category.id, output_field=IntegerField()), ) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) @@ -56,6 +65,102 @@ def transfer_news(): pprint(f'News serializer errors: {serialized_data.errors}') +def update_en_gb_locales(): + """ + Update default locales (en-GB) + """ + news = News.objects.filter(old_id__isnull=False) + + update_news = [] + for news_item in tqdm(news): + news_item.slugs = add_locale('en-GB', news_item.slugs) + news_item.title = add_locale('en-GB', news_item.title) + news_item.locale_to_description_is_active = add_locale('en-GB', news_item.locale_to_description_is_active) + news_item.description = add_locale('en-GB', news_item.description) + news_item.subtitle = add_locale('en-GB', news_item.subtitle) + update_news.append(news_item) + News.objects.bulk_update(update_news, [ + 'slugs', + 'title', + 'locale_to_description_is_active', + 'description', + 'subtitle', + ]) + print(f'Updated {len(update_news)} news locales') + + +def add_views_count(): + """ + Add views count to news from page_counters + """ + + news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + counters = PageCounters.objects.filter(page_id__in=list(news)) + + update_counters = [] + for counter in tqdm(counters): + news_item = News.objects.filter(old_id=counter.page_id).first() + if news_item: + obj, _ = ViewCount.objects.update_or_create( + news=news_item, + defaults={'count': counter.count}, + ) + news_item.views_count = obj + update_counters.append(news_item) + News.objects.bulk_update(update_counters, ['views_count', ]) + print(f'Updated {len(update_counters)} news counters') + + +def add_tags(): + """ + Add news tags + """ + + news_type, _ = NewsType.objects.get_or_create(name='News') + tag_category, _ = TagCategory.objects.get_or_create(index_name='category') + tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag') + news_type.tag_categories.add(tag_category) + news_type.tag_categories.add(tag_tag) + news_type.save() + + tag_cat = { + 'category': tag_category, + 'tag': tag_tag, + } + + news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + old_news_tag = PageMetadata.objects.filter( + key__in=('category', 'tag'), + page_id__in=list(news), + ) + + count = 0 + for old_tag in tqdm(old_news_tag): + old_id = old_tag.page.id + new_tag, created = Tag.objects.get_or_create( + category=tag_cat.get(old_tag.key), + value=old_tag.value, + ) + if created: + text_value = ' '.join(new_tag.value.split('_')) + new_tag.label = {'en-GB': text_value} + new_tag.save() + + news = News.objects.filter(old_id=old_id).first() + if news: + news.tags.add(new_tag) + news.save() + count += 1 + + print(f'Updated {count} tags') + + data_types = { - 'news': [transfer_news] + 'news': [ + clear_old_news, + transfer_news, + update_en_gb_locales, + add_views_count, + add_tags, + ] } diff --git a/apps/rating/migrations/0005_auto_20191223_0850.py b/apps/rating/migrations/0005_auto_20191223_0850.py new file mode 100644 index 00000000..437056c7 --- /dev/null +++ b/apps/rating/migrations/0005_auto_20191223_0850.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-12-23 08:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rating', '0004_auto_20191114_2041'), + ] + + operations = [ + migrations.AlterField( + model_name='viewcount', + name='count', + field=models.PositiveIntegerField(), + ), + ] diff --git a/apps/rating/models.py b/apps/rating/models.py index 5db8332e..a4048128 100644 --- a/apps/rating/models.py +++ b/apps/rating/models.py @@ -23,4 +23,4 @@ class Rating(models.Model): class ViewCount(models.Model): - count = models.IntegerField() + count = models.PositiveIntegerField() diff --git a/apps/recipe/models.py b/apps/recipe/models.py index c419be4c..f8c4aedf 100644 --- a/apps/recipe/models.py +++ b/apps/recipe/models.py @@ -10,6 +10,7 @@ class RecipeQuerySet(models.QuerySet): # todo: what records are considered published? def published(self): + # TODO: проверка по полю published_at return self.filter(state__in=[self.model.PUBLISHED, self.model.PUBLISHED_EXCLUSIVE]) @@ -67,3 +68,6 @@ class Recipe(TranslatedFieldsMixin, ImageMixin, BaseAttributes): verbose_name = _('Recipe') verbose_name_plural = _('Recipes') + + # TODO: в save добавить обновление published_at если state в PUBLISHED или PUBLISHED_EXCLUSIVE + # TODO: в save добавить обновление published_at в None если state в WAITING или HIDDEN diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 15634ae7..ab6a0b4b 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,35 +1,13 @@ from rest_framework import serializers +from account.models import User from gallery.models import Image from location.models import Country from news.models import News, NewsGallery -from tag.models import Tag -from transfer.models import PageMetadata from utils.legacy_parser import parse_legacy_news_content -from utils.slug_generator import generate_unique_slug -from account.models import User class NewsSerializer(serializers.Serializer): - # old_id = page__id id -done - # news_type = 'News' создали или получили в трансфере -done - # title = {"en-GB":"some text"} из locale и title -done - # backoffice_title = page__root_title -done - # subtitle = {"en-GB":"some text"} из locale и summary -done - # description = {"en-GB":"some text"} из locale и body -done - # locale_to_description_is_active = {"en-GB": true, "fr-FR": false} из locale и true -done - # publication_date = DateField из page published_at -done ??? проверить - # publication_time = DateField из page published_at -done ??? проверить - # slugs = {"en-GB":"some slug"} из locale и slug -done - # state = page__state -done - # template = page__template -done - # country = по page__site__country_code_2 -done - # tags = по page__tags__id -progress -!!! - # gallery = в методе make_gallery из page__attachment_suffix_url -done - # created_by = page__account_id -done - # modified_by = page__account_id -done - # created = page created_at -done - locale = serializers.CharField() page__id = serializers.IntegerField() news_type_id = serializers.IntegerField() @@ -46,10 +24,6 @@ class NewsSerializer(serializers.Serializer): page__attachment_suffix_url = serializers.CharField() page__published_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S', allow_null=True) - tags = serializers.CharField(allow_null=True) - tag_cat_tag_id = serializers.IntegerField() - tag_cat_category_id = serializers.IntegerField() - def create(self, data): account = self.get_account(data) payload = { @@ -61,7 +35,7 @@ class NewsSerializer(serializers.Serializer): 'state': self.get_state(data), 'template': self.get_template(data), 'country': self.get_country(data), - 'slug': {data['locale']: data['slug']}, + 'slugs': {data['locale']: data['slug']}, 'description': self.get_description(data), 'title': {data['locale']: data['title']}, 'backoffice_title': data['page__root_title'], @@ -71,15 +45,26 @@ class NewsSerializer(serializers.Serializer): 'publication_time': self.get_publication_time(data), } - obj, _ = News.objects.update_or_create( - old_id=data['old_id'], + obj, created = News.objects.get_or_create( + old_id=payload['old_id'], defaults=payload, ) + if not created: + obj.slugs.update(payload['slugs']) + obj.title.update(payload['title']) + obj.locale_to_description_is_active.update(payload['locale_to_description_is_active']) - tags = self.get_tags(data) - for tag in tags: - obj.tags.add(tag) - obj.save() + if obj.description and payload['description']: + obj.description.update(payload['description']) + else: + obj.description = payload['description'] + + if obj.subtitle and payload['subtitle']: + obj.subtitle.update(payload['subtitle']) + else: + obj.subtitle = payload['subtitle'] + + obj.save() self.make_gallery(data, obj) return obj @@ -139,25 +124,6 @@ class NewsSerializer(serializers.Serializer): return {data['locale']: data['summary']} return None - @staticmethod - def get_tags(data): - results = [] - if not data['tags']: - return results - - meta_ids = (int(_id) for _id in data['tags'].split(',')) - tags = PageMetadata.objects.filter( - id__in=meta_ids, - value__isnull=False, - ) - for old_tag in tags: - tag, _ = Tag.objects.get_or_create( - category_id=data['tag_cat_id'], - label={data['locale']: old_tag.value}, - ) - results.append(tag) - return results - @staticmethod def make_gallery(data, obj): if not data['page__attachment_suffix_url'] or data['page__attachment_suffix_url'] == 'default/missing.png': diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index d62ff677..a8900226 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -13,7 +13,6 @@ services: MYSQL_ROOT_PASSWORD: rootPassword volumes: - gm-mysql_db:/var/lib/mysql - - .:/code # PostgreSQL database @@ -30,7 +29,6 @@ services: - "5436:5432" volumes: - gm-db:/var/lib/postgresql/data/ - - .:/code elasticsearch: From f6dac3cbcd6adc716419c333dc7d3376535170f7 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 23 Dec 2019 12:33:31 +0300 Subject: [PATCH 31/32] fix news test --- apps/news/tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/news/tests.py b/apps/news/tests.py index 42c4a694..3bc7b80b 100644 --- a/apps/news/tests.py +++ b/apps/news/tests.py @@ -31,7 +31,6 @@ class BaseTestCase(APITestCase): 'refresh_token': tokens.get('refresh_token')}) self.test_news_type = NewsType.objects.create(name="Test news type") - self.lang, created = Language.objects.get_or_create( title='Russia', locale='ru-RU' @@ -57,13 +56,11 @@ class BaseTestCase(APITestCase): ) user_role.save() - self.test_news = News.objects.create( created_by=self.user, modified_by=self.user, title={"ru-RU": "Test news"}, news_type=self.test_news_type, description={"ru-RU": "Description test news"}, - start=datetime.now() + timedelta(hours=-2), end=datetime.now() + timedelta(hours=2), state=News.PUBLISHED, slugs={'en-GB': 'test-news-slug'}, @@ -119,7 +116,6 @@ class NewsTestCase(BaseTestCase): 'id': self.test_news.id, 'description': {"ru-RU": "Description test news!"}, 'slugs': self.test_news.slugs, - 'start': self.test_news.start, 'news_type_id': self.test_news.news_type_id, 'country_id': self.country_ru.id, "site_id": self.site_ru.id From 2fb338bc59af5b2ce5eef297b3106e900e3261f2 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Mon, 23 Dec 2019 12:59:39 +0300 Subject: [PATCH 32/32] refactored method to get similar list on entities (see todo: establishment/models.py) --- apps/establishment/models.py | 8 +++++--- apps/establishment/views/web.py | 8 ++++---- apps/product/views/common.py | 28 ++++++++++++++++++++-------- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index e5514bb8..8ad1663c 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -253,8 +253,9 @@ class EstablishmentQuerySet(models.QuerySet): return Subquery( self.similar_base(establishment) .filter(**filters) - .order_by('distance')[:settings.LIMITING_QUERY_OBJECTS] - .values('id') + .order_by('distance') + .distinct() + .values_list('id', flat=True)[:settings.LIMITING_QUERY_OBJECTS] ) def similar_restaurants(self, restaurant): @@ -271,7 +272,8 @@ class EstablishmentQuerySet(models.QuerySet): 'establishment_gallery__is_main': True, } ) - return self.filter(id__in=ids_by_subquery) \ + # todo: fix this - replace ids_by_subquery.queryset on ids_by_subquery + return self.filter(id__in=ids_by_subquery.queryset) \ .annotate_intermediate_public_mark() \ .annotate_mark_similarity(mark=restaurant.public_mark) \ .order_by('mark_similarity') \ diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index d94be1d5..b4e6f776 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -44,7 +44,7 @@ class EstablishmentListView(EstablishmentMixinView, generics.ListAPIView): class EstablishmentSimilarView(EstablishmentListView): """Resource for getting a list of similar establishments.""" serializer_class = serializers.EstablishmentSimilarSerializer - pagination_class = PortionPagination + pagination_class = None def get_base_object(self): """ @@ -105,7 +105,7 @@ class RestaurantSimilarListView(EstablishmentSimilarView): base_establishment = self.get_base_object() if base_establishment: - return qs.similar_restaurants(base_establishment) + return qs.similar_restaurants(base_establishment)[:settings.QUERY_OUTPUT_OBJECTS] else: return EstablishmentMixinView.get_queryset(self) \ .none() @@ -120,7 +120,7 @@ class WinerySimilarListView(EstablishmentSimilarView): base_establishment = self.get_base_object() if base_establishment: - return qs.similar_wineries(base_establishment) + return qs.similar_wineries(base_establishment)[:settings.QUERY_OUTPUT_OBJECTS] else: return qs.none() @@ -134,7 +134,7 @@ class ArtisanProducerSimilarListView(EstablishmentSimilarView): base_establishment = self.get_base_object() if base_establishment: - return qs.similar_artisans_producers(base_establishment) + return qs.similar_artisans_producers(base_establishment)[:settings.QUERY_OUTPUT_OBJECTS] else: return qs.none() diff --git a/apps/product/views/common.py b/apps/product/views/common.py index dbb24e53..5eeaccf2 100644 --- a/apps/product/views/common.py +++ b/apps/product/views/common.py @@ -1,12 +1,13 @@ """Product app views.""" -from rest_framework import generics, permissions +from django.conf import settings from django.shortcuts import get_object_or_404 -from product.models import Product +from rest_framework import generics, permissions + from comment.models import Comment -from product import filters, serializers from comment.serializers import CommentRUDSerializer +from product import filters, serializers +from product.models import Product from utils.views import FavoritesCreateDestroyMixinView -from utils.pagination import PortionPagination class ProductBaseView(generics.GenericAPIView): @@ -35,7 +36,15 @@ class ProductListView(ProductBaseView, generics.ListAPIView): class ProductSimilarView(ProductListView): """Resource for getting a list of similar product.""" serializer_class = serializers.ProductBaseSerializer - pagination_class = PortionPagination + pagination_class = None + + def get_base_object(self): + """ + Return base product instance for a getting list of similar products. + """ + product = get_object_or_404(Product.objects.all(), + slug=self.kwargs.get('slug')) + return product class ProductDetailView(ProductBaseView, generics.RetrieveAPIView): @@ -95,7 +104,10 @@ class SimilarListView(ProductSimilarView): def get_queryset(self): """Overridden get_queryset method.""" - return super().get_queryset() \ - .has_location() \ - .similar(slug=self.kwargs.get('slug')) + qs = super(SimilarListView, self).get_queryset() + base_product = self.get_base_object() + if base_product: + return qs.has_location().similar(slug=self.kwargs.get('slug'))[:settings.QUERY_OUTPUT_OBJECTS] + else: + return qs.none()