From 6d9686b64a96f9ffe52fa40ef55afe576ea063a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Mon, 2 Dec 2019 15:12:17 +0300 Subject: [PATCH 01/17] Fix error --- apps/account/models.py | 4 +++- apps/account/serializers/back.py | 19 ++++++++++--------- apps/account/views/back.py | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/apps/account/models.py b/apps/account/models.py index c212ffda..956baa56 100644 --- a/apps/account/models.py +++ b/apps/account/models.py @@ -287,7 +287,9 @@ class User(AbstractUser): class UserRole(ProjectBaseMixin): """UserRole model.""" - user = models.ForeignKey(User, verbose_name=_('User'), on_delete=models.CASCADE) + user = models.ForeignKey('account.User', + verbose_name=_('User'), + on_delete=models.CASCADE) role = models.ForeignKey(Role, verbose_name=_('Role'), on_delete=models.SET_NULL, null=True) establishment = models.ForeignKey(Establishment, verbose_name=_('Establishment'), on_delete=models.SET_NULL, null=True, blank=True) diff --git a/apps/account/serializers/back.py b/apps/account/serializers/back.py index 57c3fb42..b2316734 100644 --- a/apps/account/serializers/back.py +++ b/apps/account/serializers/back.py @@ -13,15 +13,6 @@ class RoleSerializer(serializers.ModelSerializer): ] -class UserRoleSerializer(serializers.ModelSerializer): - class Meta: - model = models.UserRole - fields = [ - 'user', - 'role' - ] - - class BackUserSerializer(serializers.ModelSerializer): class Meta: model = User @@ -49,3 +40,13 @@ class BackDetailUserSerializer(BackUserSerializer): user.set_password(validated_data['password']) user.save() return user + + +class UserRoleSerializer(serializers.ModelSerializer): + class Meta: + model = models.UserRole + fields = [ + 'role', + 'user', + 'establishment' + ] diff --git a/apps/account/views/back.py b/apps/account/views/back.py index b3d77d1e..80775b3a 100644 --- a/apps/account/views/back.py +++ b/apps/account/views/back.py @@ -13,7 +13,7 @@ class RoleLstView(generics.ListCreateAPIView): class UserRoleLstView(generics.ListCreateAPIView): serializer_class = serializers.UserRoleSerializer - queryset = models.Role.objects.all() + queryset = models.UserRole.objects.all() class UserLstView(generics.ListCreateAPIView): From fdc63f2677f34c2f5c4543fdd8f1ae6de2c2f5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Tue, 3 Dec 2019 10:20:13 +0300 Subject: [PATCH 02/17] Add API methods for site-feature --- apps/main/serializers.py | 55 +++++++++++++--------------------------- apps/main/urls/back.py | 8 +++++- apps/main/views/back.py | 10 ++++++++ 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/apps/main/serializers.py b/apps/main/serializers.py index 561a50c0..b93382d5 100644 --- a/apps/main/serializers.py +++ b/apps/main/serializers.py @@ -21,27 +21,6 @@ class FeatureSerializer(serializers.ModelSerializer): ) -class SiteFeatureSerializer(serializers.ModelSerializer): - id = serializers.IntegerField(source='feature.id') - slug = serializers.CharField(source='feature.slug') - priority = serializers.IntegerField(source='feature.priority') - route = serializers.CharField(source='feature.route.name') - source = serializers.IntegerField(source='feature.source') - nested = RecursiveFieldSerializer(many=True, allow_null=True) - - class Meta: - """Meta class.""" - model = models.SiteFeature - fields = ('main', - 'id', - 'slug', - 'priority', - 'route', - 'source', - 'nested', - ) - - class CurrencySerializer(ProjectModelSerializer): """Currency serializer.""" @@ -56,6 +35,23 @@ class CurrencySerializer(ProjectModelSerializer): ] +class SiteFeatureSerializer(serializers.ModelSerializer): + """Site feature serializer.""" + + class Meta: + """Meta class.""" + + model = models.SiteFeature + fields = ( + 'id', + 'site_settings', + 'feature', + 'published', + 'main', + 'nested' + ) + + class SiteSettingsSerializer(serializers.ModelSerializer): """Site settings serializer.""" @@ -140,7 +136,7 @@ class SiteBackOfficeSerializer(SiteSerializer): class FeatureSerializer(serializers.ModelSerializer): - """Site feature serializer.""" + """Feature serializer.""" class Meta: """Meta class.""" @@ -155,21 +151,6 @@ class FeatureSerializer(serializers.ModelSerializer): ) -# class SiteFeatureSerializer(serializers.ModelSerializer): -# """Site feature serializer.""" -# -# class Meta: -# """Meta class.""" -# -# model = models.SiteFeature -# fields = ( -# 'id', -# 'published', -# 'site_settings', -# 'feature', -# ) - - class AwardBaseSerializer(serializers.ModelSerializer): """Award base serializer.""" diff --git a/apps/main/urls/back.py b/apps/main/urls/back.py index 99e6a50f..609e61f7 100644 --- a/apps/main/urls/back.py +++ b/apps/main/urls/back.py @@ -13,5 +13,11 @@ urlpatterns = [ path('site-settings//', views.SiteSettingsBackOfficeView.as_view(), name='site-settings'), path('feature/', views.FeatureBackView.as_view(), name='feature-list-create'), - path('feature//', views.FeatureRUDBackView.as_view(), name='feature-rud') + path('feature//', views.FeatureRUDBackView.as_view(), name='feature-rud'), + path('site-feature/', views.SiteFeatureBackView.as_view(), + name='site-feature-list-create'), + path('site-feature//', views.SiteFeatureRUDBackView.as_view(), + name='site-feature-rud'), ] + + diff --git a/apps/main/views/back.py b/apps/main/views/back.py index 76c99e3d..adc0196a 100644 --- a/apps/main/views/back.py +++ b/apps/main/views/back.py @@ -44,11 +44,21 @@ class FeatureBackView(generics.ListCreateAPIView): serializer_class = serializers.FeatureSerializer +class SiteFeatureBackView(generics.ListCreateAPIView): + """Feature list or create View.""" + serializer_class = serializers.SiteFeatureSerializer + + class FeatureRUDBackView(generics.RetrieveUpdateDestroyAPIView): """Feature RUD View.""" serializer_class = serializers.FeatureSerializer +class SiteFeatureRUDBackView(generics.RetrieveUpdateDestroyAPIView): + """Feature RUD View.""" + serializer_class = serializers.SiteFeatureSerializer + + class SiteSettingsBackOfficeView(SiteSettingsView): """Site settings View.""" serializer_class = serializers.SiteSettingsBackOfficeSerializer From 4b9561fe752e55c851316923e8b84f11b8841295 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 11:50:36 +0300 Subject: [PATCH 03/17] Fix typo --- apps/establishment/views/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index cfd4880e..15421ee7 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -38,7 +38,7 @@ class EstablishmentListView(EstablishmentMixinView, generics.ListAPIView): .with_extended_address_related().with_currency_related() \ .with_certain_tag_category_related('category', 'restaurant_category') \ .with_certain_tag_category_related('cuisine', 'restaurant_cuisine') \ - .with_ceratin_tag_category_related('shop_category', 'artisan_category') + .with_certain_tag_category_related('shop_category', 'artisan_category') class EstablishmentRetrieveView(EstablishmentMixinView, generics.RetrieveAPIView): From 201e02b4b794a37050769b01008372d9c186c23a Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 13:29:37 +0300 Subject: [PATCH 04/17] remove empty favs --- apps/tag/filters.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/tag/filters.py b/apps/tag/filters.py index 5e2b31a7..46470ca1 100644 --- a/apps/tag/filters.py +++ b/apps/tag/filters.py @@ -73,7 +73,10 @@ class TagsFilterSet(TagsBaseFilterSet): def by_establishment_type(self, queryset, name, value): if value == EstablishmentType.ARTISAN: - return models.Tag.objects.by_category_index_name('shop_category')[0:8] + qs = models.Tag.objects.by_category_index_name('shop_category') + if self.request.country_code and self.request.country_code not in settings.INTERNATIONAL_COUNTRY_CODES: + qs = qs.filter(establishments__address__city__country__code=self.request.country_code).distinct('id') + return qs.exclude(establishments__isnull=True)[0:8] return queryset.by_establishment_type(value) # TMP TODO remove it later From e7de51373d99a8b3d9a31f13545488fb84c0e287 Mon Sep 17 00:00:00 2001 From: dormantman Date: Tue, 3 Dec 2019 14:39:25 +0300 Subject: [PATCH 05/17] Correction of finding the visual center of points --- apps/search_indexes/filters.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 7a3993c0..4537124d 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -11,7 +11,7 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @staticmethod def calculate_center(first, second): - if second[1] < 0 <= first[1]: + if second[1] < 0 < first[1]: reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) diff = (reverse_first + reverse_second) / 2 @@ -21,6 +21,16 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): else: result_part = 180 - (180 - first[1] - diff) + elif second[1] < 0 > first[1]: + diff = abs(abs(second[1]) - abs(first[1])) + + if diff > 90: + reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) + result_part = (reverse_first + reverse_second) / 2 + + else: + result_part = (first[1] + second[1]) / 2 + else: result_part = (first[1] + second[1]) / 2 From 637903e6135304021d28c443e4dc8e1bfbe1a43a Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 3 Dec 2019 15:15:02 +0300 Subject: [PATCH 06/17] fix scheduler --- apps/account/serializers/common.py | 4 +++ .../management/commands/fix_scheduler.py | 36 +++++++++++++++++++ apps/transfer/serializers/establishment.py | 2 +- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 apps/establishment/management/commands/fix_scheduler.py diff --git a/apps/account/serializers/common.py b/apps/account/serializers/common.py index d2933747..d2ce0974 100644 --- a/apps/account/serializers/common.py +++ b/apps/account/serializers/common.py @@ -92,7 +92,11 @@ class UserBaseSerializer(serializers.ModelSerializer): model = models.User fields = ( + 'id', 'fullname', + 'first_name', + 'last_name', + 'email', 'cropped_image_url', 'image_url', ) diff --git a/apps/establishment/management/commands/fix_scheduler.py b/apps/establishment/management/commands/fix_scheduler.py new file mode 100644 index 00000000..78422935 --- /dev/null +++ b/apps/establishment/management/commands/fix_scheduler.py @@ -0,0 +1,36 @@ +from django.core.management.base import BaseCommand +from tqdm import tqdm + +from establishment.models import Establishment +from transfer.models import Establishments +from transfer.serializers.establishment import EstablishmentSerializer +from timetable.models import Timetable +from django.db import transaction + + +class Command(BaseCommand): + help = 'Fix scheduler' + + @transaction.atomic + def handle(self, *args, **kwargs): + count = 0 + establishments = Establishment.objects.all() + old_est_list = Establishments.objects.prefetch_related( + 'schedules_set', + ) + # remove old records of Timetable + Timetable.objects.all().delete() + + for est in tqdm(establishments, desc="Fix scheduler"): + old_est = old_est_list.filter(id=est.old_id).first() + + if old_est and old_est.schedules_set.exists(): + old_schedule = old_est.schedules_set.first() + timetable = old_schedule.timetable + if timetable: + new_schedules = EstablishmentSerializer.get_schedules(timetable) + est.schedule.add(*new_schedules) + est.save() + count += 1 + + self.stdout.write(self.style.WARNING(f'Update {count} objects.')) diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index 0e5f55d4..a287d61b 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -125,7 +125,7 @@ class EstablishmentSerializer(serializers.ModelSerializer): weekdays = { 'su': Timetable.SUNDAY, 'mo': Timetable.MONDAY, - 'tu': Timetable.THURSDAY, + 'tu': Timetable.TUESDAY, 'we': Timetable.WEDNESDAY, 'th': Timetable.THURSDAY, 'fr': Timetable.FRIDAY, From 4940a4472db324a19faa01aa7a0be04ef575ee3e Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 3 Dec 2019 15:54:15 +0300 Subject: [PATCH 07/17] add additional fields to UserBaseSerializer --- apps/account/serializers/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/account/serializers/common.py b/apps/account/serializers/common.py index d2ce0974..09136934 100644 --- a/apps/account/serializers/common.py +++ b/apps/account/serializers/common.py @@ -93,6 +93,7 @@ class UserBaseSerializer(serializers.ModelSerializer): model = models.User fields = ( 'id', + 'username', 'fullname', 'first_name', 'last_name', From fee8a3b1209e165e988975e9a21c5e04842d93a0 Mon Sep 17 00:00:00 2001 From: dormantman Date: Tue, 3 Dec 2019 17:16:02 +0300 Subject: [PATCH 08/17] Correction of finding the center of points with positive --- apps/search_indexes/filters.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 4537124d..ce77bc06 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -21,15 +21,9 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): else: result_part = 180 - (180 - first[1] - diff) - elif second[1] < 0 > first[1]: - diff = abs(abs(second[1]) - abs(first[1])) - - if diff > 90: - reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) - result_part = (reverse_first + reverse_second) / 2 - - else: - result_part = (first[1] + second[1]) / 2 + elif second[1] < 0 > first[1] or second[1] > 0 < first[1]: + reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) + result_part = ((reverse_first + reverse_second) / 2) * (-1 + (second[1] < 0) * 2) else: result_part = (first[1] + second[1]) / 2 From 7cf34f0741738ac03c16980e001c46adc38f1668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Tue, 3 Dec 2019 18:30:33 +0300 Subject: [PATCH 09/17] fix --- apps/main/serializers.py | 69 ++++++++++++---------------------------- apps/main/views/back.py | 4 +-- 2 files changed, 23 insertions(+), 50 deletions(-) diff --git a/apps/main/serializers.py b/apps/main/serializers.py index b93382d5..a41a643f 100644 --- a/apps/main/serializers.py +++ b/apps/main/serializers.py @@ -17,10 +17,11 @@ class FeatureSerializer(serializers.ModelSerializer): fields = ( 'id', 'slug', - 'priority' + 'priority', + 'route', + 'site_settings', ) - class CurrencySerializer(ProjectModelSerializer): """Currency serializer.""" @@ -36,20 +37,24 @@ class CurrencySerializer(ProjectModelSerializer): class SiteFeatureSerializer(serializers.ModelSerializer): - """Site feature serializer.""" + id = serializers.IntegerField(source='feature.id') + slug = serializers.CharField(source='feature.slug') + priority = serializers.IntegerField(source='feature.priority') + route = serializers.CharField(source='feature.route.name') + source = serializers.IntegerField(source='feature.source') + nested = RecursiveFieldSerializer(many=True, allow_null=True) class Meta: """Meta class.""" - model = models.SiteFeature - fields = ( - 'id', - 'site_settings', - 'feature', - 'published', - 'main', - 'nested' - ) + fields = ('main', + 'id', + 'slug', + 'priority', + 'route', + 'source', + 'nested', + ) class SiteSettingsSerializer(serializers.ModelSerializer): @@ -95,23 +100,15 @@ class SiteSettingsBackOfficeSerializer(SiteSettingsSerializer): ] -class SiteSerializer(serializers.ModelSerializer): +class SiteSerializer(SiteSettingsSerializer): country = CountrySerializer() class Meta: """Meta class.""" model = models.SiteSettings - fields = [ - 'subdomain', - 'site_url', - 'country', - 'default_site', - 'pinterest_page_url', - 'twitter_page_url', - 'facebook_page_url', - 'instagram_page_url', - 'contact_email', - 'currency' + fields = SiteSettingsSerializer.Meta.fields + [ + 'id', + 'country' ] @@ -125,30 +122,6 @@ class SiteShortSerializer(serializers.ModelSerializer): ] -class SiteBackOfficeSerializer(SiteSerializer): - """Serializer for back office.""" - - class Meta(SiteSerializer.Meta): - """Meta class.""" - fields = SiteSerializer.Meta.fields + [ - 'id', - ] - - -class FeatureSerializer(serializers.ModelSerializer): - """Feature serializer.""" - - class Meta: - """Meta class.""" - - model = models.Feature - fields = ( - 'id', - 'slug', - 'priority', - 'route', - 'site_settings', - ) class AwardBaseSerializer(serializers.ModelSerializer): diff --git a/apps/main/views/back.py b/apps/main/views/back.py index adc0196a..1faf79a0 100644 --- a/apps/main/views/back.py +++ b/apps/main/views/back.py @@ -61,9 +61,9 @@ class SiteFeatureRUDBackView(generics.RetrieveUpdateDestroyAPIView): class SiteSettingsBackOfficeView(SiteSettingsView): """Site settings View.""" - serializer_class = serializers.SiteSettingsBackOfficeSerializer + serializer_class = serializers.SiteSerializer class SiteListBackOfficeView(SiteListView): """Site settings View.""" - serializer_class = serializers.SiteBackOfficeSerializer + serializer_class = serializers.SiteSerializer From 848008cae13c05b15b04cebd3540174529487584 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 19:02:41 +0300 Subject: [PATCH 10/17] tags dynamic filters --- apps/search_indexes/filters.py | 122 ++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 23 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index ce77bc06..3101fdc2 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -4,6 +4,7 @@ from django_elasticsearch_dsl_drf.filter_backends import SearchFilterBackend, \ FacetedSearchFilterBackend, GeoSpatialFilteringFilterBackend from search_indexes.utils import OBJECT_FIELD_PROPERTIES from six import iteritems +from functools import reduce class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @@ -56,10 +57,30 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): :param view: :return: """ - def makefilter(cur_facet): - def myfilter(x): + def make_filter(cur_facet): + def _filter(x): return cur_facet['facet']._params['field'] != next(iter(x._params)) - return myfilter + return _filter + + def make_tags_filter(cur_facet, tags_to_remove_ids): + def _filter(x): + if hasattr(x, '_params') and (x._params.get('must') or x._params.get('should')): + ret = [] + for t in ['must', 'should']: + terms = x._params.get(t) + if terms: + for term in terms: + if cur_facet['facet']._params['field'] != next(iter(term._params)): + return True # different fields. preserve filter + else: + ret.append(next(iter(term._params.values())) not in tags_to_remove_ids) + return all(ret) + if cur_facet['facet']._params['field'] != next(iter(x._params)): + return True # different fields. preserve filter + else: + return next(iter(x._params.values())) not in tags_to_remove_ids + return _filter + __facets = self.construct_facets(request, view) setattr(view.paginator, 'facets_computed', {}) for __field, __facet in iteritems(__facets): @@ -71,29 +92,84 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): 'global' ).bucket(__field, agg) else: - qs = queryset.__copy__() - qs.query = queryset.query._clone() - filterer = makefilter(__facet) - for param_type in ['must', 'must_not', 'should']: - if qs.query._proxied._params.get(param_type): - qs.query._proxied._params[param_type] = list( - filter( - filterer, qs.query._proxied._params[param_type] + if __field != 'tag' or not request.query_params.getlist('tags_id__in'): + qs = queryset.__copy__() + qs.query = queryset.query._clone() + filterer = make_filter(__facet) + for param_type in ['must', 'must_not', 'should']: + if qs.query._proxied._params.get(param_type): + qs.query._proxied._params[param_type] = list( + filter( + filterer, qs.query._proxied._params[param_type] + ) ) - ) - sh = qs.query._proxied._params.get('should') - if (not sh or not len(sh)) \ - and qs.query._proxied._params.get('minimum_should_match'): - qs.query._proxied._params.pop('minimum_should_match') - facet_name = '_filter_' + __field - qs.aggs.bucket( - facet_name, - 'filter', - filter=agg_filter - ).bucket(__field, agg) - view.paginator.facets_computed.update({facet_name: qs.execute().aggregations[facet_name]}) + sh = qs.query._proxied._params.get('should') + if (not sh or not len(sh)) \ + and qs.query._proxied._params.get('minimum_should_match'): + qs.query._proxied._params.pop('minimum_should_match') + facet_name = '_filter_' + __field + qs.aggs.bucket( + facet_name, + 'filter', + filter=agg_filter + ).bucket(__field, agg) + view.paginator.facets_computed.update({facet_name: qs.execute().aggregations[facet_name]}) + else: + tag_facets = [] + facet_name = '_filter_' + __field + for category_tags_ids in request.query_params.getlist('tags_id__in'): + tags_to_remove = category_tags_ids.split('__') + qs = queryset.__copy__() + qs.query = queryset.query._clone() + filterer = make_tags_filter(__facet, tags_to_remove) + for param_type in ['must', 'should']: + if qs.query._proxied._params.get(param_type): + if qs.query._proxied._params.get(param_type): + qs.query._proxied._params[param_type] = list( + filter( + filterer, qs.query._proxied._params[param_type] + ) + ) + sh = qs.query._proxied._params.get('should') + if (not sh or not len(sh)) \ + and qs.query._proxied._params.get('minimum_should_match'): + qs.query._proxied._params.pop('minimum_should_match') + qs.aggs.bucket( + facet_name, + 'filter', + filter=agg_filter + ).bucket(__field, agg) + tag_facets.append(qs.execute().aggregations[facet_name]) + view.paginator.facets_computed.update({facet_name: self.merge_buckets(tag_facets)}) return queryset + @staticmethod + def merge_buckets(buckets: list): + """Reduces all buckets preserving class""" + result_bucket = buckets[0] + for bucket in buckets[1:]: + for tag in bucket.tag.buckets._l_: + if tag not in result_bucket.tag.buckets._l_: + result_bucket.tag.buckets._l_.append(tag) + def reducer(prev, cur): + try: + index = list(map(lambda x: x['key'], prev)).index(cur['key']) + if cur['doc_count'] < prev[index]['doc_count']: + prev[index]['doc_count'] = cur['doc_count'] + except ValueError: + prev.append(cur) + return prev + + result_bucket.tag.buckets._l_ = list(reduce( + reducer, result_bucket.tag.buckets._l_, [] + )) + result_bucket.doc_count = reduce( + lambda prev, cur: prev + cur['doc_count'], + result_bucket.tag.buckets._l_, + 0 + ) + return result_bucket + class CustomSearchFilterBackend(SearchFilterBackend): """Custom SearchFilterBackend.""" From c00075feec564c87e13cdb6fbe8da998c89c45dd Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 19:39:20 +0300 Subject: [PATCH 11/17] empty booking response --- apps/booking/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/booking/views.py b/apps/booking/views.py index 06ef9273..b3f85bff 100644 --- a/apps/booking/views.py +++ b/apps/booking/views.py @@ -34,7 +34,7 @@ class CheckWhetherBookingAvailable(generics.GenericAPIView): periods = response['periods'] periods_by_name = {period['period']: period for period in periods if 'period' in period} if not periods_by_name: - return None + return response period_template = iter(periods_by_name.values()).__next__().copy() period_template.pop('total_left_seats') From 013cd864c29d621a1e25cddb821d384e5fa6a935 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 20:48:35 +0300 Subject: [PATCH 12/17] tags dynamic filters #2 --- apps/search_indexes/filters.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 3101fdc2..a12c37f7 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -149,25 +149,23 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): result_bucket = buckets[0] for bucket in buckets[1:]: for tag in bucket.tag.buckets._l_: - if tag not in result_bucket.tag.buckets._l_: - result_bucket.tag.buckets._l_.append(tag) + result_bucket.tag.buckets.append(tag) + def reducer(prev, cur): - try: - index = list(map(lambda x: x['key'], prev)).index(cur['key']) - if cur['doc_count'] < prev[index]['doc_count']: - prev[index]['doc_count'] = cur['doc_count'] - except ValueError: + """Unique by key""" + if not len(list(filter(lambda x: x['key'] == cur['key'], prev))): prev.append(cur) return prev - result_bucket.tag.buckets._l_ = list(reduce( - reducer, result_bucket.tag.buckets._l_, [] - )) - result_bucket.doc_count = reduce( - lambda prev, cur: prev + cur['doc_count'], + buckets_count = len(buckets) + result_bucket.tag.buckets = list(filter(lambda t: t is not None, [ + tag if len(list(filter(lambda t: t['key'] == tag['key'], result_bucket.tag.buckets._l_))) == buckets_count else None + for tag in result_bucket.tag.buckets._l_])) # here we remove tags which don't present in any bucket + result_bucket.tag.buckets = list(reduce( + reducer, result_bucket.tag.buckets._l_, - 0 - ) + [] + )) return result_bucket From 9c84842c111fd5a65a970369f36ad34ed2d6344c Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 22:35:38 +0300 Subject: [PATCH 13/17] tags dynamic filters #3 --- apps/search_indexes/filters.py | 37 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index a12c37f7..ec67fe7a 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -4,7 +4,7 @@ from django_elasticsearch_dsl_drf.filter_backends import SearchFilterBackend, \ FacetedSearchFilterBackend, GeoSpatialFilteringFilterBackend from search_indexes.utils import OBJECT_FIELD_PROPERTIES from six import iteritems -from functools import reduce +from tag.models import TagCategory class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @@ -92,7 +92,7 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): 'global' ).bucket(__field, agg) else: - if __field != 'tag' or not request.query_params.getlist('tags_id__in'): + if __field != 'tag': qs = queryset.__copy__() qs.query = queryset.query._clone() filterer = make_filter(__facet) @@ -116,9 +116,11 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): view.paginator.facets_computed.update({facet_name: qs.execute().aggregations[facet_name]}) else: tag_facets = [] + preserve_ids = [] facet_name = '_filter_' + __field - for category_tags_ids in request.query_params.getlist('tags_id__in'): - tags_to_remove = category_tags_ids.split('__') + for category in TagCategory.objects.prefetch_related('tags').filter(public=True, + value_type=TagCategory.LIST): + tags_to_remove = list(map(lambda t: t.id, category.tags.all())) qs = queryset.__copy__() qs.query = queryset.query._clone() filterer = make_tags_filter(__facet, tags_to_remove) @@ -140,32 +142,19 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): filter=agg_filter ).bucket(__field, agg) tag_facets.append(qs.execute().aggregations[facet_name]) - view.paginator.facets_computed.update({facet_name: self.merge_buckets(tag_facets)}) + preserve_ids.append(tags_to_remove) + view.paginator.facets_computed.update({facet_name: self.merge_buckets(tag_facets, preserve_ids)}) return queryset @staticmethod - def merge_buckets(buckets: list): + def merge_buckets(buckets: list, presrve_ids: list): """Reduces all buckets preserving class""" result_bucket = buckets[0] - for bucket in buckets[1:]: + result_bucket.tag.buckets = list(filter(lambda x: x in presrve_ids[0], result_bucket.tag.buckets._l_)) + for bucket, ids in list(zip(buckets, presrve_ids))[1:]: for tag in bucket.tag.buckets._l_: - result_bucket.tag.buckets.append(tag) - - def reducer(prev, cur): - """Unique by key""" - if not len(list(filter(lambda x: x['key'] == cur['key'], prev))): - prev.append(cur) - return prev - - buckets_count = len(buckets) - result_bucket.tag.buckets = list(filter(lambda t: t is not None, [ - tag if len(list(filter(lambda t: t['key'] == tag['key'], result_bucket.tag.buckets._l_))) == buckets_count else None - for tag in result_bucket.tag.buckets._l_])) # here we remove tags which don't present in any bucket - result_bucket.tag.buckets = list(reduce( - reducer, - result_bucket.tag.buckets._l_, - [] - )) + if tag['key'] in ids: + result_bucket.tag.buckets.append(tag) return result_bucket From eaf5c41e1db19fbfc60246f2eef04a55be0b8b3b Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 22:53:31 +0300 Subject: [PATCH 14/17] tags dynamic filters #4 --- apps/search_indexes/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index ec67fe7a..8040e4b4 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -150,7 +150,7 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): def merge_buckets(buckets: list, presrve_ids: list): """Reduces all buckets preserving class""" result_bucket = buckets[0] - result_bucket.tag.buckets = list(filter(lambda x: x in presrve_ids[0], result_bucket.tag.buckets._l_)) + result_bucket.tag.buckets = list(filter(lambda x: x['key'] in presrve_ids[0], result_bucket.tag.buckets._l_)) for bucket, ids in list(zip(buckets, presrve_ids))[1:]: for tag in bucket.tag.buckets._l_: if tag['key'] in ids: From f9fb2aa17ea661f7a1aae97872d624b9f8e0bd5b Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 3 Dec 2019 23:49:45 +0300 Subject: [PATCH 15/17] tags dynamic filters #4 --- apps/search_indexes/filters.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 8040e4b4..bc48b029 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -120,7 +120,7 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): facet_name = '_filter_' + __field for category in TagCategory.objects.prefetch_related('tags').filter(public=True, value_type=TagCategory.LIST): - tags_to_remove = list(map(lambda t: t.id, category.tags.all())) + tags_to_remove = list(map(lambda t: str(t.id), category.tags.all())) qs = queryset.__copy__() qs.query = queryset.query._clone() filterer = make_tags_filter(__facet, tags_to_remove) @@ -142,16 +142,16 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): filter=agg_filter ).bucket(__field, agg) tag_facets.append(qs.execute().aggregations[facet_name]) - preserve_ids.append(tags_to_remove) + preserve_ids.append(list(map(int, tags_to_remove))) view.paginator.facets_computed.update({facet_name: self.merge_buckets(tag_facets, preserve_ids)}) return queryset @staticmethod - def merge_buckets(buckets: list, presrve_ids: list): + def merge_buckets(buckets: list, preserve_ids: list): """Reduces all buckets preserving class""" result_bucket = buckets[0] - result_bucket.tag.buckets = list(filter(lambda x: x['key'] in presrve_ids[0], result_bucket.tag.buckets._l_)) - for bucket, ids in list(zip(buckets, presrve_ids))[1:]: + result_bucket.tag.buckets = list(filter(lambda x: x['key'] in preserve_ids[0], result_bucket.tag.buckets._l_)) + for bucket, ids in list(zip(buckets, preserve_ids))[1:]: for tag in bucket.tag.buckets._l_: if tag['key'] in ids: result_bucket.tag.buckets.append(tag) From 9f882ca65643aa0455b2892737f89588f51d2107 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 00:33:18 +0300 Subject: [PATCH 16/17] tags dynamic filters #4 (boost search via ES) --- apps/search_indexes/documents/__init__.py | 2 ++ apps/search_indexes/documents/tag_category.py | 33 +++++++++++++++++++ apps/search_indexes/filters.py | 9 +++-- project/settings/development.py | 1 + project/settings/local.py | 1 + project/settings/production.py | 1 + project/settings/stage.py | 1 + 7 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 apps/search_indexes/documents/tag_category.py diff --git a/apps/search_indexes/documents/__init__.py b/apps/search_indexes/documents/__init__.py index c357f29e..60a9b580 100644 --- a/apps/search_indexes/documents/__init__.py +++ b/apps/search_indexes/documents/__init__.py @@ -1,6 +1,7 @@ from search_indexes.documents.establishment import EstablishmentDocument from search_indexes.documents.news import NewsDocument from search_indexes.documents.product import ProductDocument +from search_indexes.documents.tag_category import TagCategoryDocument from search_indexes.tasks import es_update # todo: make signal to update documents on related fields @@ -8,5 +9,6 @@ __all__ = [ 'EstablishmentDocument', 'NewsDocument', 'ProductDocument', + 'TagCategoryDocument', 'es_update', ] \ No newline at end of file diff --git a/apps/search_indexes/documents/tag_category.py b/apps/search_indexes/documents/tag_category.py new file mode 100644 index 00000000..ed69c8b7 --- /dev/null +++ b/apps/search_indexes/documents/tag_category.py @@ -0,0 +1,33 @@ +"""Product app documents.""" +from django.conf import settings +from django_elasticsearch_dsl import Document, Index, fields +from tag import models + +TagCategoryIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, 'tag_category')) +TagCategoryIndex.settings(number_of_shards=2, number_of_replicas=2) + + +@TagCategoryIndex.doc_type +class TagCategoryDocument(Document): + """TagCategory document.""" + + tags = fields.ListField(fields.ObjectField( + properties={ + 'id': fields.IntegerField(), + 'value': fields.KeywordField(), + }, + )) + + class Django: + model = models.TagCategory + fields = ( + 'id', + 'index_name', + 'public', + 'value_type' + ) + related_models = [models.Tag] + + + def get_queryset(self): + return super().get_queryset().with_base_related() diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index bc48b029..27df0699 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -4,6 +4,7 @@ from django_elasticsearch_dsl_drf.filter_backends import SearchFilterBackend, \ FacetedSearchFilterBackend, GeoSpatialFilteringFilterBackend from search_indexes.utils import OBJECT_FIELD_PROPERTIES from six import iteritems +from search_indexes.documents import TagCategoryDocument from tag.models import TagCategory @@ -118,9 +119,11 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): tag_facets = [] preserve_ids = [] facet_name = '_filter_' + __field - for category in TagCategory.objects.prefetch_related('tags').filter(public=True, - value_type=TagCategory.LIST): - tags_to_remove = list(map(lambda t: str(t.id), category.tags.all())) + all_tag_categories = TagCategoryDocument.search() \ + .filter('term', public=True) \ + .filter('term', value_type=TagCategory.LIST) + for category in all_tag_categories: + tags_to_remove = list(map(lambda t: str(t.id), category.tags)) qs = queryset.__copy__() qs.query = queryset.query._clone() filterer = make_tags_filter(__facet, tags_to_remove) diff --git a/project/settings/development.py b/project/settings/development.py index f850aad7..88b88789 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -42,6 +42,7 @@ ELASTICSEARCH_INDEX_NAMES = { 'search_indexes.documents.news': 'development_news', 'search_indexes.documents.establishment': 'development_establishment', 'search_indexes.documents.product': 'development_product', + 'search_indexes.documents.tag_category': 'development_tag_category', } # ELASTICSEARCH_DSL_AUTOSYNC = False diff --git a/project/settings/local.py b/project/settings/local.py index 6a592a46..644098bb 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -92,6 +92,7 @@ ELASTICSEARCH_INDEX_NAMES = { # 'search_indexes.documents.news': 'local_news', 'search_indexes.documents.establishment': 'local_establishment', 'search_indexes.documents.product': 'local_product', + 'search_indexes.documents.tag_category': 'local_tag_category', } ELASTICSEARCH_DSL_AUTOSYNC = False diff --git a/project/settings/production.py b/project/settings/production.py index 7ef2dc62..3020c6f0 100644 --- a/project/settings/production.py +++ b/project/settings/production.py @@ -36,6 +36,7 @@ ELASTICSEARCH_INDEX_NAMES = { 'search_indexes.documents.news': 'development_news', # temporarily disabled 'search_indexes.documents.establishment': 'development_establishment', 'search_indexes.documents.product': 'development_product', + 'search_indexes.documents.tag_category': 'development_tag_category', } sentry_sdk.init( diff --git a/project/settings/stage.py b/project/settings/stage.py index 95285034..dbc6844a 100644 --- a/project/settings/stage.py +++ b/project/settings/stage.py @@ -23,6 +23,7 @@ ELASTICSEARCH_DSL = { ELASTICSEARCH_INDEX_NAMES = { # 'search_indexes.documents.news': 'stage_news', #temporarily disabled 'search_indexes.documents.establishment': 'stage_establishment', + 'search_indexes.documents.tag_category': 'stage_tag_category', } COOKIE_DOMAIN = '.id-east.ru' From f5079940148d135ba361a0dbc084a3aa31588075 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 00:49:46 +0300 Subject: [PATCH 17/17] tags dynamic filters #5 (add wine-colors) --- apps/search_indexes/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 27df0699..644b47fd 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -121,7 +121,7 @@ class CustomFacetedSearchFilterBackend(FacetedSearchFilterBackend): facet_name = '_filter_' + __field all_tag_categories = TagCategoryDocument.search() \ .filter('term', public=True) \ - .filter('term', value_type=TagCategory.LIST) + .filter(Q('term', value_type=TagCategory.LIST) | Q('match', index_name='wine-color')) for category in all_tag_categories: tags_to_remove = list(map(lambda t: str(t.id), category.tags)) qs = queryset.__copy__()