From 68b8aa427cc1e9ca7512cc600a16425872edf39d Mon Sep 17 00:00:00 2001 From: Anatoly Date: Thu, 19 Dec 2019 16:27:58 +0300 Subject: [PATCH] added counters --- .../0028_guide_initial_objects_counter.py | 18 +++ apps/collection/models.py | 142 ++++++++++++++++-- apps/collection/serializers/common.py | 13 +- apps/collection/transfer_data.py | 39 +++-- apps/collection/views/back.py | 10 +- 5 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 apps/collection/migrations/0028_guide_initial_objects_counter.py diff --git a/apps/collection/migrations/0028_guide_initial_objects_counter.py b/apps/collection/migrations/0028_guide_initial_objects_counter.py new file mode 100644 index 00000000..21f6649c --- /dev/null +++ b/apps/collection/migrations/0028_guide_initial_objects_counter.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-12-19 13:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('collection', '0027_auto_20191218_0753'), + ] + + operations = [ + migrations.AddField( + model_name='guide', + name='count_objects_during_init', + field=models.PositiveIntegerField(default=0, help_text='* after rebuild guide, refresh count of related guide elements', verbose_name='count of related guide elements during initialization'), + ), + ] diff --git a/apps/collection/models.py b/apps/collection/models.py index b3dd44ef..4b063388 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -8,8 +8,8 @@ from django.utils.translation import gettext_lazy as _ from mptt.models import MPTTModel, TreeForeignKey from location.models import Country, Region, WineRegion, WineSubRegion -from translation.models import Language from review.models import Review +from translation.models import Language from utils.models import IntermediateGalleryModelMixin, GalleryModelMixin from utils.models import ( ProjectBaseMixin, TJSONField, TranslatedFieldsMixin, @@ -178,6 +178,68 @@ class GuideQuerySet(models.QuerySet): """Return QuerySet filtered by country code.""" return self.filter(country_json__id__contains=country_id) + def annotate_in_restaurant_section(self): + """Annotate flag if GuideElement in RestaurantSectionNode.""" + return self.annotate( + in_restaurant_section=models.Case( + models.When( + guideelement__guide_element_type__name='EstablishmentNode', + guideelement__parent__guide_element_type__name='RestaurantSectionNode', + then=True), + default=False, + output_field=models.BooleanField(default=False) + ) + ) + + def annotate_in_shop_section(self): + """Annotate flag if GuideElement in ShopSectionNode.""" + return self.annotate( + in_shop_section=models.Case( + models.When( + guideelement__guide_element_type__name='EstablishmentNode', + guideelement__parent__guide_element_type__name='ShopSectionNode', + then=True), + default=False, + output_field=models.BooleanField(default=False) + ) + ) + + def annotate_restaurant_counter(self): + """Return QuerySet with annotated field - restaurant_counter.""" + return self.annotate_in_restaurant_section().annotate( + restaurant_counter=models.Count( + 'guideelement', + filter=models.Q(in_restaurant_section=True) & + models.Q(guideelement__parent_id__isnull=False), + distinct=True)) + + def annotate_shop_counter(self): + """Return QuerySet with annotated field - shop_counter.""" + return self.annotate_in_shop_section().annotate( + shop_counter=models.Count( + 'guideelement', + filter=models.Q(in_shop_section=True) & + models.Q(guideelement__parent_id__isnull=False), + distinct=True)) + + def annotate_wine_counter(self): + """Return QuerySet with annotated field - shop_counter.""" + return self.annotate_in_restaurant_section().annotate( + wine_counter=models.Count( + 'guideelement', + filter=models.Q(guideelement__guide_element_type__name='WineNode') & + models.Q(guideelement__parent_id__isnull=False), + distinct=True)) + + def annotate_present_objects_counter(self): + """Return QuerySet with annotated field - present_objects_counter.""" + return self.annotate_in_restaurant_section().annotate( + present_objects_counter=models.Count( + 'guideelement', + filter=models.Q(guideelement__guide_element_type__name__in=['EstablishmentNode', 'WineNode']) & + models.Q(guideelement__parent_id__isnull=False), + distinct=True)) + class Guide(ProjectBaseMixin, CollectionNameMixin, CollectionDateMixin): """Guide model.""" @@ -191,7 +253,6 @@ class Guide(ProjectBaseMixin, CollectionNameMixin, CollectionDateMixin): (WAITING, 'waiting'), (REMOVING, 'removing'), (BUILDING, 'building'), - ) start = models.DateTimeField(null=True, @@ -210,6 +271,10 @@ class Guide(ProjectBaseMixin, CollectionNameMixin, CollectionDateMixin): verbose_name=_('site settings')) state = models.PositiveSmallIntegerField(default=WAITING, choices=STATE_CHOICES, verbose_name=_('state')) + count_objects_during_init = models.PositiveIntegerField( + default=0, + help_text=_('* after rebuild guide, refresh count of related guide elements'), + verbose_name=_('count of related guide elements during initialization')) old_id = models.IntegerField(blank=True, null=True) objects = GuideQuerySet.as_manager() @@ -223,16 +288,45 @@ class Guide(ProjectBaseMixin, CollectionNameMixin, CollectionDateMixin): """String method.""" return f'{self.name}' - @property - def entities(self): - """Return entities and its count.""" - # todo: to work - return { - 'Current': 0, - 'Initial': 0, - 'Restaurants': 0, - 'Shops': 0, - } + # todo: for test use, use annotation instead + # @property + # def restaurant_counter_prop(self): + # counter = 0 + # root_node = GuideElement.objects.get_root_node(self) + # if root_node: + # descendants = GuideElement.objects.get_root_node(self).get_descendants() + # if descendants: + # restaurant_section_nodes = descendants.filter( + # guide_element_type__name='RestaurantSectionNode') + # counter = descendants.filter(guide_element_type__name='EstablishmentNode', + # parent__in=restaurant_section_nodes, + # parent_id__isnull=True).count() + # return counter + # + # @property + # def shop_counter_prop(self): + # counter = 0 + # root_node = GuideElement.objects.get_root_node(self) + # if root_node: + # descendants = GuideElement.objects.get_root_node(self).get_descendants() + # if descendants: + # shop_section_nodes = descendants.filter( + # guide_element_type__name='ShopSectionNode') + # counter = descendants.filter(guide_element_type__name='EstablishmentNode', + # parent__in=shop_section_nodes, + # parent_id__isnull=True).count() + # return counter + # + # @property + # def wine_counter_prop(self): + # counter = 0 + # root_node = GuideElement.objects.get_root_node(self) + # if root_node: + # descendants = GuideElement.objects.get_root_node(self).get_descendants() + # if descendants: + # counter = descendants.filter(guide_element_type__name='WineNode', + # parent_id__isnull=True).count() + # return counter class AdvertorialQuerySet(models.QuerySet): @@ -439,9 +533,31 @@ class GuideElementSection(ProjectBaseMixin): verbose_name_plural = _('guide element sections') +class GuideElementManager(models.Manager): + """Manager for model GuideElement.""" + + def get_root_node(self, guide): + """Return guide root element node.""" + qs = self.filter(guide=guide, guide_element_type__name='Root') + if qs.exists(): + return qs.first() + + class GuideElementQuerySet(models.QuerySet): """QuerySet for model Guide elements.""" + def restaurant_nodes(self): + """Return GuideElement with type RestaurantSectionNode.""" + return self.filter(guide_element_type__name='RestaurantSectionNode') + + def shop_nodes(self): + """Return GuideElement with type ShopSectionNode.""" + return self.filter(guide_element_type__name='ShopSectionNode') + + def wine_nodes(self): + """Return GuideElement with type WineNode.""" + return self.filter(guide_element_type__name='WineNode') + class GuideElement(ProjectBaseMixin, MPTTModel): """Frozen state of elements of guide instance.""" @@ -475,7 +591,7 @@ class GuideElement(ProjectBaseMixin, MPTTModel): old_id = models.PositiveIntegerField(blank=True, null=True, default=None, verbose_name=_('old id')) - objects = GuideElementQuerySet.as_manager() + objects = GuideElementManager.from_queryset(GuideElementQuerySet)() class Meta: """Meta class.""" diff --git a/apps/collection/serializers/common.py b/apps/collection/serializers/common.py index 8b8a3566..c3de708f 100644 --- a/apps/collection/serializers/common.py +++ b/apps/collection/serializers/common.py @@ -88,9 +88,13 @@ class GuideBaseSerializer(serializers.ModelSerializer): source='guide_type') site_detail = SiteShortSerializer(read_only=True, source='site') - entities = serializers.DictField(read_only=True) guide_filters = GuideFilterBaseSerialzer(read_only=True, source='guidefilter') + # counters + restaurant_counter = serializers.IntegerField(read_only=True) + shop_counter = serializers.IntegerField(read_only=True) + wine_counter = serializers.IntegerField(read_only=True) + present_objects_counter = serializers.IntegerField(read_only=True) class Meta: model = models.Guide @@ -107,8 +111,12 @@ class GuideBaseSerializer(serializers.ModelSerializer): 'site_detail', 'state', 'state_display', - 'entities', 'guide_filters', + 'restaurant_counter', + 'shop_counter', + 'wine_counter', + 'present_objects_counter', + 'count_objects_during_init', ] extra_kwargs = { 'guide_type': {'write_only': True}, @@ -116,6 +124,7 @@ class GuideBaseSerializer(serializers.ModelSerializer): 'state': {'write_only': True}, 'start': {'required': True}, 'slug': {'required': True}, + 'count_objects_during_init': {'read_only': True} } diff --git a/apps/collection/transfer_data.py b/apps/collection/transfer_data.py index a268f62b..5e796b4c 100644 --- a/apps/collection/transfer_data.py +++ b/apps/collection/transfer_data.py @@ -1,5 +1,7 @@ +import operator from pprint import pprint +from django.db.models import Subquery from tqdm import tqdm from collection.models import GuideElementSection, GuideElementSectionCategory, \ @@ -13,7 +15,6 @@ from review.models import Review from transfer.models import Guides, GuideFilters, GuideSections, GuideElements, \ GuideAds, LabelPhotos from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer -from django.db.models import Subquery def transfer_guide(): @@ -163,7 +164,7 @@ def transfer_guide_elements_bulk(): return qs.first() objects_to_update = [] - base_queryset = GuideElements.objects.all() + base_queryset = GuideElements.objects.filter(guide_id=407) for old_id, type, establishment_id, review_id, wine_region_id, \ wine_id, color, order_number, city_id, section_id, guide_id \ @@ -203,19 +204,19 @@ def transfer_guide_elements_bulk(): # create parents GuideElement.objects.bulk_create(objects_to_update) - pprint(f'CREATED PARENT GUIDE ELEMENTS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') # attach child guide elements - queryset_values = base_queryset.filter(parent_id__isnull=False) \ - .order_by('-parent_id') \ + created_child = 0 + queryset_values = base_queryset.exclude(parent_id__isnull=True) \ .values_list('id', 'type', 'establishment_id', 'review_id', 'wine_region_id', 'wine_id', 'color', 'order_number', 'city_id', - 'section_id', 'guide_id', 'parent_id') + 'section_id', 'guide_id', 'rgt', 'lft', 'parent_id') for old_id, type, establishment_id, review_id, wine_region_id, \ - wine_id, color, order_number, city_id, section_id, guide_id, parent_id \ - in tqdm(sorted(queryset_values, key=lambda value: value[len(value)-1]), + wine_id, color, order_number, city_id, section_id, guide_id, \ + lft, rgt, parent_id \ + in tqdm(sorted(queryset_values, key=lambda value: operator.itemgetter(-2, -1)(value)), desc='Check child guide elements'): if not GuideElement.objects.filter(old_id=old_id).exists(): # check old guide @@ -223,7 +224,7 @@ def transfer_guide_elements_bulk(): old_guide = Guides.objects.exclude(title__icontains='test') \ .filter(id=guide_id) if old_guide.exists(): - GuideElement.objects.create( + guide_element, created = GuideElement.objects.get_or_create( old_id=old_id, guide_element_type=get_guide_element_type(type), establishment=get_establishment(establishment_id), @@ -241,13 +242,29 @@ def transfer_guide_elements_bulk(): level=1, guide=get_guide(guide_id), ) + if created: created_child += 1 - pprint(f'CREATED CHILD GUIDE ELEMENTS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') - print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') + print(f'CREATED {created_child} OBJECTS') # rebuild trees GuideElement._tree_manager.rebuild() + # record the total count of descendants objects + objects_to_update = [] + + for guide in tqdm(Guide.objects.all(), + desc='update count_of_initial_objects field values'): + count_of_initial_objects = GuideElement.objects.get_root_node(guide) \ + .get_descendants() \ + .filter(guide_element_type__name__in=['EstablishmentNode', 'WineNode']) \ + .count() + guide.count_objects_during_init = count_of_initial_objects + objects_to_update.append(guide) + + # update count_of_initial_objects field values + Guide.objects.bulk_update(objects_to_update, ['count_objects_during_init', ]) + print(f'COUNT OF UPDATED OBJECTS: {len(objects_to_update)}') + def transfer_guide_element_advertorials(): """Transfer Guide Advertorials model.""" diff --git a/apps/collection/views/back.py b/apps/collection/views/back.py index 278e1691..14107f83 100644 --- a/apps/collection/views/back.py +++ b/apps/collection/views/back.py @@ -7,7 +7,6 @@ from rest_framework.response import Response from collection import models, serializers from utils.views import BindObjectMixin -from django.db.models import Prefetch class CollectionViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): @@ -27,10 +26,17 @@ class CollectionViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): class GuideBaseView(generics.GenericAPIView): """ViewSet for Guide model.""" - queryset = models.Guide.objects.with_base_related() serializer_class = serializers.GuideBaseSerializer permission_classes = (permissions.IsAuthenticated, ) + def get_queryset(self): + """Overridden get_queryset method.""" + return models.Guide.objects.with_base_related() \ + .annotate_restaurant_counter() \ + .annotate_shop_counter() \ + .annotate_wine_counter() \ + .annotate_present_objects_counter() + class GuideFilterBaseView(generics.GenericAPIView): """ViewSet for GuideFilter model."""