From 076d14fd5bf29eef97d09a0dc783ca0fbec904c4 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Mon, 2 Dec 2019 13:51:48 +0300 Subject: [PATCH] added filter model --- .gitignore | 2 +- .../0019_advertorial_guidefilter.py | 41 +++++++ apps/collection/models.py | 28 +++-- apps/collection/transfer_data.py | 12 ++- apps/transfer/mixins.py | 6 ++ apps/transfer/models.py | 2 +- apps/transfer/serializers/guide.py | 102 ++++++++++-------- 7 files changed, 127 insertions(+), 66 deletions(-) create mode 100644 apps/collection/migrations/0019_advertorial_guidefilter.py diff --git a/.gitignore b/.gitignore index 1bb5c4b0..90e4f23f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,9 @@ logs/ # dev ./docker-compose.override.yml - celerybeat-schedule local_files celerybeat.pid /gm_viktor.dump /docker-compose.dump.yml +/gm_production_20191029.sql diff --git a/apps/collection/migrations/0019_advertorial_guidefilter.py b/apps/collection/migrations/0019_advertorial_guidefilter.py new file mode 100644 index 00000000..180a1e0e --- /dev/null +++ b/apps/collection/migrations/0019_advertorial_guidefilter.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.7 on 2019-12-02 10:11 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('collection', '0018_auto_20191127_1047'), + ] + + operations = [ + migrations.CreateModel( + name='GuideFilter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('establishment_type_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='establishment types')), + ('country_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='countries')), + ('region_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='regions')), + ('sub_region_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='sub regions')), + ('wine_region_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='wine regions')), + ('with_mark', models.BooleanField(default=True, help_text='exclude empty marks?', verbose_name='with mark')), + ('locale_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='locales')), + ('max_mark', models.FloatField(help_text='mark under', null=True, verbose_name='max mark')), + ('min_mark', models.FloatField(help_text='mark over', null=True, verbose_name='min mark')), + ('review_vintage_json', django.contrib.postgres.fields.jsonb.JSONField(verbose_name='review vintage years')), + ('review_state_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='review states')), + ('old_id', models.IntegerField(blank=True, null=True)), + ('guide', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='collection.Guide', verbose_name='guide')), + ], + options={ + 'verbose_name': 'guide filter', + 'verbose_name_plural': 'guide filters', + }, + ), + ] diff --git a/apps/collection/models.py b/apps/collection/models.py index af8b2a33..7d037a85 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -188,29 +188,25 @@ class GuideFilter(ProjectBaseMixin): """Guide filter model.""" establishment_type_json = JSONField(blank=True, null=True, verbose_name='establishment types') - country_code_json = JSONField(blank=True, null=True, - verbose_name='countries') - region_code_json = JSONField(blank=True, null=True, - verbose_name='regions') - sub_region_code_json = JSONField(blank=True, null=True, - verbose_name='sub regions') + country_json = JSONField(blank=True, null=True, + verbose_name='countries') + region_json = JSONField(blank=True, null=True, + verbose_name='regions') + sub_region_json = JSONField(blank=True, null=True, + verbose_name='sub regions') wine_region_json = JSONField(blank=True, null=True, verbose_name='wine regions') - wine_classification_json = JSONField(blank=True, null=True, - verbose_name='wine classifications') - wine_color_json = JSONField(blank=True, null=True, - verbose_name='wine colors') - wine_type_json = JSONField(blank=True, null=True, - verbose_name='wine types') with_mark = models.BooleanField(default=True, verbose_name=_('with mark'), help_text=_('exclude empty marks?')) locale_json = JSONField(blank=True, null=True, verbose_name='locales') - max_mark = models.PositiveSmallIntegerField(verbose_name=_('max mark'), - help_text=_('mark under')) - min_mark = models.PositiveSmallIntegerField(verbose_name=_('min mark'), - help_text=_('mark over')) + max_mark = models.FloatField(verbose_name=_('max mark'), + null=True, + help_text=_('mark under')) + min_mark = models.FloatField(verbose_name=_('min mark'), + null=True, + help_text=_('mark over')) review_vintage_json = JSONField(verbose_name='review vintage years') review_state_json = JSONField(blank=True, null=True, verbose_name='review states') diff --git a/apps/collection/transfer_data.py b/apps/collection/transfer_data.py index 9307e6a0..9eeec541 100644 --- a/apps/collection/transfer_data.py +++ b/apps/collection/transfer_data.py @@ -5,6 +5,7 @@ from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer def transfer_guide(): """Transfer Guide model.""" + errors = [] queryset = Guides.objects.exclude(title__icontains='test') serialized_data = GuideSerializer( data=list(queryset.values()), @@ -12,19 +13,24 @@ def transfer_guide(): if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"transfer guide errors: {serialized_data.errors}") + for d in serialized_data.errors: errors.append(d) if d else None + pprint(f"transfer_guide errors: {errors}") def transfer_guide_filter(): """Transfer GuideFilter model.""" - queryset = GuideFilters.objects.all() + errors = [] + queryset = GuideFilters.objects.exclude(guide__title__icontains='test') \ + .exclude(guide__id__isnull=True) serialized_data = GuideFilterSerializer( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"transfer guide filter errors: {serialized_data.errors}") + for d in serialized_data.errors: errors.append(d) if d else None + pprint(f"transfer_guide_filter errors: {errors}\n" + f"COUNT: {len(errors)}") data_types = { diff --git a/apps/transfer/mixins.py b/apps/transfer/mixins.py index 7b43b210..30537a14 100644 --- a/apps/transfer/mixins.py +++ b/apps/transfer/mixins.py @@ -39,6 +39,12 @@ class TransferSerializerMixin(serializers.ModelSerializer): qs = self.Meta.model.objects.filter(**validated_data) if not qs.exists(): return super().create(validated_data) + # try: + # qs = self.Meta.model.objects.filter(**validated_data) + # if not qs.exists(): + # return super().create(validated_data) + # except Exception: + # breakpoint() @property def tag_category(self): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 1ac40e80..fa3d898b 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -370,7 +370,7 @@ class GuideFilters(MigrateMixin): states = models.CharField(max_length=255, blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() - guide_id = models.IntegerField(blank=True, null=True) + guide = models.ForeignKey(Guides, models.DO_NOTHING, blank=True, null=True) class Meta: managed = False diff --git a/apps/transfer/serializers/guide.py b/apps/transfer/serializers/guide.py index 3fd5cc32..c49e34b2 100644 --- a/apps/transfer/serializers/guide.py +++ b/apps/transfer/serializers/guide.py @@ -1,5 +1,5 @@ from itertools import chain - +from django.utils.text import slugify import yaml from pycountry import countries, subdivisions from rest_framework import serializers @@ -66,7 +66,7 @@ class GuideSerializer(TransferSerializerMixin): class GuideFilterSerializer(TransferSerializerMixin): id = serializers.IntegerField() - year = serializers.CharField() + year = serializers.CharField(allow_null=True) establishment_type = serializers.CharField(allow_null=True) countries = serializers.CharField(allow_null=True) regions = serializers.CharField(allow_null=True) @@ -75,8 +75,8 @@ class GuideFilterSerializer(TransferSerializerMixin): locales = serializers.CharField(allow_null=True) states = serializers.CharField(allow_null=True) - max_mark = serializers.IntegerField(allow_null=True) - min_mark = serializers.IntegerField(allow_null=True) + max_mark = serializers.FloatField(allow_null=True) + min_mark = serializers.FloatField(allow_null=True) marks_only = serializers.NullBooleanField() guide_id = serializers.IntegerField() @@ -149,25 +149,33 @@ class GuideFilterSerializer(TransferSerializerMixin): country_code_alpha_3: str, sub_region_code_alpha_3: str = None): country = self.get_country(country_code_alpha_3) - country_code_alpha_2 = country.code.upper() - region_qs = Region.objects.filter(code__iexact=region_code_alpha_3, - country__code__iexact=country_code_alpha_2) + if country: + country_code_alpha_2 = country.code.upper() + region_qs = Region.objects.filter(code__iexact=region_code_alpha_3, + country__code__iexact=country_code_alpha_2) - if region_qs.exists(): - return region_qs.first() + if region_qs.exists(): + return region_qs.first() - # If region isn't existed, check sub region for parent_code (region code) - if sub_region_code_alpha_3: - # sub region - subdivision = subdivisions.get( - code=f"{country_code_alpha_2}-{sub_region_code_alpha_3}") - if subdivision: - subdivision_region = subdivisions.get(code=subdivision.parent_code) - obj = Region.objects.create( - name=subdivision_region.name, - code=subdivision_region.code, - country=country) - return obj + # If region isn't existed, check sub region for parent_code (region code) + if sub_region_code_alpha_3: + # sub region + subdivision = subdivisions.get( + code=f"{country_code_alpha_2}-{sub_region_code_alpha_3}") + if subdivision: + # try with parent code + subdivision_region = subdivisions.get(code=subdivision.__dict__.get('_fields') + .get('parent_code')) + if not subdivision_region: + # try with parent + subdivision_region = subdivisions.get(code=subdivision.__dict__.get('_fields') + .get('parent')) + if subdivision_region: + obj = Region.objects.create( + name=subdivision_region.name, + code=subdivision_region.code, + country=country) + return obj def validate_year(self, value): return self.parse_ruby_helper(value) @@ -210,12 +218,12 @@ class GuideFilterSerializer(TransferSerializerMixin): attrs['review_vintage_json'] = self.get_review_vintage(attrs.pop('year')) attrs['establishment_type_json'] = self.get_establishment_type_ids( attrs.pop('establishment_type')) - attrs['country_code_json'] = self.get_country_ids(attrs.pop('countries')) + attrs['country_json'] = self.get_country_ids(attrs.pop('countries')) attrs['region_json'] = self.get_region_ids(regions=regions, sub_regions=sub_regions) attrs['sub_region_json'] = self.get_sub_region_ids(sub_regions) attrs['wine_region_json'] = self.get_wine_region_ids(attrs.pop('wine_regions')) - attrs['with_mark'] = attrs.pop('marks_only') + attrs['with_mark'] = attrs.pop('marks_only') or True attrs['locale_json'] = self.get_locale_ids(attrs.pop('locales')) attrs['review_state_json'] = self.get_review_state(attrs.pop('states')) attrs['guide'] = self.get_guide(attrs.pop('guide_id')) @@ -223,8 +231,8 @@ class GuideFilterSerializer(TransferSerializerMixin): def get_review_vintage(self, year): if hasattr(year, '__iter__'): - return {'vintage': set(int(i) for i in set(year) if i.isdigit())} - return {'vintage': {year}} + return {'vintage': list(set(int(i) for i in set(year) if i.isdigit()))} + return {'vintage': [year, ]} def get_establishment_type_ids(self, establishment_types): establishment_type_ids = [] @@ -234,22 +242,24 @@ class GuideFilterSerializer(TransferSerializerMixin): if not establishment_type_qs.exists(): obj = EstablishmentType.objects.create( name={'en-GB': establishment_type.capitalize()}, - index_name=establishment_type.lower()) + index_name=slugify(establishment_type)) else: obj = establishment_type_qs.first() establishment_type_ids.append(obj.id) - return {'id': set(establishment_type_ids)} + return {'id': list(set(establishment_type_ids))} def get_country_ids(self, country_codes_alpha_3): country_ids = [] if country_codes_alpha_3: for code_alpha_3 in country_codes_alpha_3: - country = self.get_country(code_alpha_3) - if not country: - raise serializers.ValidationError({'detail': f'Country with alpha code 3 -' - f'{code_alpha_3}, is not found.'}) - country_ids.append(country.id) - return {'id': set(country_ids)} + # Code can be an empty string. + if code_alpha_3 and not code_alpha_3 == 'AAA': + country = self.get_country(code_alpha_3) + if not country: + raise serializers.ValidationError({'detail': f'Country with alpha code 3 -' + f'{code_alpha_3}, is not found.'}) + country_ids.append(country.id) + return {'id': list(set(country_ids))} def get_region_ids(self, regions, sub_regions): region_ids = [] @@ -266,7 +276,7 @@ class GuideFilterSerializer(TransferSerializerMixin): sub_region_code_alpha_3=sub_region_code_alpha_3) if region: region_ids.append(region.id) - return {'id': region_ids} + return {'id': list(set(region_ids))} def get_sub_region_ids(self, sub_regions): sub_region_ids = [] @@ -291,13 +301,13 @@ class GuideFilterSerializer(TransferSerializerMixin): else: subdivision = subdivisions.get(code=region.code.upper()) if subdivision: - sub_region = Region.objects.get_or_create( + sub_region, _ = Region.objects.get_or_create( name=subdivision.name, - code=subdivisions.parent_code, + code=subdivision.code, parent_region=region, country=region.country) sub_region_ids.append(sub_region.id) - return {'id': set(sub_region_ids)} + return {'id': list(set(sub_region_ids))} def get_wine_region_ids(self, wine_regions): wine_region_ids = [] @@ -308,17 +318,21 @@ class GuideFilterSerializer(TransferSerializerMixin): raise serializers.ValidationError({ 'detail': f'Wine region - {wine_region}, is not found.'}) wine_region_ids.append(qs.first().id) - return {'id': set(wine_region_ids)} + return {'id': list(set(wine_region_ids))} def get_locale_ids(self, locales): locale_ids = [] if locales: for locale in [locale for locale in locales if locale]: - qs = Language.objects.filter(locale=locale) + if len(locale) == 2: + qs = Language.objects.filter(locale__startswith=locale) + else: + qs = Language.objects.filter(locale=locale) if not qs.exists(): raise serializers.ValidationError({ 'detail': f'Language with locale - {locale}, is not found.'}) - return {'id': set(locale_ids)} + locale_ids.extend(qs.values_list('id', flat=True)) + return {'id': list(set(locale_ids))} def get_review_state(self, states): review_states = [] @@ -328,11 +342,9 @@ class GuideFilterSerializer(TransferSerializerMixin): review_states.append(Review.READY) else: review_states.append(Review.TO_INVESTIGATE) - return {'state': set(review_states)} + return {'state': list(set(review_states))} def get_guide(self, old_guide_id: int): qs = Guide.objects.filter(old_id=old_guide_id) - if not qs.exists(): - raise serializers.ValidationError({'detail': f'Guide with old_id - {old_guide_id}, ' - f'is not found.'}) - return qs.first() + if qs.exists(): + return qs.first()