From eaeebae72a1655f4c3a0b0d2cd6c909c65190a76 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Thu, 28 Nov 2019 15:31:47 +0300 Subject: [PATCH 01/27] in progress --- .../commands/check_guide_dependencies.py | 120 ++++++++ .../migrations/0018_auto_20191127_1047.py | 78 +++++ apps/collection/models.py | 134 ++++++++- apps/collection/transfer_data.py | 37 +++ apps/transfer/management/commands/transfer.py | 2 + apps/transfer/models.py | 8 + apps/transfer/serializers/guide.py | 283 ++++++++++++++++++ requirements/base.txt | 3 + 8 files changed, 652 insertions(+), 13 deletions(-) create mode 100644 apps/collection/management/commands/check_guide_dependencies.py create mode 100644 apps/collection/migrations/0018_auto_20191127_1047.py create mode 100644 apps/collection/transfer_data.py create mode 100644 apps/transfer/serializers/guide.py diff --git a/apps/collection/management/commands/check_guide_dependencies.py b/apps/collection/management/commands/check_guide_dependencies.py new file mode 100644 index 00000000..1f2afb9d --- /dev/null +++ b/apps/collection/management/commands/check_guide_dependencies.py @@ -0,0 +1,120 @@ +import re +from pprint import pprint + +from django.core.management.base import BaseCommand +from tqdm import tqdm + +from establishment.models import Establishment +from location.models import City +from location.models import WineRegion +from product.models import Product +from review.models import Review +from tag.models import Tag +from transfer.models import GuideElements + + +def decorator(f): + def decorate(self): + print(f'{"-"*20}start {f.__name__}{"-"*20}') + f(self) + print(f'{"-"*20}end {f.__name__}{"-"*20}\n') + return decorate + + +class Command(BaseCommand): + help = """Check guide dependencies.""" + + @decorator + def count_of_guide_relative_dependencies(self): + for field in GuideElements._meta.fields: + if field.name not in ['id', 'lft', 'rgt', 'depth', + 'children_count', 'parent', 'order_number']: + filters = {f'{field.name}__isnull': False, } + qs = GuideElements.objects.filter(**filters).values_list(field.name, flat=True) + print(f"COUNT OF {field.name}'s: {len(set(qs))}") + + @decorator + def check_regions(self): + wine_region_old_ids = set(GuideElements.objects.filter(wine_region_id__isnull=False) + .values_list('wine_region_id', flat=True)) + not_existed_wine_regions = [] + for old_id in tqdm(wine_region_old_ids): + if not WineRegion.objects.filter(old_id=old_id).exists(): + not_existed_wine_regions.append(old_id) + print(f'NOT EXISTED WINE REGIONS: {len(not_existed_wine_regions)}') + pprint(f'{not_existed_wine_regions}') + + @decorator + def check_establishments(self): + establishment_old_ids = set(GuideElements.objects.filter(establishment_id__isnull=False) + .values_list('establishment_id', flat=True)) + not_existed_establishments = [] + for old_id in tqdm(establishment_old_ids): + if not Establishment.objects.filter(old_id=old_id).exists(): + not_existed_establishments.append(old_id) + print(f'NOT EXISTED ESTABLISHMENTS: {len(not_existed_establishments)}') + pprint(f'{not_existed_establishments}') + + @decorator + def check_reviews(self): + review_old_ids = set(GuideElements.objects.filter(review_id__isnull=False) + .values_list('review_id', flat=True)) + not_existed_reviews = [] + for old_id in tqdm(review_old_ids): + if not Review.objects.filter(old_id=old_id).exists(): + not_existed_reviews.append(old_id) + print(f'NOT EXISTED REVIEWS: {len(not_existed_reviews)}') + pprint(f'{not_existed_reviews}') + + @decorator + def check_wines(self): + wine_old_ids = set(GuideElements.objects.filter(wine_id__isnull=False) + .values_list('wine_id', flat=True)) + not_existed_wines = [] + for old_id in tqdm(wine_old_ids): + if not Product.objects.filter(old_id=old_id).exists(): + not_existed_wines.append(old_id) + print(f'NOT EXISTED WINES: {len(not_existed_wines)}') + pprint(f'{not_existed_wines}') + + @decorator + def check_wine_color(self): + raw_wine_color_nodes = set(GuideElements.objects.exclude(color__iexact='') + .filter(color__isnull=False) + .values_list('color', flat=True)) + raw_wine_colors = [i[:-11] for i in raw_wine_color_nodes] + raw_wine_color_index_names = [] + re_exp = '[A-Z][^A-Z]*' + for raw_wine_color in tqdm(raw_wine_colors): + result = re.findall(re_exp, rf'{raw_wine_color}') + if result and len(result) >= 2: + wine_color = '-'.join(result) + else: + wine_color = result[0] + raw_wine_color_index_names.append(wine_color.lower()) + not_existed_wine_colors = [] + for index_name in raw_wine_color_index_names: + if not Tag.objects.filter(value=index_name).exists(): + not_existed_wine_colors.append(index_name) + print(f'NOT EXISTED WINE COLOR: {len(not_existed_wine_colors)}') + pprint(f'{not_existed_wine_colors}') + + @decorator + def check_cities(self): + city_old_ids = set(GuideElements.objects.filter(city_id__isnull=False) + .values_list('city_id', flat=True)) + not_existed_cities = [] + for old_id in tqdm(city_old_ids): + if not City.objects.filter(old_id=old_id).exists(): + not_existed_cities.append(old_id) + print(f'NOT EXISTED CITIES: {len(not_existed_cities)}') + pprint(f'{not_existed_cities}') + + def handle(self, *args, **kwargs): + self.count_of_guide_relative_dependencies() + self.check_regions() + self.check_establishments() + self.check_reviews() + self.check_wines() + self.check_wine_color() + self.check_cities() diff --git a/apps/collection/migrations/0018_auto_20191127_1047.py b/apps/collection/migrations/0018_auto_20191127_1047.py new file mode 100644 index 00000000..c6f54ee2 --- /dev/null +++ b/apps/collection/migrations/0018_auto_20191127_1047.py @@ -0,0 +1,78 @@ +# Generated by Django 2.2.7 on 2019-11-27 10:47 + +import django.contrib.postgres.fields.jsonb +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0039_sitefeature_old_id'), + ('collection', '0017_collection_old_id'), + ] + + operations = [ + migrations.CreateModel( + name='GuideType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('name', models.SlugField(max_length=255, unique=True, verbose_name='code')), + ], + options={ + 'verbose_name': 'guide type', + 'verbose_name_plural': 'guide types', + }, + ), + migrations.RemoveField( + model_name='guide', + name='advertorials', + ), + migrations.RemoveField( + model_name='guide', + name='collection', + ), + migrations.RemoveField( + model_name='guide', + name='parent', + ), + migrations.AddField( + model_name='guide', + name='old_id', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='guide', + name='site', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='main.SiteSettings', verbose_name='site settings'), + ), + migrations.AddField( + model_name='guide', + name='slug', + field=models.SlugField(max_length=255, null=True, unique=True, verbose_name='slug'), + ), + migrations.AddField( + model_name='guide', + name='state', + field=models.PositiveSmallIntegerField(choices=[(0, 'built'), (1, 'waiting'), (2, 'removing'), (3, 'building')], default=1, verbose_name='state'), + ), + migrations.AddField( + model_name='guide', + name='vintage', + field=models.IntegerField(null=True, validators=[django.core.validators.MinValueValidator(1900), django.core.validators.MaxValueValidator(2100)], verbose_name='guide vintage year'), + ), + migrations.AddField( + model_name='guide', + name='guide_type', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='collection.GuideType', verbose_name='type'), + ), + migrations.AlterField( + model_name='guide', + name='start', + field=models.DateTimeField(null=True, verbose_name='start'), + ), + ] diff --git a/apps/collection/models.py b/apps/collection/models.py index d98f8a59..af8b2a33 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -1,6 +1,8 @@ from django.contrib.contenttypes.fields import ContentType from django.contrib.postgres.fields import JSONField +from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +import re from django.utils.translation import gettext_lazy as _ from utils.models import ProjectBaseMixin, URLImageMixin @@ -85,26 +87,64 @@ class Collection(ProjectBaseMixin, CollectionDateMixin, verbose_name_plural = _('collections') +class GuideTypeQuerySet(models.QuerySet): + """QuerySet for model GuideType.""" + + +class GuideType(ProjectBaseMixin): + """GuideType model.""" + + name = models.SlugField(max_length=255, unique=True, + verbose_name=_('code')) + + objects = GuideTypeQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('guide type') + verbose_name_plural = _('guide types') + + def __str__(self): + """Overridden str dunder method.""" + return self.name + + class GuideQuerySet(models.QuerySet): """QuerySet for Guide.""" - def by_collection_id(self, collection_id): - """Filter by collection id""" - return self.filter(collection=collection_id) - class Guide(ProjectBaseMixin, CollectionNameMixin, CollectionDateMixin): """Guide model.""" - parent = models.ForeignKey( - 'self', verbose_name=_('parent'), on_delete=models.CASCADE, - null=True, blank=True, default=None + BUILT = 0 + WAITING = 1 + REMOVING = 2 + BUILDING = 3 + + STATE_CHOICES = ( + (BUILT, 'built'), + (WAITING, 'waiting'), + (REMOVING, 'removing'), + (BUILDING, 'building'), + ) - advertorials = JSONField( - _('advertorials'), null=True, blank=True, - default=None, help_text='{"key":"value"}') - collection = models.ForeignKey(Collection, on_delete=models.CASCADE, - null=True, blank=True, default=None, - verbose_name=_('collection')) + + start = models.DateTimeField(null=True, + verbose_name=_('start')) + vintage = models.IntegerField(validators=[MinValueValidator(1900), + MaxValueValidator(2100)], + null=True, + verbose_name=_('guide vintage year')) + slug = models.SlugField(max_length=255, unique=True, null=True, + verbose_name=_('slug')) + guide_type = models.ForeignKey('GuideType', on_delete=models.PROTECT, + null=True, + verbose_name=_('type')) + site = models.ForeignKey('main.SiteSettings', on_delete=models.SET_NULL, + null=True, + verbose_name=_('site settings')) + state = models.PositiveSmallIntegerField(default=WAITING, choices=STATE_CHOICES, + verbose_name=_('state')) + old_id = models.IntegerField(blank=True, null=True) objects = GuideQuerySet.as_manager() @@ -116,3 +156,71 @@ class Guide(ProjectBaseMixin, CollectionNameMixin, CollectionDateMixin): def __str__(self): """String method.""" return f'{self.name}' + + +class AdvertorialQuerySet(models.QuerySet): + """QuerySet for model Advertorial.""" + + +class Advertorial(ProjectBaseMixin): + """Guide advertorial model.""" + number_of_pages = models.PositiveIntegerField( + verbose_name=_('number of pages'), + help_text=_('the total number of reserved pages')) + right_pages = models.PositiveIntegerField( + verbose_name=_('number of right pages'), + help_text=_('the number of right pages (which are part of total number).')) + old_id = models.IntegerField(blank=True, null=True) + + objects = AdvertorialQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('advertorial') + verbose_name_plural = _('advertorials') + + +class GuideFilterQuerySet(models.QuerySet): + """QuerySet for model GuideFilter.""" + + +class GuideFilter(ProjectBaseMixin): + """Guide filter model.""" + establishment_type_json = JSONField(blank=True, null=True, + verbose_name='establishment types') + country_code_json = JSONField(blank=True, null=True, + verbose_name='countries') + region_code_json = JSONField(blank=True, null=True, + verbose_name='regions') + sub_region_code_json = JSONField(blank=True, null=True, + verbose_name='sub regions') + wine_region_json = JSONField(blank=True, null=True, + verbose_name='wine regions') + wine_classification_json = JSONField(blank=True, null=True, + verbose_name='wine classifications') + wine_color_json = JSONField(blank=True, null=True, + verbose_name='wine colors') + wine_type_json = JSONField(blank=True, null=True, + verbose_name='wine types') + with_mark = models.BooleanField(default=True, + verbose_name=_('with mark'), + help_text=_('exclude empty marks?')) + locale_json = JSONField(blank=True, null=True, + verbose_name='locales') + max_mark = models.PositiveSmallIntegerField(verbose_name=_('max mark'), + help_text=_('mark under')) + min_mark = models.PositiveSmallIntegerField(verbose_name=_('min mark'), + help_text=_('mark over')) + review_vintage_json = JSONField(verbose_name='review vintage years') + review_state_json = JSONField(blank=True, null=True, + verbose_name='review states') + guide = models.OneToOneField(Guide, on_delete=models.CASCADE, + verbose_name=_('guide')) + old_id = models.IntegerField(blank=True, null=True) + + objects = GuideFilterQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('guide filter') + verbose_name_plural = _('guide filters') diff --git a/apps/collection/transfer_data.py b/apps/collection/transfer_data.py new file mode 100644 index 00000000..9307e6a0 --- /dev/null +++ b/apps/collection/transfer_data.py @@ -0,0 +1,37 @@ +from pprint import pprint +from transfer.models import Guides, GuideFilters +from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer + + +def transfer_guide(): + """Transfer Guide model.""" + queryset = Guides.objects.exclude(title__icontains='test') + serialized_data = GuideSerializer( + data=list(queryset.values()), + many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"transfer guide errors: {serialized_data.errors}") + + +def transfer_guide_filter(): + """Transfer GuideFilter model.""" + queryset = GuideFilters.objects.all() + serialized_data = GuideFilterSerializer( + data=list(queryset.values()), + many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"transfer guide filter errors: {serialized_data.errors}") + + +data_types = { + 'guides': [ + transfer_guide, + ], + 'guide_filters': [ + transfer_guide_filter, + ] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 2d0ce399..3b562b0d 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -41,6 +41,8 @@ class Command(BaseCommand): 'newsletter_subscriber', # подписчики на рассылку - переносить после переноса пользователей №1 'purchased_plaques', # №6 - перенос купленных тарелок 'fill_city_gallery', # №3 - перенос галереи городов + 'guides', + 'guide_filters', ] def handle(self, *args, **options): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 37f9217a..1ac40e80 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -5,11 +5,19 @@ # * Make sure each ForeignKey has `on_delete` set to the desired behavior. # * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table # Feel free to rename the models, but don't rename db_table values or field names. +import yaml from django.contrib.gis.db import models from transfer.mixins import MigrateMixin +def convert_entry(loader, node): + return {e[0]: e[1] for e in loader.construct_pairs(node)} + + +yaml.add_constructor('!ruby/hash:ActiveSupport::HashWithIndifferentAccess', convert_entry) + + # models.ForeignKey(ForeignModel, models.DO_NOTHING, blank=True, null=True) class Sites(MigrateMixin): diff --git a/apps/transfer/serializers/guide.py b/apps/transfer/serializers/guide.py new file mode 100644 index 00000000..5accbe64 --- /dev/null +++ b/apps/transfer/serializers/guide.py @@ -0,0 +1,283 @@ +from itertools import chain + +import yaml +from pycountry import countries, subdivisions +from rest_framework import serializers + +from collection.models import Guide, GuideType, GuideFilter +from establishment.models import EstablishmentType +from location.models import Country, Region +from main.models import SiteSettings +from transfer.mixins import TransferSerializerMixin + + +class GuideSerializer(TransferSerializerMixin): + id = serializers.IntegerField() + title = serializers.CharField() + vintage = serializers.IntegerField() + slug = serializers.CharField() + state = serializers.CharField() + site_id = serializers.IntegerField() + inserter_field = serializers.CharField() + + class Meta: + model = Guide + fields = ( + 'id', + 'title', + 'vintage', + 'slug', + 'state', + 'site_id', + 'inserter_field', + ) + + def validate(self, attrs): + """Overridden validate method.""" + attrs['old_id'] = attrs.pop('id') + attrs['name'] = attrs.pop('title') + attrs['vintage'] = int(attrs.pop('vintage')) + attrs['state'] = self.get_state(attrs.pop('state')) + attrs['site'] = self.get_site(attrs.pop('site_id')) + attrs['guide_type'] = self.get_guide_type(attrs.pop('inserter_field')) + return attrs + + def get_state(self, state: str): + if state == 'built': + return Guide.BUILT + elif state == 'removing': + return Guide.REMOVING + elif state == 'building': + return Guide.BUILDING + else: + return Guide.WAITING + + def get_site(self, site_id): + qs = SiteSettings.objects.filter(old_id=site_id) + if qs.exists(): + return qs.first() + + def get_guide_type(self, inserter_field): + guide_type, _ = GuideType.objects.get_or_create(name=inserter_field) + return guide_type + + +class GuideFilterSerializer(TransferSerializerMixin): + id = serializers.IntegerField() + year = serializers.CharField() + establishment_type = serializers.CharField(allow_null=True) + countries = serializers.CharField(allow_null=True) + regions = serializers.CharField(allow_null=True) + subregions = serializers.CharField(allow_null=True) + wine_regions = serializers.CharField(allow_null=True) + wine_classifications = serializers.CharField(allow_null=True) + wine_colors = serializers.CharField(allow_null=True) + wine_types = serializers.CharField(allow_null=True) + locales = serializers.CharField(allow_null=True) + states = serializers.CharField(allow_null=True) + + max_mark = serializers.IntegerField(allow_null=True) + min_mark = serializers.IntegerField(allow_null=True) + marks_only = serializers.NullBooleanField() + guide_id = serializers.IntegerField() + + class Meta: + model = GuideFilter + fields = ( + 'id', + 'year', + 'establishment_type', + 'countries', + 'regions', + 'subregions', + 'wine_regions', + 'wine_classifications', + 'wine_colors', + 'wine_types', + 'max_mark', + 'min_mark', + 'marks_only', + 'locales', + 'states', + 'guide_id', + ) + + @staticmethod + def parse_ruby_helper(raw_value: str): + """Parse RubyActiveSupport records""" + def convert_entry(loader, node): + return {e[0]: e[1] for e in loader.construct_pairs(node)} + + loader = yaml.Loader + loader.add_constructor('!ruby/hash:ActiveSupport::HashWithIndifferentAccess', convert_entry) + return yaml.load(raw_value, Loader=loader) if raw_value else None + + @staticmethod + def get_country_alpha_2(country_code_alpha3: str): + country = countries.get(alpha_3=country_code_alpha3.upper()) + return {'code_alpha_2': country.alpha2.lower() if country else None, + 'name': country.name if country else None,} + + @staticmethod + def parse_dictionary(dictionary: dict): + """ + Exclude root values from dictionary. + Convert {key_2: [value_1, value_2]} into + [value_1, value_2] + """ + return list(chain.from_iterable(dictionary.values())) + + @staticmethod + def parse_nested_dictionary(dictionary: dict): + """ + Exclude root values from dictionary. + Convert {key_1: {key_2: [value_1, value_2]}} into + [value_1, value_2] + """ + l = [] + for i in dictionary: + l.append(list(chain.from_iterable(list(dictionary[i].values())))) + return list(chain.from_iterable(l)) + + def get_country(self, code_alpha_3: str) -> Country: + country = self.get_country_alpha_2(code_alpha_3) + country_name = country['name'] + country_code = country['code_alpha_2'] + if country_name and country_code: + country, _ = Country.objects.get_or_create( + code__icontains=country_code, + name__contains={'en-GB': country_name}, + defaults={ + 'code': country_code, + 'name': {'en-GB': country_name} + } + ) + return country + + def get_region(self, region_code_alpha_3: str, + country_code_alpha_3: str, + sub_region_code_alpha_3: str = None): + country = self.get_country(country_code_alpha_3) + country_code_alpha_2 = country.code.upper() + region_qs = Region.objects.filter(code__iexact=region_code_alpha_3, + country__code__iexact=country_code_alpha_2) + + # If region isn't existed, check sub region for parent_code (region code) + if not region_qs.exists() and sub_region_code_alpha_3: + # sub region + subdivision = subdivisions.get( + code=f"{country_code_alpha_2}-{sub_region_code_alpha_3}") + if subdivision: + subdivision_region = subdivisions.get(parent_code=subdivision.parent_code) + obj = Region.objects.create( + name=subdivision_region.name, + code=subdivision_region.code, + country=country) + return obj + else: + return region_qs.first() + + def validate_year(self, value): + return self.parse_ruby_helper(value) + + def validate_establishment_type(self, value): + return self.parse_ruby_helper(value) + + def validate_countries(self, value): + return self.parse_ruby_helper(value) + + def validate_regions(self, value): + return self.parse_ruby_helper(value) + + def validate_sub_regions(self, value): + return self.parse_ruby_helper(value) + + def validate_wine_regions(self, value): + return self.parse_ruby_helper(value) + + def validate_wine_classifications(self, value): + return self.parse_ruby_helper(value) + + def validate_wine_colors(self, value): + return self.parse_ruby_helper(value) + + def validate_wine_types(self, value): + return self.parse_ruby_helper(value) + + def validate_locales(self, value): + return self.parse_ruby_helper(value) + + def validate_states(self, value): + return self.parse_ruby_helper(value) + + def validate(self, attrs): + sub_regions = attrs.pop('subregions') + regions = attrs.pop('regions') + + attrs['old_id'] = attrs.pop('id') + attrs['review_vintage_json'] = self.get_review_vintage(self.attrs.pop('year')) + attrs['establishment_type_json'] = self.get_establishment_type_ids( + self.attrs.pop('establishment_type')) + attrs['country_code_json'] = self.get_country_ids(attrs.pop('country_json')) + attrs['region_json'] = self.get_region_ids(regions, sub_regions) + attrs['sub_region_json'] = self.get_sub_region_ids(regions, sub_regions) + return attrs + + def get_review_vintage(self, year): + return {'vintage': [int(i) for i in set(year) if i.isdigit()]} + + def get_establishment_type_ids(self, establishment_types): + establishment_type_ids = [] + for establishment_type in establishment_types: + establishment_type_qs = EstablishmentType.objects.filter(index_name__iexact=establishment_type) + if not establishment_type_qs.exists(): + obj = EstablishmentType.objects.create( + name={'en-GB': establishment_type.capitalize}, + index_name=establishment_type.lower()) + establishment_type_ids.append(obj.id) + return {'id': establishment_type_ids} + + def get_country_ids(self, country_codes_alpha_3): + country_ids = [] + for code_alpha_3 in country_codes_alpha_3: + country_ids.append(self.get_country(code_alpha_3).id) + return {'id': country_ids} + + def get_region_ids(self, regions, sub_regions): + region_ids = [] + for country_code_alpha_3 in regions: + region_codes = regions[country_code_alpha_3] + for region_code_alpha_3 in region_codes: + region = self.get_region( + region_code_alpha_3=region_code_alpha_3, + country_code_alpha_3=country_code_alpha_3, + sub_region_code_alpha_3=sub_regions[country_code_alpha_3][region_code_alpha_3]) + if region: + region_ids.append(region.id) + return {'id': region_ids} + + def get_sub_region_ids(self, sub_regions): + sub_region_ids = [] + for country_code_alpha_3 in sub_regions: + for region_code_alpha_3 in sub_regions[country_code_alpha_3]: + region_codes = sub_regions[country_code_alpha_3][region_code_alpha_3] + for sub_region_code_alpha_3 in region_codes: + region = self.get_region( + region_code_alpha_3=region_code_alpha_3, + country_code_alpha_3=country_code_alpha_3, + sub_region_code_alpha_3=sub_region_code_alpha_3) + if region: + subdivision = subdivisions.get(parent_code=region.code.upper()) + if subdivision: + sub_region = Region.objects.create( + name=subdivision.name, + code=subdivisions.code, + parent_region=region, + country=region.country) + sub_region_ids.append(sub_region.id) + + return {'id': sub_region_ids} + + +# {'FRA': ['H', 'U']} REGIONS +# {'FRA': {'N': ['32']}} SUB REGIONS diff --git a/requirements/base.txt b/requirements/base.txt index 94e7ca27..aadc7301 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -57,3 +57,6 @@ redis==3.2.0 django_redis==4.10.0 # used byes indexing cache kombu==4.6.6 celery==4.3.0 + +# country information +pycountry==19.8.18 From 0fb28467fbb99fa51a2ebf2cf5bc8afcac2b5d29 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 29 Nov 2019 13:26:05 +0300 Subject: [PATCH 02/27] updated serializer --- _dockerfiles/db/Dockerfile | 2 +- apps/location/models.py | 2 +- apps/transfer/serializers/guide.py | 207 ++++++++++++++++++----------- 3 files changed, 133 insertions(+), 78 deletions(-) diff --git a/_dockerfiles/db/Dockerfile b/_dockerfiles/db/Dockerfile index 45c707d9..e8a9ded3 100644 --- a/_dockerfiles/db/Dockerfile +++ b/_dockerfiles/db/Dockerfile @@ -1,3 +1,3 @@ -FROM mdillon/postgis:9.5 +FROM mdillon/postgis:latest RUN localedef -i ru_RU -c -f UTF-8 -A /usr/share/locale/locale.alias ru_RU.UTF-8 ENV LANG ru_RU.utf8 diff --git a/apps/location/models.py b/apps/location/models.py index 3f104644..50f09b92 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -197,7 +197,7 @@ class WineRegionQuerySet(models.QuerySet): return self.exclude(wines__isnull=value) -class WineRegion(models.Model, TranslatedFieldsMixin): +class WineRegion(TranslatedFieldsMixin, models.Model): """Wine region model.""" name = models.CharField(_('name'), max_length=255) country = models.ForeignKey(Country, on_delete=models.PROTECT, diff --git a/apps/transfer/serializers/guide.py b/apps/transfer/serializers/guide.py index 5accbe64..692ebedc 100644 --- a/apps/transfer/serializers/guide.py +++ b/apps/transfer/serializers/guide.py @@ -6,9 +6,11 @@ from rest_framework import serializers from collection.models import Guide, GuideType, GuideFilter from establishment.models import EstablishmentType -from location.models import Country, Region +from location.models import Country, Region, WineRegion from main.models import SiteSettings from transfer.mixins import TransferSerializerMixin +from translation.models import Language +from review.models import Review class GuideSerializer(TransferSerializerMixin): @@ -70,9 +72,6 @@ class GuideFilterSerializer(TransferSerializerMixin): regions = serializers.CharField(allow_null=True) subregions = serializers.CharField(allow_null=True) wine_regions = serializers.CharField(allow_null=True) - wine_classifications = serializers.CharField(allow_null=True) - wine_colors = serializers.CharField(allow_null=True) - wine_types = serializers.CharField(allow_null=True) locales = serializers.CharField(allow_null=True) states = serializers.CharField(allow_null=True) @@ -91,9 +90,6 @@ class GuideFilterSerializer(TransferSerializerMixin): 'regions', 'subregions', 'wine_regions', - 'wine_classifications', - 'wine_colors', - 'wine_types', 'max_mark', 'min_mark', 'marks_only', @@ -112,12 +108,6 @@ class GuideFilterSerializer(TransferSerializerMixin): loader.add_constructor('!ruby/hash:ActiveSupport::HashWithIndifferentAccess', convert_entry) return yaml.load(raw_value, Loader=loader) if raw_value else None - @staticmethod - def get_country_alpha_2(country_code_alpha3: str): - country = countries.get(alpha_3=country_code_alpha3.upper()) - return {'code_alpha_2': country.alpha2.lower() if country else None, - 'name': country.name if country else None,} - @staticmethod def parse_dictionary(dictionary: dict): """ @@ -140,19 +130,20 @@ class GuideFilterSerializer(TransferSerializerMixin): return list(chain.from_iterable(l)) def get_country(self, code_alpha_3: str) -> Country: - country = self.get_country_alpha_2(code_alpha_3) - country_name = country['name'] - country_code = country['code_alpha_2'] - if country_name and country_code: - country, _ = Country.objects.get_or_create( - code__icontains=country_code, - name__contains={'en-GB': country_name}, - defaults={ - 'code': country_code, - 'name': {'en-GB': country_name} - } - ) - return country + country = countries.get(alpha_3=code_alpha_3.upper()) + if country: + country_name = country.name + country_code = country.alpha_2 + if country_name and country_code: + country, _ = Country.objects.get_or_create( + code__icontains=country_code, + name__contains={'en-GB': country_name}, + defaults={ + 'code': country_code, + 'name': {'en-GB': country_name} + } + ) + return country def get_region(self, region_code_alpha_3: str, country_code_alpha_3: str, @@ -162,20 +153,21 @@ class GuideFilterSerializer(TransferSerializerMixin): region_qs = Region.objects.filter(code__iexact=region_code_alpha_3, country__code__iexact=country_code_alpha_2) + if region_qs.exists(): + return region_qs.first() + # If region isn't existed, check sub region for parent_code (region code) - if not region_qs.exists() and sub_region_code_alpha_3: + if sub_region_code_alpha_3: # sub region subdivision = subdivisions.get( code=f"{country_code_alpha_2}-{sub_region_code_alpha_3}") if subdivision: - subdivision_region = subdivisions.get(parent_code=subdivision.parent_code) + subdivision_region = subdivisions.get(code=subdivision.parent_code) obj = Region.objects.create( name=subdivision_region.name, code=subdivision_region.code, country=country) return obj - else: - return region_qs.first() def validate_year(self, value): return self.parse_ruby_helper(value) @@ -189,7 +181,7 @@ class GuideFilterSerializer(TransferSerializerMixin): def validate_regions(self, value): return self.parse_ruby_helper(value) - def validate_sub_regions(self, value): + def validate_subregions(self, value): return self.parse_ruby_helper(value) def validate_wine_regions(self, value): @@ -215,69 +207,132 @@ class GuideFilterSerializer(TransferSerializerMixin): regions = attrs.pop('regions') attrs['old_id'] = attrs.pop('id') - attrs['review_vintage_json'] = self.get_review_vintage(self.attrs.pop('year')) + attrs['review_vintage_json'] = self.get_review_vintage(attrs.pop('year')) attrs['establishment_type_json'] = self.get_establishment_type_ids( - self.attrs.pop('establishment_type')) - attrs['country_code_json'] = self.get_country_ids(attrs.pop('country_json')) - attrs['region_json'] = self.get_region_ids(regions, sub_regions) - attrs['sub_region_json'] = self.get_sub_region_ids(regions, sub_regions) + attrs.pop('establishment_type')) + attrs['country_code_json'] = self.get_country_ids(attrs.pop('countries')) + attrs['region_json'] = self.get_region_ids(regions=regions, + sub_regions=sub_regions) + attrs['sub_region_json'] = self.get_sub_region_ids(sub_regions) + attrs['wine_region_json'] = self.get_wine_region_ids(attrs.pop('wine_regions')) + attrs['with_mark'] = attrs.pop('marks_only') + attrs['locale_json'] = self.get_locale_ids(attrs.pop('locales')) + attrs['review_state_json'] = self.get_review_state(attrs.pop('states')) + attrs['guide'] = self.get_guide(attrs.pop('guide_id')) return attrs def get_review_vintage(self, year): - return {'vintage': [int(i) for i in set(year) if i.isdigit()]} + if hasattr(year, '__iter__'): + return {'vintage': set(int(i) for i in set(year) if i.isdigit())} + return {'vintage': {year}} def get_establishment_type_ids(self, establishment_types): establishment_type_ids = [] - for establishment_type in establishment_types: - establishment_type_qs = EstablishmentType.objects.filter(index_name__iexact=establishment_type) - if not establishment_type_qs.exists(): - obj = EstablishmentType.objects.create( - name={'en-GB': establishment_type.capitalize}, - index_name=establishment_type.lower()) + if establishment_types: + for establishment_type in establishment_types: + establishment_type_qs = EstablishmentType.objects.filter(index_name__iexact=establishment_type) + if not establishment_type_qs.exists(): + obj = EstablishmentType.objects.create( + name={'en-GB': establishment_type.capitalize()}, + index_name=establishment_type.lower()) + else: + obj = establishment_type_qs.first() establishment_type_ids.append(obj.id) - return {'id': establishment_type_ids} + return {'id': set(establishment_type_ids)} def get_country_ids(self, country_codes_alpha_3): country_ids = [] - for code_alpha_3 in country_codes_alpha_3: - country_ids.append(self.get_country(code_alpha_3).id) - return {'id': country_ids} + if country_codes_alpha_3: + for code_alpha_3 in country_codes_alpha_3: + country = self.get_country(code_alpha_3) + if not country: + raise serializers.ValidationError({'detail': f'Country with alpha code 3 -' + f'{code_alpha_3}, is not found.'}) + country_ids.append(country.id) + return {'id': set(country_ids)} def get_region_ids(self, regions, sub_regions): region_ids = [] - for country_code_alpha_3 in regions: - region_codes = regions[country_code_alpha_3] - for region_code_alpha_3 in region_codes: - region = self.get_region( - region_code_alpha_3=region_code_alpha_3, - country_code_alpha_3=country_code_alpha_3, - sub_region_code_alpha_3=sub_regions[country_code_alpha_3][region_code_alpha_3]) - if region: - region_ids.append(region.id) + if regions: + for country_code_alpha_3 in regions: + for region_code_alpha_3 in regions[country_code_alpha_3]: + # Get region from sub region code. + if not sub_regions or country_code_alpha_3 not in sub_regions: + raise serializers.ValidationError({'detail': f'Sub regions is blanked.'}) + if region_code_alpha_3 in sub_regions[country_code_alpha_3]: + for sub_region_code_alpha_3 in sub_regions[country_code_alpha_3][region_code_alpha_3]: + region = self.get_region( + region_code_alpha_3=region_code_alpha_3, + country_code_alpha_3=country_code_alpha_3, + sub_region_code_alpha_3=sub_region_code_alpha_3) + if region: + region_ids.append(region.id) return {'id': region_ids} def get_sub_region_ids(self, sub_regions): sub_region_ids = [] - for country_code_alpha_3 in sub_regions: - for region_code_alpha_3 in sub_regions[country_code_alpha_3]: - region_codes = sub_regions[country_code_alpha_3][region_code_alpha_3] - for sub_region_code_alpha_3 in region_codes: - region = self.get_region( - region_code_alpha_3=region_code_alpha_3, - country_code_alpha_3=country_code_alpha_3, - sub_region_code_alpha_3=sub_region_code_alpha_3) - if region: - subdivision = subdivisions.get(parent_code=region.code.upper()) - if subdivision: - sub_region = Region.objects.create( - name=subdivision.name, - code=subdivisions.code, - parent_region=region, - country=region.country) - sub_region_ids.append(sub_region.id) + if sub_regions: + for country_code_alpha_3 in sub_regions: + # FRA etc. + for region_code_alpha_3 in sub_regions[country_code_alpha_3]: + # B, C, A etc. + for sub_region_code_alpha_3 in sub_regions[country_code_alpha_3][region_code_alpha_3]: + # 24, 32 etc. + # Get parent region + region = self.get_region( + region_code_alpha_3=region_code_alpha_3, + country_code_alpha_3=country_code_alpha_3, + sub_region_code_alpha_3=sub_region_code_alpha_3) + if region: + sub_region_qs = Region.objects.filter(parent_region__code=region.code) + if sub_region_qs.exists(): + sub_region_ids.append(sub_region_qs.first().id) + else: + subdivision = subdivisions.get(code=region.code.upper()) + if subdivision: + sub_region = Region.objects.get_or_create( + name=subdivision.name, + code=subdivisions.parent_code, + parent_region=region, + country=region.country) + sub_region_ids.append(sub_region.id) + return {'id': set(sub_region_ids)} - return {'id': sub_region_ids} + def get_wine_region_ids(self, wine_regions): + wine_region_ids = [] + if wine_regions: + for wine_region in wine_regions: + qs = WineRegion.objects.filter(name__iexact=wine_region) + if not qs.exists(): + raise serializers.ValidationError({ + 'detail': f'Wine region - {wine_region}, is not found.'}) + wine_region_ids.append(qs.first().id) + return {'id': set(wine_region_ids)} + def get_locale_ids(self, locales): + locale_ids = [] + if locales: + for locale in [locale for locale in locales if locale]: + qs = Language.objects.filter(locale=locale) + if not qs.exists(): + raise serializers.ValidationError({ + 'detail': f'{locale} is not found.'}) + return {'id': set(locale_ids)} -# {'FRA': ['H', 'U']} REGIONS -# {'FRA': {'N': ['32']}} SUB REGIONS + def get_review_state(self, states): + review_states = [] + if states: + for state in [state for state in states if state]: + if state == 'published': + review_states.append(Review.READY) + else: + review_states.append(Review.TO_INVESTIGATE) + return {'state': set(review_states)} + + def get_guide(self, old_guide_id: int): + qs = Guide.objects.filter(old_id=old_guide_id) + if not qs.exists(): + raise serializers.ValidationError({'detail': f'Guide with old id - ' + f'{old_guide_id}, ' + f'is not found.'}) + return qs.first() From 52872ce364100254a0eee5fbc77eb1f68f8aac0f Mon Sep 17 00:00:00 2001 From: Anatoly Date: Fri, 29 Nov 2019 14:09:00 +0300 Subject: [PATCH 03/27] updated --- .gitignore | 2 + apps/transfer/serializers/guide.py | 70 +++++++++++++++--------------- project/settings/local.py | 19 ++++++-- 3 files changed, 53 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index 78187ac0..1bb5c4b0 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ logs/ celerybeat-schedule local_files celerybeat.pid +/gm_viktor.dump +/docker-compose.dump.yml diff --git a/apps/transfer/serializers/guide.py b/apps/transfer/serializers/guide.py index 692ebedc..3fd5cc32 100644 --- a/apps/transfer/serializers/guide.py +++ b/apps/transfer/serializers/guide.py @@ -257,16 +257,15 @@ class GuideFilterSerializer(TransferSerializerMixin): for country_code_alpha_3 in regions: for region_code_alpha_3 in regions[country_code_alpha_3]: # Get region from sub region code. - if not sub_regions or country_code_alpha_3 not in sub_regions: - raise serializers.ValidationError({'detail': f'Sub regions is blanked.'}) - if region_code_alpha_3 in sub_regions[country_code_alpha_3]: - for sub_region_code_alpha_3 in sub_regions[country_code_alpha_3][region_code_alpha_3]: - region = self.get_region( - region_code_alpha_3=region_code_alpha_3, - country_code_alpha_3=country_code_alpha_3, - sub_region_code_alpha_3=sub_region_code_alpha_3) - if region: - region_ids.append(region.id) + if sub_regions and country_code_alpha_3 in sub_regions: + if region_code_alpha_3 in sub_regions[country_code_alpha_3]: + for sub_region_code_alpha_3 in sub_regions[country_code_alpha_3][region_code_alpha_3]: + region = self.get_region( + region_code_alpha_3=region_code_alpha_3, + country_code_alpha_3=country_code_alpha_3, + sub_region_code_alpha_3=sub_region_code_alpha_3) + if region: + region_ids.append(region.id) return {'id': region_ids} def get_sub_region_ids(self, sub_regions): @@ -274,28 +273,30 @@ class GuideFilterSerializer(TransferSerializerMixin): if sub_regions: for country_code_alpha_3 in sub_regions: # FRA etc. - for region_code_alpha_3 in sub_regions[country_code_alpha_3]: - # B, C, A etc. - for sub_region_code_alpha_3 in sub_regions[country_code_alpha_3][region_code_alpha_3]: - # 24, 32 etc. - # Get parent region - region = self.get_region( - region_code_alpha_3=region_code_alpha_3, - country_code_alpha_3=country_code_alpha_3, - sub_region_code_alpha_3=sub_region_code_alpha_3) - if region: - sub_region_qs = Region.objects.filter(parent_region__code=region.code) - if sub_region_qs.exists(): - sub_region_ids.append(sub_region_qs.first().id) - else: - subdivision = subdivisions.get(code=region.code.upper()) - if subdivision: - sub_region = Region.objects.get_or_create( - name=subdivision.name, - code=subdivisions.parent_code, - parent_region=region, - country=region.country) - sub_region_ids.append(sub_region.id) + if country_code_alpha_3 in sub_regions: + for region_code_alpha_3 in sub_regions[country_code_alpha_3]: + # B, C, A etc. + if region_code_alpha_3 in sub_regions[country_code_alpha_3]: + for sub_region_code_alpha_3 in sub_regions[country_code_alpha_3][region_code_alpha_3]: + # 24, 32 etc. + # Get parent region + region = self.get_region( + region_code_alpha_3=region_code_alpha_3, + country_code_alpha_3=country_code_alpha_3, + sub_region_code_alpha_3=sub_region_code_alpha_3) + if region: + sub_region_qs = Region.objects.filter(parent_region__code=region.code) + if sub_region_qs.exists(): + sub_region_ids.append(sub_region_qs.first().id) + else: + subdivision = subdivisions.get(code=region.code.upper()) + if subdivision: + sub_region = Region.objects.get_or_create( + name=subdivision.name, + code=subdivisions.parent_code, + parent_region=region, + country=region.country) + sub_region_ids.append(sub_region.id) return {'id': set(sub_region_ids)} def get_wine_region_ids(self, wine_regions): @@ -316,7 +317,7 @@ class GuideFilterSerializer(TransferSerializerMixin): qs = Language.objects.filter(locale=locale) if not qs.exists(): raise serializers.ValidationError({ - 'detail': f'{locale} is not found.'}) + 'detail': f'Language with locale - {locale}, is not found.'}) return {'id': set(locale_ids)} def get_review_state(self, states): @@ -332,7 +333,6 @@ class GuideFilterSerializer(TransferSerializerMixin): def get_guide(self, old_guide_id: int): qs = Guide.objects.filter(old_id=old_guide_id) if not qs.exists(): - raise serializers.ValidationError({'detail': f'Guide with old id - ' - f'{old_guide_id}, ' + raise serializers.ValidationError({'detail': f'Guide with old_id - {old_guide_id}, ' f'is not found.'}) return qs.first() diff --git a/project/settings/local.py b/project/settings/local.py index c8974c40..b610aba2 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -30,10 +30,21 @@ MEDIA_ROOT = os.path.join(PUBLIC_ROOT, MEDIA_LOCATION) THUMBNAIL_DEBUG = True # ADDED TRANSFER APP -# INSTALLED_APPS.append('transfer.apps.TransferConfig') +INSTALLED_APPS.append('transfer.apps.TransferConfig') # DATABASES -DATABASES.update({ +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USERNAME'), + 'PASSWORD': os.environ.get('DB_PASSWORD'), + 'HOST': os.environ.get('DB_HOSTNAME'), + 'PORT': os.environ.get('DB_PORT'), + 'OPTIONS': { + 'options': '-c search_path=gm' + }, + }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', # 'HOST': '172.22.0.1', @@ -41,7 +52,9 @@ DATABASES.update({ 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', - 'PASSWORD': 'octosecret123'}}) + 'PASSWORD': 'octosecret123' + }, +} # LOGGING From 076d14fd5bf29eef97d09a0dc783ca0fbec904c4 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Mon, 2 Dec 2019 13:51:48 +0300 Subject: [PATCH 04/27] added filter model --- .gitignore | 2 +- .../0019_advertorial_guidefilter.py | 41 +++++++ apps/collection/models.py | 28 +++-- apps/collection/transfer_data.py | 12 ++- apps/transfer/mixins.py | 6 ++ apps/transfer/models.py | 2 +- apps/transfer/serializers/guide.py | 102 ++++++++++-------- 7 files changed, 127 insertions(+), 66 deletions(-) create mode 100644 apps/collection/migrations/0019_advertorial_guidefilter.py diff --git a/.gitignore b/.gitignore index 1bb5c4b0..90e4f23f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,9 @@ logs/ # dev ./docker-compose.override.yml - celerybeat-schedule local_files celerybeat.pid /gm_viktor.dump /docker-compose.dump.yml +/gm_production_20191029.sql diff --git a/apps/collection/migrations/0019_advertorial_guidefilter.py b/apps/collection/migrations/0019_advertorial_guidefilter.py new file mode 100644 index 00000000..180a1e0e --- /dev/null +++ b/apps/collection/migrations/0019_advertorial_guidefilter.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.7 on 2019-12-02 10:11 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('collection', '0018_auto_20191127_1047'), + ] + + operations = [ + migrations.CreateModel( + name='GuideFilter', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('establishment_type_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='establishment types')), + ('country_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='countries')), + ('region_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='regions')), + ('sub_region_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='sub regions')), + ('wine_region_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='wine regions')), + ('with_mark', models.BooleanField(default=True, help_text='exclude empty marks?', verbose_name='with mark')), + ('locale_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='locales')), + ('max_mark', models.FloatField(help_text='mark under', null=True, verbose_name='max mark')), + ('min_mark', models.FloatField(help_text='mark over', null=True, verbose_name='min mark')), + ('review_vintage_json', django.contrib.postgres.fields.jsonb.JSONField(verbose_name='review vintage years')), + ('review_state_json', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='review states')), + ('old_id', models.IntegerField(blank=True, null=True)), + ('guide', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='collection.Guide', verbose_name='guide')), + ], + options={ + 'verbose_name': 'guide filter', + 'verbose_name_plural': 'guide filters', + }, + ), + ] diff --git a/apps/collection/models.py b/apps/collection/models.py index af8b2a33..7d037a85 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -188,29 +188,25 @@ class GuideFilter(ProjectBaseMixin): """Guide filter model.""" establishment_type_json = JSONField(blank=True, null=True, verbose_name='establishment types') - country_code_json = JSONField(blank=True, null=True, - verbose_name='countries') - region_code_json = JSONField(blank=True, null=True, - verbose_name='regions') - sub_region_code_json = JSONField(blank=True, null=True, - verbose_name='sub regions') + country_json = JSONField(blank=True, null=True, + verbose_name='countries') + region_json = JSONField(blank=True, null=True, + verbose_name='regions') + sub_region_json = JSONField(blank=True, null=True, + verbose_name='sub regions') wine_region_json = JSONField(blank=True, null=True, verbose_name='wine regions') - wine_classification_json = JSONField(blank=True, null=True, - verbose_name='wine classifications') - wine_color_json = JSONField(blank=True, null=True, - verbose_name='wine colors') - wine_type_json = JSONField(blank=True, null=True, - verbose_name='wine types') with_mark = models.BooleanField(default=True, verbose_name=_('with mark'), help_text=_('exclude empty marks?')) locale_json = JSONField(blank=True, null=True, verbose_name='locales') - max_mark = models.PositiveSmallIntegerField(verbose_name=_('max mark'), - help_text=_('mark under')) - min_mark = models.PositiveSmallIntegerField(verbose_name=_('min mark'), - help_text=_('mark over')) + max_mark = models.FloatField(verbose_name=_('max mark'), + null=True, + help_text=_('mark under')) + min_mark = models.FloatField(verbose_name=_('min mark'), + null=True, + help_text=_('mark over')) review_vintage_json = JSONField(verbose_name='review vintage years') review_state_json = JSONField(blank=True, null=True, verbose_name='review states') diff --git a/apps/collection/transfer_data.py b/apps/collection/transfer_data.py index 9307e6a0..9eeec541 100644 --- a/apps/collection/transfer_data.py +++ b/apps/collection/transfer_data.py @@ -5,6 +5,7 @@ from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer def transfer_guide(): """Transfer Guide model.""" + errors = [] queryset = Guides.objects.exclude(title__icontains='test') serialized_data = GuideSerializer( data=list(queryset.values()), @@ -12,19 +13,24 @@ def transfer_guide(): if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"transfer guide errors: {serialized_data.errors}") + for d in serialized_data.errors: errors.append(d) if d else None + pprint(f"transfer_guide errors: {errors}") def transfer_guide_filter(): """Transfer GuideFilter model.""" - queryset = GuideFilters.objects.all() + errors = [] + queryset = GuideFilters.objects.exclude(guide__title__icontains='test') \ + .exclude(guide__id__isnull=True) serialized_data = GuideFilterSerializer( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"transfer guide filter errors: {serialized_data.errors}") + for d in serialized_data.errors: errors.append(d) if d else None + pprint(f"transfer_guide_filter errors: {errors}\n" + f"COUNT: {len(errors)}") data_types = { diff --git a/apps/transfer/mixins.py b/apps/transfer/mixins.py index 7b43b210..30537a14 100644 --- a/apps/transfer/mixins.py +++ b/apps/transfer/mixins.py @@ -39,6 +39,12 @@ class TransferSerializerMixin(serializers.ModelSerializer): qs = self.Meta.model.objects.filter(**validated_data) if not qs.exists(): return super().create(validated_data) + # try: + # qs = self.Meta.model.objects.filter(**validated_data) + # if not qs.exists(): + # return super().create(validated_data) + # except Exception: + # breakpoint() @property def tag_category(self): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 1ac40e80..fa3d898b 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -370,7 +370,7 @@ class GuideFilters(MigrateMixin): states = models.CharField(max_length=255, blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() - guide_id = models.IntegerField(blank=True, null=True) + guide = models.ForeignKey(Guides, models.DO_NOTHING, blank=True, null=True) class Meta: managed = False diff --git a/apps/transfer/serializers/guide.py b/apps/transfer/serializers/guide.py index 3fd5cc32..c49e34b2 100644 --- a/apps/transfer/serializers/guide.py +++ b/apps/transfer/serializers/guide.py @@ -1,5 +1,5 @@ from itertools import chain - +from django.utils.text import slugify import yaml from pycountry import countries, subdivisions from rest_framework import serializers @@ -66,7 +66,7 @@ class GuideSerializer(TransferSerializerMixin): class GuideFilterSerializer(TransferSerializerMixin): id = serializers.IntegerField() - year = serializers.CharField() + year = serializers.CharField(allow_null=True) establishment_type = serializers.CharField(allow_null=True) countries = serializers.CharField(allow_null=True) regions = serializers.CharField(allow_null=True) @@ -75,8 +75,8 @@ class GuideFilterSerializer(TransferSerializerMixin): locales = serializers.CharField(allow_null=True) states = serializers.CharField(allow_null=True) - max_mark = serializers.IntegerField(allow_null=True) - min_mark = serializers.IntegerField(allow_null=True) + max_mark = serializers.FloatField(allow_null=True) + min_mark = serializers.FloatField(allow_null=True) marks_only = serializers.NullBooleanField() guide_id = serializers.IntegerField() @@ -149,25 +149,33 @@ class GuideFilterSerializer(TransferSerializerMixin): country_code_alpha_3: str, sub_region_code_alpha_3: str = None): country = self.get_country(country_code_alpha_3) - country_code_alpha_2 = country.code.upper() - region_qs = Region.objects.filter(code__iexact=region_code_alpha_3, - country__code__iexact=country_code_alpha_2) + if country: + country_code_alpha_2 = country.code.upper() + region_qs = Region.objects.filter(code__iexact=region_code_alpha_3, + country__code__iexact=country_code_alpha_2) - if region_qs.exists(): - return region_qs.first() + if region_qs.exists(): + return region_qs.first() - # If region isn't existed, check sub region for parent_code (region code) - if sub_region_code_alpha_3: - # sub region - subdivision = subdivisions.get( - code=f"{country_code_alpha_2}-{sub_region_code_alpha_3}") - if subdivision: - subdivision_region = subdivisions.get(code=subdivision.parent_code) - obj = Region.objects.create( - name=subdivision_region.name, - code=subdivision_region.code, - country=country) - return obj + # If region isn't existed, check sub region for parent_code (region code) + if sub_region_code_alpha_3: + # sub region + subdivision = subdivisions.get( + code=f"{country_code_alpha_2}-{sub_region_code_alpha_3}") + if subdivision: + # try with parent code + subdivision_region = subdivisions.get(code=subdivision.__dict__.get('_fields') + .get('parent_code')) + if not subdivision_region: + # try with parent + subdivision_region = subdivisions.get(code=subdivision.__dict__.get('_fields') + .get('parent')) + if subdivision_region: + obj = Region.objects.create( + name=subdivision_region.name, + code=subdivision_region.code, + country=country) + return obj def validate_year(self, value): return self.parse_ruby_helper(value) @@ -210,12 +218,12 @@ class GuideFilterSerializer(TransferSerializerMixin): attrs['review_vintage_json'] = self.get_review_vintage(attrs.pop('year')) attrs['establishment_type_json'] = self.get_establishment_type_ids( attrs.pop('establishment_type')) - attrs['country_code_json'] = self.get_country_ids(attrs.pop('countries')) + attrs['country_json'] = self.get_country_ids(attrs.pop('countries')) attrs['region_json'] = self.get_region_ids(regions=regions, sub_regions=sub_regions) attrs['sub_region_json'] = self.get_sub_region_ids(sub_regions) attrs['wine_region_json'] = self.get_wine_region_ids(attrs.pop('wine_regions')) - attrs['with_mark'] = attrs.pop('marks_only') + attrs['with_mark'] = attrs.pop('marks_only') or True attrs['locale_json'] = self.get_locale_ids(attrs.pop('locales')) attrs['review_state_json'] = self.get_review_state(attrs.pop('states')) attrs['guide'] = self.get_guide(attrs.pop('guide_id')) @@ -223,8 +231,8 @@ class GuideFilterSerializer(TransferSerializerMixin): def get_review_vintage(self, year): if hasattr(year, '__iter__'): - return {'vintage': set(int(i) for i in set(year) if i.isdigit())} - return {'vintage': {year}} + return {'vintage': list(set(int(i) for i in set(year) if i.isdigit()))} + return {'vintage': [year, ]} def get_establishment_type_ids(self, establishment_types): establishment_type_ids = [] @@ -234,22 +242,24 @@ class GuideFilterSerializer(TransferSerializerMixin): if not establishment_type_qs.exists(): obj = EstablishmentType.objects.create( name={'en-GB': establishment_type.capitalize()}, - index_name=establishment_type.lower()) + index_name=slugify(establishment_type)) else: obj = establishment_type_qs.first() establishment_type_ids.append(obj.id) - return {'id': set(establishment_type_ids)} + return {'id': list(set(establishment_type_ids))} def get_country_ids(self, country_codes_alpha_3): country_ids = [] if country_codes_alpha_3: for code_alpha_3 in country_codes_alpha_3: - country = self.get_country(code_alpha_3) - if not country: - raise serializers.ValidationError({'detail': f'Country with alpha code 3 -' - f'{code_alpha_3}, is not found.'}) - country_ids.append(country.id) - return {'id': set(country_ids)} + # Code can be an empty string. + if code_alpha_3 and not code_alpha_3 == 'AAA': + country = self.get_country(code_alpha_3) + if not country: + raise serializers.ValidationError({'detail': f'Country with alpha code 3 -' + f'{code_alpha_3}, is not found.'}) + country_ids.append(country.id) + return {'id': list(set(country_ids))} def get_region_ids(self, regions, sub_regions): region_ids = [] @@ -266,7 +276,7 @@ class GuideFilterSerializer(TransferSerializerMixin): sub_region_code_alpha_3=sub_region_code_alpha_3) if region: region_ids.append(region.id) - return {'id': region_ids} + return {'id': list(set(region_ids))} def get_sub_region_ids(self, sub_regions): sub_region_ids = [] @@ -291,13 +301,13 @@ class GuideFilterSerializer(TransferSerializerMixin): else: subdivision = subdivisions.get(code=region.code.upper()) if subdivision: - sub_region = Region.objects.get_or_create( + sub_region, _ = Region.objects.get_or_create( name=subdivision.name, - code=subdivisions.parent_code, + code=subdivision.code, parent_region=region, country=region.country) sub_region_ids.append(sub_region.id) - return {'id': set(sub_region_ids)} + return {'id': list(set(sub_region_ids))} def get_wine_region_ids(self, wine_regions): wine_region_ids = [] @@ -308,17 +318,21 @@ class GuideFilterSerializer(TransferSerializerMixin): raise serializers.ValidationError({ 'detail': f'Wine region - {wine_region}, is not found.'}) wine_region_ids.append(qs.first().id) - return {'id': set(wine_region_ids)} + return {'id': list(set(wine_region_ids))} def get_locale_ids(self, locales): locale_ids = [] if locales: for locale in [locale for locale in locales if locale]: - qs = Language.objects.filter(locale=locale) + if len(locale) == 2: + qs = Language.objects.filter(locale__startswith=locale) + else: + qs = Language.objects.filter(locale=locale) if not qs.exists(): raise serializers.ValidationError({ 'detail': f'Language with locale - {locale}, is not found.'}) - return {'id': set(locale_ids)} + locale_ids.extend(qs.values_list('id', flat=True)) + return {'id': list(set(locale_ids))} def get_review_state(self, states): review_states = [] @@ -328,11 +342,9 @@ class GuideFilterSerializer(TransferSerializerMixin): review_states.append(Review.READY) else: review_states.append(Review.TO_INVESTIGATE) - return {'state': set(review_states)} + return {'state': list(set(review_states))} def get_guide(self, old_guide_id: int): qs = Guide.objects.filter(old_id=old_guide_id) - if not qs.exists(): - raise serializers.ValidationError({'detail': f'Guide with old_id - {old_guide_id}, ' - f'is not found.'}) - return qs.first() + if qs.exists(): + return qs.first() From c5992afec0e096b7f477d574652aeba74b29cfe0 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 13:50:55 +0300 Subject: [PATCH 05/27] add opening_at to schedule search results --- apps/search_indexes/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index 67a2bff0..7bd3995a 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -168,6 +168,7 @@ class ScheduleDocumentSerializer(serializers.Serializer): weekday = serializers.IntegerField() weekday_display = serializers.CharField() closed_at = serializers.CharField() + opening_at = serializers.CharField() class InFavoritesMixin(DocumentSerializer): From 552e08e24249de183d24a966a5d4adeed96d28b9 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 13:52:39 +0300 Subject: [PATCH 06/27] opening_at for establishment --- apps/search_indexes/documents/establishment.py | 1 + apps/timetable/models.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 9e1d0a82..aca81f3f 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -116,6 +116,7 @@ class EstablishmentDocument(Document): 'weekday': fields.IntegerField(attr='weekday'), 'weekday_display': fields.KeywordField(attr='get_weekday_display'), 'closed_at': fields.KeywordField(attr='closed_at_str'), + 'opening_at': fields.KeywordField(attr='opening_at_str'), } )) address = fields.ObjectField( diff --git a/apps/timetable/models.py b/apps/timetable/models.py index cf7f8d94..90a6ae38 100644 --- a/apps/timetable/models.py +++ b/apps/timetable/models.py @@ -39,6 +39,10 @@ class Timetable(ProjectBaseMixin): def closed_at_str(self): return str(self.closed_at) if self.closed_at else None + @property + def opening_at_str(self): + return str(self.opening_at) if self.opening_at else None + @property def opening_time(self): return self.opening_at or self.lunch_start or self.dinner_start From 627f7562ec2fcdacc84013139d7ee594d68f45b8 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Wed, 4 Dec 2019 14:21:54 +0300 Subject: [PATCH 07/27] finalizing transfer guides --- apps/collection/admin.py | 21 ++ ...ntsectioncategory_guidewinecolorsection.py | 56 ++++ .../migrations/0021_guideelementtype.py | 24 ++ .../migrations/0022_guideelement.py | 48 +++ .../collection/migrations/0023_advertorial.py | 31 ++ apps/collection/models.py | 126 +++++++- apps/collection/transfer_data.py | 286 +++++++++++++++++- apps/transfer/management/commands/transfer.py | 6 + apps/transfer/mixins.py | 6 - apps/transfer/models.py | 4 +- apps/transfer/serializers/guide.py | 28 +- project/settings/base.py | 1 + requirements/base.txt | 3 + 13 files changed, 615 insertions(+), 25 deletions(-) create mode 100644 apps/collection/migrations/0020_guideelementsection_guideelementsectioncategory_guidewinecolorsection.py create mode 100644 apps/collection/migrations/0021_guideelementtype.py create mode 100644 apps/collection/migrations/0022_guideelement.py create mode 100644 apps/collection/migrations/0023_advertorial.py diff --git a/apps/collection/admin.py b/apps/collection/admin.py index 2e2c22e2..1b876e91 100644 --- a/apps/collection/admin.py +++ b/apps/collection/admin.py @@ -1,4 +1,6 @@ from django.contrib.gis import admin +from mptt.admin import DraggableMPTTAdmin, TreeRelatedFieldListFilter +from utils.admin import BaseModelAdminMixin from collection import models @@ -11,3 +13,22 @@ class CollectionAdmin(admin.ModelAdmin): @admin.register(models.Guide) class GuideAdmin(admin.ModelAdmin): """Guide admin.""" + + +@admin.register(models.GuideElementType) +class GuideElementType(admin.ModelAdmin): + """Guide element admin.""" + + +@admin.register(models.GuideElement) +class GuideElementAdmin(DraggableMPTTAdmin, BaseModelAdminMixin, admin.ModelAdmin): + """Guide element admin.""" + raw_id_fields = [ + 'guide_element_type', 'establishment', 'review', + 'wine_region', 'product', 'city', + 'wine_color_section', 'section', 'guide', + 'parent', + ] + # list_filter = ( + # ('parent', TreeRelatedFieldListFilter), + # ) diff --git a/apps/collection/migrations/0020_guideelementsection_guideelementsectioncategory_guidewinecolorsection.py b/apps/collection/migrations/0020_guideelementsection_guideelementsectioncategory_guidewinecolorsection.py new file mode 100644 index 00000000..06aaddfb --- /dev/null +++ b/apps/collection/migrations/0020_guideelementsection_guideelementsectioncategory_guidewinecolorsection.py @@ -0,0 +1,56 @@ +# Generated by Django 2.2.7 on 2019-12-02 14:05 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('collection', '0019_advertorial_guidefilter'), + ] + + operations = [ + migrations.CreateModel( + name='GuideElementSectionCategory', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('name', models.CharField(max_length=255, verbose_name='category name')), + ], + options={ + 'verbose_name': 'guide element section category', + 'verbose_name_plural': 'guide element section categories', + }, + ), + migrations.CreateModel( + name='GuideWineColorSection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('name', models.CharField(max_length=255, verbose_name='section name')), + ], + options={ + 'verbose_name': 'guide wine color section', + 'verbose_name_plural': 'guide wine color sections', + }, + ), + migrations.CreateModel( + name='GuideElementSection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('name', models.CharField(max_length=255, verbose_name='section name')), + ('old_id', models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id')), + ('category', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='collection.GuideElementSectionCategory', verbose_name='category')), + ], + options={ + 'verbose_name': 'guide element section', + 'verbose_name_plural': 'guide element sections', + }, + ), + ] diff --git a/apps/collection/migrations/0021_guideelementtype.py b/apps/collection/migrations/0021_guideelementtype.py new file mode 100644 index 00000000..7a199051 --- /dev/null +++ b/apps/collection/migrations/0021_guideelementtype.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.7 on 2019-12-02 14:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('collection', '0020_guideelementsection_guideelementsectioncategory_guidewinecolorsection'), + ] + + operations = [ + migrations.CreateModel( + name='GuideElementType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=50, verbose_name='name')), + ], + options={ + 'verbose_name': 'guide element type', + 'verbose_name_plural': 'guide element types', + }, + ), + ] diff --git a/apps/collection/migrations/0022_guideelement.py b/apps/collection/migrations/0022_guideelement.py new file mode 100644 index 00000000..77c2220c --- /dev/null +++ b/apps/collection/migrations/0022_guideelement.py @@ -0,0 +1,48 @@ +# Generated by Django 2.2.7 on 2019-12-02 14:44 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import mptt.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('review', '0018_auto_20191117_1117'), + ('product', '0018_purchasedproduct'), + ('location', '0030_auto_20191120_1010'), + ('establishment', '0067_auto_20191122_1244'), + ('collection', '0021_guideelementtype'), + ] + + operations = [ + migrations.CreateModel( + name='GuideElement', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('priority', models.IntegerField(blank=True, default=None, null=True)), + ('old_id', models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id')), + ('lft', models.PositiveIntegerField(db_index=True, editable=False)), + ('rght', models.PositiveIntegerField(db_index=True, editable=False)), + ('tree_id', models.PositiveIntegerField(db_index=True, editable=False)), + ('level', models.PositiveIntegerField(db_index=True, editable=False)), + ('city', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='location.City')), + ('establishment', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='establishment.Establishment')), + ('guide', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='collection.Guide')), + ('guide_element_type', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='collection.GuideElementType', verbose_name='guide element type')), + ('parent', mptt.fields.TreeForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='collection.GuideElement')), + ('product', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='product.Product')), + ('review', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='review.Review')), + ('section', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='collection.GuideElementSection')), + ('wine_color_section', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='collection.GuideWineColorSection')), + ('wine_region', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='location.WineRegion')), + ], + options={ + 'verbose_name': 'guide element', + 'verbose_name_plural': 'guide elements', + }, + ), + ] diff --git a/apps/collection/migrations/0023_advertorial.py b/apps/collection/migrations/0023_advertorial.py new file mode 100644 index 00000000..4917729f --- /dev/null +++ b/apps/collection/migrations/0023_advertorial.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.7 on 2019-12-03 13:20 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('collection', '0022_guideelement'), + ] + + operations = [ + migrations.CreateModel( + name='Advertorial', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')), + ('number_of_pages', models.PositiveIntegerField(help_text='the total number of reserved pages', verbose_name='number of pages')), + ('right_pages', models.PositiveIntegerField(help_text='the number of right pages (which are part of total number).', verbose_name='number of right pages')), + ('old_id', models.IntegerField(blank=True, null=True)), + ('guide_element', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='advertorial', to='collection.GuideElement', verbose_name='guide element')), + ], + options={ + 'verbose_name': 'advertorial', + 'verbose_name_plural': 'advertorials', + }, + ), + ] diff --git a/apps/collection/models.py b/apps/collection/models.py index 7d037a85..58ca9f07 100644 --- a/apps/collection/models.py +++ b/apps/collection/models.py @@ -1,8 +1,9 @@ +import re +from mptt.models import MPTTModel, TreeForeignKey from django.contrib.contenttypes.fields import ContentType from django.contrib.postgres.fields import JSONField from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models -import re from django.utils.translation import gettext_lazy as _ from utils.models import ProjectBaseMixin, URLImageMixin @@ -170,6 +171,9 @@ class Advertorial(ProjectBaseMixin): right_pages = models.PositiveIntegerField( verbose_name=_('number of right pages'), help_text=_('the number of right pages (which are part of total number).')) + guide_element = models.OneToOneField('GuideElement', on_delete=models.CASCADE, + related_name='advertorial', + verbose_name=_('guide element')) old_id = models.IntegerField(blank=True, null=True) objects = AdvertorialQuerySet.as_manager() @@ -220,3 +224,123 @@ class GuideFilter(ProjectBaseMixin): """Meta class.""" verbose_name = _('guide filter') verbose_name_plural = _('guide filters') + + +class GuideElementType(models.Model): + """Model for type of guide elements.""" + + name = models.CharField(max_length=50, + verbose_name=_('name')) + + class Meta: + """Meta class.""" + verbose_name = _('guide element type') + verbose_name_plural = _('guide element types') + + def __str__(self): + """Overridden str dunder.""" + return self.name + + +class GuideWineColorSectionQuerySet(models.QuerySet): + """QuerySet for model GuideWineColorSection.""" + + +class GuideWineColorSection(ProjectBaseMixin): + """Sections for wine colors.""" + + name = models.CharField(max_length=255, verbose_name=_('section name')) + + objects = GuideWineColorSectionQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('guide wine color section') + verbose_name_plural = _('guide wine color sections') + + +class GuideElementSectionCategoryQuerySet(models.QuerySet): + """QuerySet for model GuideElementSectionCategory.""" + + +class GuideElementSectionCategory(ProjectBaseMixin): + """Section category for guide element.""" + + name = models.CharField(max_length=255, + verbose_name=_('category name')) + + objects = GuideElementSectionCategoryQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('guide element section category') + verbose_name_plural = _('guide element section categories') + + +class GuideElementSectionQuerySet(models.QuerySet): + """QuerySet for model GuideElementSection.""" + + +class GuideElementSection(ProjectBaseMixin): + """Sections for guide element.""" + + name = models.CharField(max_length=255, verbose_name=_('section name')) + category = models.ForeignKey(GuideElementSectionCategory, on_delete=models.PROTECT, + verbose_name=_('category')) + old_id = models.PositiveIntegerField(blank=True, null=True, default=None, + verbose_name=_('old id')) + + objects = GuideElementSectionQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('guide element section') + verbose_name_plural = _('guide element sections') + + +class GuideElementQuerySet(models.QuerySet): + """QuerySet for model Guide elements.""" + + +class GuideElement(ProjectBaseMixin, MPTTModel): + """Frozen state of elements of guide instance.""" + + guide_element_type = models.ForeignKey('GuideElementType', on_delete=models.SET_NULL, + null=True, + verbose_name=_('guide element type')) + establishment = models.ForeignKey('establishment.Establishment', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + review = models.ForeignKey('review.Review', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + wine_region = models.ForeignKey('location.WineRegion', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + product = models.ForeignKey('product.Product', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + priority = models.IntegerField(null=True, blank=True, default=None) + city = models.ForeignKey('location.City', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + wine_color_section = models.ForeignKey('GuideWineColorSection', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + section = models.ForeignKey('GuideElementSection', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + guide = models.ForeignKey('Guide', on_delete=models.SET_NULL, + null=True, blank=True, default=None) + parent = TreeForeignKey('self', on_delete=models.CASCADE, + null=True, blank=True, + related_name='children') + old_id = models.PositiveIntegerField(blank=True, null=True, default=None, + verbose_name=_('old id')) + + objects = GuideElementQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('guide element') + verbose_name_plural = _('guide elements') + + class MPTTMeta: + order_insertion_by = ['guide_element_type'] + + def __str__(self): + """Overridden dunder method.""" + return self.guide_element_type.name if self.guide_element_type else self.id diff --git a/apps/collection/transfer_data.py b/apps/collection/transfer_data.py index 9eeec541..5d705b53 100644 --- a/apps/collection/transfer_data.py +++ b/apps/collection/transfer_data.py @@ -1,6 +1,15 @@ from pprint import pprint -from transfer.models import Guides, GuideFilters +from tqdm import tqdm +from establishment.models import Establishment +from review.models import Review +from location.models import WineRegion, City +from product.models import Product +from transfer.models import Guides, GuideFilters, GuideSections, GuideElements, \ + GuideAds from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer +from collection.models import GuideElementSection, GuideElementSectionCategory, \ + GuideWineColorSection, GuideElementType, GuideElement, \ + Guide, Advertorial def transfer_guide(): @@ -14,14 +23,15 @@ def transfer_guide(): serialized_data.save() else: for d in serialized_data.errors: errors.append(d) if d else None - pprint(f"transfer_guide errors: {errors}") + pprint(f"ERRORS: {errors}") + print(f'COUNT OF SERIALIZED OBJECTS: {queryset.values().count()}') def transfer_guide_filter(): """Transfer GuideFilter model.""" errors = [] queryset = GuideFilters.objects.exclude(guide__title__icontains='test') \ - .exclude(guide__id__isnull=True) + .exclude(guide__isnull=True) serialized_data = GuideFilterSerializer( data=list(queryset.values()), many=True) @@ -29,8 +39,250 @@ def transfer_guide_filter(): serialized_data.save() else: for d in serialized_data.errors: errors.append(d) if d else None - pprint(f"transfer_guide_filter errors: {errors}\n" - f"COUNT: {len(errors)}") + pprint(f'ERRORS: {errors}') + print(f"COUNT: {len(errors)}") + print(f'COUNT OF SERIALIZED OBJECTS: {queryset.values().count()}') + + +def transfer_guide_element_section(): + """Transfer GuideSections model.""" + created_count = 0 + category, _ = GuideElementSectionCategory.objects.get_or_create( + name='shop_category') + queryset_values = GuideSections.objects.values_list('id', 'value_name') + for old_id, section_name in tqdm(queryset_values): + obj, created = GuideElementSection.objects.get_or_create( + name=section_name, + category=category, + old_id=old_id, + ) + if created: created_count += 1 + print(f'OBJECTS CREATED: {created_count}') + + +def transfer_guide_wine_color_section(): + """Transfer GuideElements model (only wine color sections).""" + created_count = 0 + queryset_values = GuideElements.objects.raw( + """ + select distinct(color), + 1 as id + from guide_elements where color is not null; + """ + ) + for section_name in tqdm([i.color for i in queryset_values]): + obj, created = GuideWineColorSection.objects.get_or_create( + name=section_name + ) + if created: created_count += 1 + print(f'OBJECTS CREATED: {created_count}') + + +def transfer_guide_element_type(): + """Transfer GuideElements model (only element types).""" + created_count = 0 + queryset_values = GuideElements.objects.raw( + """ + select distinct(type), + 1 as id + from guide_elements; + """ + ) + for element_type in tqdm([i.type for i in queryset_values]): + obj, created = GuideElementType.objects.get_or_create( + name=element_type + ) + if created: created_count += 1 + print(f'OBJECTS CREATED: {created_count}') + + +def transfer_guide_elements_bulk(): + """Transfer Guide elements via bulk_create.""" + def get_guide_element_type(guide_element_type: str): + if guide_element_type: + qs = GuideElementType.objects.filter(name__iexact=guide_element_type) + if qs.exists(): + return qs.first() + + def get_establishment(old_id: int): + if old_id: + qs = Establishment.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_review(old_id: int): + if old_id: + qs = Review.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_wine_region(old_id: int): + if old_id: + qs = WineRegion.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_wine(old_id: int): + if old_id: + qs = Product.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_wine_color_section(color_section: str): + if color_section: + qs = GuideWineColorSection.objects.filter(name__iexact=color_section) + if qs.exists(): + return qs.first() + + def get_city(old_id: int): + if old_id: + qs = City.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_guide_element_section(old_id: int): + if old_id: + qs = GuideElementSection.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_guide(old_id): + if old_id: + qs = Guide.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + def get_parent(old_id): + if old_id: + qs = GuideElement.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + objects_to_update = [] + base_queryset = GuideElements.objects.all() + + for old_id, type, establishment_id, review_id, wine_region_id, \ + wine_id, color, order_number, city_id, section_id, guide_id \ + in tqdm(base_queryset.filter(parent_id__isnull=True) + .values_list('id', 'type', 'establishment_id', + 'review_id', 'wine_region_id', 'wine_id', + 'color', 'order_number', 'city_id', + 'section_id', 'guide_id'), + desc='Check parent guide elements'): + if not GuideElement.objects.filter(old_id=old_id).exists(): + guide = GuideElement( + old_id=old_id, + guide_element_type=get_guide_element_type(type), + establishment=get_establishment(establishment_id), + review=get_review(review_id), + wine_region=get_wine_region(wine_region_id), + product=get_wine(wine_id), + wine_color_section=get_wine_color_section(color), + priority=order_number, + city=get_city(city_id), + section=get_guide_element_section(section_id), + parent=None, + lft=1, + rght=1, + tree_id=1, + level=1, + ) + # check old guide + if not guide_id: + objects_to_update.append(guide) + else: + old_guide = Guides.objects.exclude(title__icontains='test') \ + .filter(id=guide_id) + if old_guide.exists(): + guide.guide = get_guide(guide_id) + objects_to_update.append(guide) + + # create parents + GuideElement.objects.bulk_create(objects_to_update) + pprint(f'CREATED PARENT GUIDE ELEMENTS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') + print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') + + # attach child guide elements + queryset_values = base_queryset.filter(parent_id__isnull=False) \ + .order_by('-parent_id') \ + .values_list('id', 'type', 'establishment_id', + 'review_id', 'wine_region_id', 'wine_id', + 'color', 'order_number', 'city_id', + 'section_id', 'guide_id', 'parent_id') + for old_id, type, establishment_id, review_id, wine_region_id, \ + wine_id, color, order_number, city_id, section_id, guide_id, parent_id \ + in tqdm(sorted(queryset_values, key=lambda value: value[len(value)-1]), + desc='Check child guide elements'): + if not GuideElement.objects.filter(old_id=old_id).exists(): + # check old guide + if guide_id: + old_guide = Guides.objects.exclude(title__icontains='test') \ + .filter(id=guide_id) + if old_guide.exists(): + GuideElement.objects.create( + old_id=old_id, + guide_element_type=get_guide_element_type(type), + establishment=get_establishment(establishment_id), + review=get_review(review_id), + wine_region=get_wine_region(wine_region_id), + product=get_wine(wine_id), + wine_color_section=get_wine_color_section(color), + priority=order_number, + city=get_city(city_id), + section=get_guide_element_section(section_id), + parent=get_parent(parent_id), + lft=1, + rght=1, + tree_id=1, + level=1, + guide=get_guide(guide_id), + ) + + pprint(f'CREATED CHILD GUIDE ELEMENTS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') + print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') + + # rebuild trees + GuideElement._tree_manager.rebuild() + + +def transfer_guide_element_advertorials(): + """Transfer Guide Advertorials model.""" + def get_guide_element(old_id: int): + if old_id: + qs = GuideElement.objects.filter(old_id=old_id) + legacy_qs = GuideElements.objects.exclude(guide__isnull=True) \ + .exclude(guide__title__icontains='test') \ + .filter(id=guide_ad_node_id) + if qs.exists() and legacy_qs.exists(): + return qs.first() + elif legacy_qs.exists() and not qs.exists(): + raise ValueError(f'Guide element was not transfer correctly - {old_id}.') + + objects_to_update = [] + advertorials = GuideAds.objects.exclude(nb_pages__isnull=True) \ + .exclude(nb_right_pages__isnull=True) \ + .exclude(guide_ad_node_id__isnull=True) \ + .values_list('id', 'nb_pages', 'nb_right_pages', + 'guide_ad_node_id') + for old_id, nb_pages, nb_right_pages, guide_ad_node_id in tqdm(advertorials): + # check guide element + guide_element = get_guide_element(guide_ad_node_id) + + if not Advertorial.objects.filter(old_id=old_id).exists() and guide_element: + objects_to_update.append( + Advertorial( + old_id=old_id, + number_of_pages=nb_pages, + right_pages=nb_right_pages, + guide_element=guide_element, + ) + ) + + # create related child + Advertorial.objects.bulk_create(objects_to_update) + + pprint(f'CREATED ADVERTORIALS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') + print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') data_types = { @@ -39,5 +291,29 @@ data_types = { ], 'guide_filters': [ transfer_guide_filter, + ], + 'guide_element_sections': [ + transfer_guide_element_section, + ], + 'guide_wine_color_sections': [ + transfer_guide_wine_color_section, + ], + 'guide_element_types': [ + transfer_guide_element_type, + ], + 'guide_elements_bulk': [ + transfer_guide_elements_bulk, + ], + 'guide_element_advertorials': [ + transfer_guide_element_advertorials + ], + 'guide_complete': [ + transfer_guide, # transfer guides from Guides + transfer_guide_filter, # transfer guide filters from GuideFilters + transfer_guide_element_section, # partial transfer element section from GuideSections + transfer_guide_wine_color_section, # partial transfer wine color section from GuideSections + transfer_guide_element_type, # partial transfer section types from GuideElements + transfer_guide_elements_bulk, # transfer result of GuideFilters from GuideElements + transfer_guide_element_advertorials, # transfer advertorials that linked to GuideElements ] } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 3b562b0d..d59bacfc 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -43,6 +43,12 @@ class Command(BaseCommand): 'fill_city_gallery', # №3 - перенос галереи городов 'guides', 'guide_filters', + 'guide_element_sections', + 'guide_wine_color_sections', + 'guide_element_types', + 'guide_elements_bulk', + 'guide_element_advertorials', + 'guide_complete', ] def handle(self, *args, **options): diff --git a/apps/transfer/mixins.py b/apps/transfer/mixins.py index 30537a14..7b43b210 100644 --- a/apps/transfer/mixins.py +++ b/apps/transfer/mixins.py @@ -39,12 +39,6 @@ class TransferSerializerMixin(serializers.ModelSerializer): qs = self.Meta.model.objects.filter(**validated_data) if not qs.exists(): return super().create(validated_data) - # try: - # qs = self.Meta.model.objects.filter(**validated_data) - # if not qs.exists(): - # return super().create(validated_data) - # except Exception: - # breakpoint() @property def tag_category(self): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index fa3d898b..2ebcfb65 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -389,7 +389,7 @@ class GuideSections(MigrateMixin): class Meta: managed = False - db_table = 'guide_elements' + db_table = 'guide_sections' class GuideElements(MigrateMixin): @@ -406,7 +406,7 @@ class GuideElements(MigrateMixin): guide_ad = models.ForeignKey(GuideAds, models.DO_NOTHING, blank=True, null=True) city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) section = models.ForeignKey('GuideSections', models.DO_NOTHING, blank=True, null=True) - guide_id = models.IntegerField(blank=True, null=True) + guide = models.ForeignKey('Guides', models.DO_NOTHING, blank=True, null=True) parent = models.ForeignKey('self', models.DO_NOTHING, blank=True, null=True) lft = models.IntegerField() rgt = models.IntegerField() diff --git a/apps/transfer/serializers/guide.py b/apps/transfer/serializers/guide.py index c49e34b2..f0cddd02 100644 --- a/apps/transfer/serializers/guide.py +++ b/apps/transfer/serializers/guide.py @@ -1,16 +1,17 @@ from itertools import chain -from django.utils.text import slugify + import yaml +from django.utils.text import slugify from pycountry import countries, subdivisions from rest_framework import serializers -from collection.models import Guide, GuideType, GuideFilter +from collection import models from establishment.models import EstablishmentType from location.models import Country, Region, WineRegion from main.models import SiteSettings +from review.models import Review from transfer.mixins import TransferSerializerMixin from translation.models import Language -from review.models import Review class GuideSerializer(TransferSerializerMixin): @@ -23,7 +24,7 @@ class GuideSerializer(TransferSerializerMixin): inserter_field = serializers.CharField() class Meta: - model = Guide + model = models.Guide fields = ( 'id', 'title', @@ -46,13 +47,13 @@ class GuideSerializer(TransferSerializerMixin): def get_state(self, state: str): if state == 'built': - return Guide.BUILT + return models.Guide.BUILT elif state == 'removing': - return Guide.REMOVING + return models.Guide.REMOVING elif state == 'building': - return Guide.BUILDING + return models.Guide.BUILDING else: - return Guide.WAITING + return models.Guide.WAITING def get_site(self, site_id): qs = SiteSettings.objects.filter(old_id=site_id) @@ -60,7 +61,7 @@ class GuideSerializer(TransferSerializerMixin): return qs.first() def get_guide_type(self, inserter_field): - guide_type, _ = GuideType.objects.get_or_create(name=inserter_field) + guide_type, _ = models.GuideType.objects.get_or_create(name=inserter_field) return guide_type @@ -81,7 +82,7 @@ class GuideFilterSerializer(TransferSerializerMixin): guide_id = serializers.IntegerField() class Meta: - model = GuideFilter + model = models.GuideFilter fields = ( 'id', 'year', @@ -98,6 +99,11 @@ class GuideFilterSerializer(TransferSerializerMixin): 'guide_id', ) + def create(self, validated_data): + qs = self.Meta.model.objects.filter(guide=validated_data.get('guide')) + if not qs.exists(): + return super().create(validated_data) + @staticmethod def parse_ruby_helper(raw_value: str): """Parse RubyActiveSupport records""" @@ -345,6 +351,6 @@ class GuideFilterSerializer(TransferSerializerMixin): return {'state': list(set(review_states))} def get_guide(self, old_guide_id: int): - qs = Guide.objects.filter(old_id=old_guide_id) + qs = models.Guide.objects.filter(old_id=old_guide_id) if qs.exists(): return qs.first() diff --git a/project/settings/base.py b/project/settings/base.py index 2c6f0cf0..6648f0af 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -99,6 +99,7 @@ EXTERNAL_APPS = [ 'storages', 'sorl.thumbnail', 'timezonefinder', + 'mptt', ] diff --git a/requirements/base.txt b/requirements/base.txt index aadc7301..90e5b2d5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -60,3 +60,6 @@ celery==4.3.0 # country information pycountry==19.8.18 + +# sql-tree +django-mptt==0.9.1 From b2fb341c101e28410a71624b7ecf014c78a23347 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 17:13:38 +0300 Subject: [PATCH 08/27] fix wines faceted search --- apps/search_indexes/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index a5b952d7..5fa845f2 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -346,7 +346,7 @@ class ProductDocumentViewSet(BaseDocumentViewSet): faceted_search_fields = { 'tag': { - 'field': 'wine_colors.id', + 'field': 'tags.id', 'enabled': True, 'facet': TermsFacet, 'options': { From 1ae11f0b617894edb71d3ebdf904cafa0b152386 Mon Sep 17 00:00:00 2001 From: dormantman Date: Wed, 4 Dec 2019 17:27:49 +0300 Subject: [PATCH 09/27] System tags removed --- apps/establishment/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 140188d7..b708de63 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -447,7 +447,7 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, def visible_tags(self): return super().visible_tags \ .exclude(category__index_name__in=['guide', 'collection', 'purchased_item', - 'business_tag', 'business_tags_de']) \ + 'business_tag', 'business_tags_de', 'tag', ]) \ \ # todo: recalculate toque_number From d0378ad14e49700971b1bfc34fbba6546870b345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 4 Dec 2019 17:27:51 +0300 Subject: [PATCH 10/27] Fix options --- apps/establishment/views/back.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/views/back.py b/apps/establishment/views/back.py index d3afbf2e..6fa4d821 100644 --- a/apps/establishment/views/back.py +++ b/apps/establishment/views/back.py @@ -165,7 +165,7 @@ class EmployeeListCreateView(generics.ListCreateAPIView): pagination_class = None -class EstablishmentEmployeeListView(generics.ListAPIView): +class EstablishmentEmployeeListView(generics.ListCreateAPIView): """Establishment emplyoees list view.""" permission_classes = (permissions.AllowAny, ) serializer_class = serializers.EstablishmentEmployeeBackSerializer From dee353199dd38340ed3dead4a740817a23e3c16c Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 17:30:54 +0300 Subject: [PATCH 11/27] add schedule to establishments list --- apps/establishment/serializers/common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 37432a21..e6e489ee 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -405,6 +405,12 @@ class EstablishmentSimilarSerializer(EstablishmentBaseSerializer): """Serializer for Establishment model.""" address = AddressDetailSerializer(read_only=True) + schedule = ScheduleRUDSerializer(many=True, allow_null=True) + + class Meta(EstablishmentBaseSerializer.Meta): + fields = EstablishmentBaseSerializer.Meta.fields + [ + 'schedule', + ] class EstablishmentCommentCreateSerializer(comment_serializers.CommentSerializer): From 69b40f21b7b1683839c844675d66b11949531c3a Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 18:26:03 +0300 Subject: [PATCH 12/27] last comment for mobile establishment detail --- apps/establishment/models.py | 14 ++++++++++++++ apps/establishment/serializers/common.py | 13 +++++++++++++ apps/establishment/urls/common.py | 1 - apps/establishment/urls/mobile.py | 3 ++- apps/establishment/urls/web.py | 6 +++++- apps/establishment/views/web.py | 7 +++++++ 6 files changed, 41 insertions(+), 3 deletions(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index b708de63..6c9080f0 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -251,6 +251,15 @@ class EstablishmentQuerySet(models.QuerySet): return self.filter(id__in=subquery_filter_by_distance) \ .order_by('-reviews__published_at') + def prefetch_comments(self): + """Prefetch last comment.""" + from comment.models import Comment + return self.prefetch_related( + models.Prefetch('comments', + queryset=Comment.objects.exclude(is_publish=False).order_by('-created'), + to_attr='comments_prefetched') + ) + def prefetch_actual_employees(self): """Prefetch actual employees.""" return self.prefetch_related( @@ -614,6 +623,11 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, def artisan_category_indexing(self): return self.tags.filter(category__index_name='shop_category') + @property + def last_comment(self): + if hasattr(self, 'comments_prefetched'): + return self.comments_prefetched[0] + class EstablishmentNoteQuerySet(models.QuerySet): """QuerySet for model EstablishmentNote.""" diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index e6e489ee..042a0a96 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -401,6 +401,19 @@ class EstablishmentDetailSerializer(EstablishmentBaseSerializer): ] +class MobileEstablishmentDetailSerializer(EstablishmentDetailSerializer): + """Serializer for Establishment model for mobiles.""" + + last_comment = comment_serializers.CommentRUDSerializer(allow_null=True) + + class Meta(EstablishmentDetailSerializer.Meta): + """Meta class.""" + + fields = EstablishmentDetailSerializer.Meta.fields + [ + 'last_comment', + ] + + class EstablishmentSimilarSerializer(EstablishmentBaseSerializer): """Serializer for Establishment model.""" diff --git a/apps/establishment/urls/common.py b/apps/establishment/urls/common.py index e37c38f8..faa34bd9 100644 --- a/apps/establishment/urls/common.py +++ b/apps/establishment/urls/common.py @@ -9,7 +9,6 @@ urlpatterns = [ path('', views.EstablishmentListView.as_view(), name='list'), path('recent-reviews/', views.EstablishmentRecentReviewListView.as_view(), name='recent-reviews'), - path('slug//', views.EstablishmentRetrieveView.as_view(), name='detail'), path('slug//similar/', views.EstablishmentSimilarListView.as_view(), name='similar'), path('slug//comments/', views.EstablishmentCommentListView.as_view(), name='list-comments'), path('slug//comments/create/', views.EstablishmentCommentCreateView.as_view(), diff --git a/apps/establishment/urls/mobile.py b/apps/establishment/urls/mobile.py index 2803be18..165323a9 100644 --- a/apps/establishment/urls/mobile.py +++ b/apps/establishment/urls/mobile.py @@ -5,7 +5,8 @@ from establishment import views from establishment.urls.common import urlpatterns as common_urlpatterns urlpatterns = [ - path('geo/', views.EstablishmentNearestRetrieveView.as_view(), name='nearest-establishments-list') + path('geo/', views.EstablishmentNearestRetrieveView.as_view(), name='nearest-establishments-list'), + path('slug//', views.EstablishmentMobileRetrieveView.as_view(), name='mobile-detail'), ] urlpatterns.extend(common_urlpatterns) diff --git a/apps/establishment/urls/web.py b/apps/establishment/urls/web.py index b4d1942d..4fa6595e 100644 --- a/apps/establishment/urls/web.py +++ b/apps/establishment/urls/web.py @@ -1,7 +1,11 @@ """Establishment app web urlconf.""" from establishment.urls.common import urlpatterns as common_urlpatterns +from django.urls import path +from establishment import views -urlpatterns = [] +urlpatterns = [ + path('slug//', views.EstablishmentRetrieveView.as_view(), name='web-detail'), +] urlpatterns.extend(common_urlpatterns) diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 15421ee7..ba5cb23b 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -51,6 +51,13 @@ class EstablishmentRetrieveView(EstablishmentMixinView, generics.RetrieveAPIView return super().get_queryset().with_extended_related() +class EstablishmentMobileRetrieveView(EstablishmentRetrieveView): + serializer_class = serializers.MobileEstablishmentDetailSerializer + + def get_queryset(self): + return super().get_queryset().prefetch_comments() + + class EstablishmentRecentReviewListView(EstablishmentListView): """List view for last reviewed establishments.""" From e0ad819caa943a2731d5ef788d0ca220a2419823 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 18:50:13 +0300 Subject: [PATCH 13/27] fix empty comments --- apps/establishment/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 6c9080f0..c598de55 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -625,7 +625,7 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, @property def last_comment(self): - if hasattr(self, 'comments_prefetched'): + if hasattr(self, 'comments_prefetched') and len(self.comments_prefetched): return self.comments_prefetched[0] From f44a4bb29d0626edbeedcf10e193d103b64fc649 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 4 Dec 2019 19:37:05 +0300 Subject: [PATCH 14/27] Establishments favs type --- apps/establishment/serializers/common.py | 2 ++ apps/favorites/views.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 042a0a96..da58225b 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -419,10 +419,12 @@ class EstablishmentSimilarSerializer(EstablishmentBaseSerializer): address = AddressDetailSerializer(read_only=True) schedule = ScheduleRUDSerializer(many=True, allow_null=True) + establishment_type = EstablishmentTypeGeoSerializer() class Meta(EstablishmentBaseSerializer.Meta): fields = EstablishmentBaseSerializer.Meta.fields + [ 'schedule', + 'establishment_type', ] diff --git a/apps/favorites/views.py b/apps/favorites/views.py index 444048ff..53a05469 100644 --- a/apps/favorites/views.py +++ b/apps/favorites/views.py @@ -29,7 +29,7 @@ class FavoritesEstablishmentListView(generics.ListAPIView): def get_queryset(self): """Override get_queryset method""" return Establishment.objects.filter(favorites__user=self.request.user) \ - .order_by('-favorites') + .order_by('-favorites').with_base_related() class FavoritesProductListView(generics.ListAPIView): From 29ff10fb4becbbf8ab329c3a35ef33e94db2c0a6 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Wed, 4 Dec 2019 20:13:09 +0300 Subject: [PATCH 15/27] added wine origins --- apps/establishment/admin.py | 1 + .../commands/add_establishment_wine_origin.py | 30 +++++++++++ apps/establishment/models.py | 7 ++- apps/establishment/serializers/common.py | 7 +++ apps/location/admin.py | 12 +++++ ...mentwineoriginaddress_wineoriginaddress.py | 42 +++++++++++++++ apps/location/models.py | 53 +++++++++++++++++- apps/location/serializers/common.py | 54 +++++++++++++++++-- apps/product/admin.py | 1 + .../commands/add_wine_origin_address.py | 44 +++++++++++++++ .../migrations/0019_auto_20191204_1420.py | 21 ++++++++ apps/product/models.py | 15 ++---- apps/product/serializers/common.py | 10 ++-- apps/transfer/serializers/product.py | 24 +++++++-- 14 files changed, 294 insertions(+), 27 deletions(-) create mode 100644 apps/establishment/management/commands/add_establishment_wine_origin.py create mode 100644 apps/location/migrations/0031_establishmentwineoriginaddress_wineoriginaddress.py create mode 100644 apps/product/management/commands/add_wine_origin_address.py create mode 100644 apps/product/migrations/0019_auto_20191204_1420.py diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 45716f32..2c94676f 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -136,3 +136,4 @@ class SocialNetworkAdmin(BaseModelAdminMixin, admin.ModelAdmin): class CompanyAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Admin conf for Company model.""" raw_id_fields = ['establishment', 'address', ] + diff --git a/apps/establishment/management/commands/add_establishment_wine_origin.py b/apps/establishment/management/commands/add_establishment_wine_origin.py new file mode 100644 index 00000000..cbb37907 --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_wine_origin.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand + +from location.models import WineOriginAddress, EstablishmentWineOriginAddress +from product.models import Product + + +class Command(BaseCommand): + help = 'Add to establishment wine origin object.' + + def handle(self, *args, **kwarg): + create_counter = 0 + + for product in Product.objects.exclude(establishment__isnull=True): + establishment = product.establishment + if product.wine_origins.exists(): + for wine_origin in product.wine_origins.all(): + wine_region = wine_origin.wine_region + wine_sub_region = wine_origin.wine_sub_region + if not EstablishmentWineOriginAddress.objects.filter(establishment=establishment, + wine_region=wine_region, + wine_sub_region=wine_sub_region) \ + .exists(): + EstablishmentWineOriginAddress.objects.create( + establishment=establishment, + wine_region=wine_origin.wine_region, + wine_sub_region=wine_origin.wine_sub_region, + ) + create_counter += 1 + + self.stdout.write(self.style.WARNING(f'COUNT CREATED OBJECTS: {create_counter}')) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index cb490aa4..7b028365 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -1,8 +1,8 @@ """Establishment models.""" from datetime import datetime from functools import reduce -from typing import List from operator import or_ +from typing import List import elasticsearch_dsl from django.conf import settings @@ -22,6 +22,7 @@ from timezone_field import TimeZoneField from collection.models import Collection from location.models import Address +from location.models import WineOriginAddressMixin from main.models import Award, Currency from review.models import Review from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin, @@ -590,6 +591,10 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, if qs.exists(): return qs.first().image + @property + def wine_origins_unique(self): + return self.wine_origins.distinct('wine_region') + class EstablishmentNoteQuerySet(models.QuerySet): """QuerySet for model EstablishmentNote.""" diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index cb102ff1..6822e1e1 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -16,6 +16,8 @@ from utils import exceptions as utils_exceptions from utils.serializers import ImageBaseSerializer, CarouselCreateSerializer from utils.serializers import (ProjectModelSerializer, TranslatedField, FavoritesCreateSerializer) +from location.serializers import EstablishmentWineRegionBaseSerializer, \ + EstablishmentWineOriginBaseSerializer class ContactPhonesSerializer(serializers.ModelSerializer): @@ -283,6 +285,8 @@ class EstablishmentBaseSerializer(ProjectModelSerializer): type = EstablishmentTypeBaseSerializer(source='establishment_type', read_only=True) subtypes = EstablishmentSubTypeBaseSerializer(many=True, source='establishment_subtypes') image = serializers.URLField(source='image_url', read_only=True) + wine_regions = EstablishmentWineRegionBaseSerializer(many=True, source='wine_origins_unique', + read_only=True, allow_null=True) preview_image = serializers.URLField(source='preview_image_url', allow_null=True, read_only=True) @@ -311,6 +315,7 @@ class EstablishmentBaseSerializer(ProjectModelSerializer): 'image', 'preview_image', 'new_image', + 'wine_regions', ] @@ -365,6 +370,7 @@ class EstablishmentDetailSerializer(EstablishmentBaseSerializer): range_price_carte = RangePriceSerializer(read_only=True) vintage_year = serializers.ReadOnlyField() gallery = ImageBaseSerializer(read_only=True, source='crop_gallery', many=True) + wine_origins = EstablishmentWineOriginBaseSerializer(many=True, read_only=True) class Meta(EstablishmentBaseSerializer.Meta): """Meta class.""" @@ -391,6 +397,7 @@ class EstablishmentDetailSerializer(EstablishmentBaseSerializer): 'transportation', 'vintage_year', 'gallery', + 'wine_origins', ] diff --git a/apps/location/admin.py b/apps/location/admin.py index a7610a65..0f7a3a8d 100644 --- a/apps/location/admin.py +++ b/apps/location/admin.py @@ -45,3 +45,15 @@ class AddressAdmin(admin.OSMGeoAdmin): def geo_lat(self, item): if isinstance(item.coordinates, Point): return item.coordinates.y + + +@admin.register(models.EstablishmentWineOriginAddress) +class EstablishmentWineOriginAddress(admin.ModelAdmin): + """Admin model for EstablishmentWineOriginAddress.""" + raw_id_fields = ['establishment', ] + + +@admin.register(models.WineOriginAddress) +class WineOriginAddress(admin.ModelAdmin): + """Admin page for model WineOriginAddress.""" + raw_id_fields = ['product', ] diff --git a/apps/location/migrations/0031_establishmentwineoriginaddress_wineoriginaddress.py b/apps/location/migrations/0031_establishmentwineoriginaddress_wineoriginaddress.py new file mode 100644 index 00000000..c583399e --- /dev/null +++ b/apps/location/migrations/0031_establishmentwineoriginaddress_wineoriginaddress.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.7 on 2019-12-04 14:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0067_auto_20191122_1244'), + ('product', '0019_auto_20191204_1420'), + ('location', '0030_auto_20191120_1010'), + ] + + operations = [ + migrations.CreateModel( + name='WineOriginAddress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('product', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='wine_origins', to='product.Product', verbose_name='product')), + ('wine_region', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='location.WineRegion', verbose_name='wine region')), + ('wine_sub_region', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='location.WineSubRegion', verbose_name='wine sub region')), + ], + options={ + 'verbose_name': 'wine origin address', + 'verbose_name_plural': 'wine origin addresses', + }, + ), + migrations.CreateModel( + name='EstablishmentWineOriginAddress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('establishment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='wine_origins', to='establishment.Establishment', verbose_name='product')), + ('wine_region', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='location.WineRegion', verbose_name='wine region')), + ('wine_sub_region', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='location.WineSubRegion', verbose_name='wine sub region')), + ], + options={ + 'verbose_name': 'establishment wine origin address', + 'verbose_name_plural': 'establishment wine origin addresses', + }, + ), + ] diff --git a/apps/location/models.py b/apps/location/models.py index 50f09b92..fd023443 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -192,9 +192,9 @@ class WineRegionQuerySet(models.QuerySet): def with_sub_region_related(self): return self.prefetch_related('wine_sub_region') - def having_wines(self, value = True): + def having_wines(self, value=True): """Return qs with regions, which have any wine related to them""" - return self.exclude(wines__isnull=value) + return self.exclude(wineoriginaddress__product__isnull=value) class WineRegion(TranslatedFieldsMixin, models.Model): @@ -278,6 +278,55 @@ class WineVillage(models.Model): return self.name +class WineOriginAddressMixin(models.Model): + """Model for wine origin address.""" + wine_region = models.ForeignKey('location.WineRegion', on_delete=models.CASCADE, + verbose_name=_('wine region')) + wine_sub_region = models.ForeignKey('location.WineSubRegion', on_delete=models.CASCADE, + blank=True, null=True, default=None, + verbose_name=_('wine sub region')) + + class Meta: + """Meta class.""" + abstract = True + + +class EstablishmentWineOriginAddressQuerySet(models.QuerySet): + """QuerySet for EstablishmentWineOriginAddress model.""" + + +class EstablishmentWineOriginAddress(WineOriginAddressMixin): + """Establishment wine origin address model.""" + establishment = models.ForeignKey('establishment.Establishment', on_delete=models.CASCADE, + related_name='wine_origins', + verbose_name=_('product')) + + objects = EstablishmentWineOriginAddressQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('establishment wine origin address') + verbose_name_plural = _('establishment wine origin addresses') + + +class WineOriginAddressQuerySet(models.QuerySet): + """QuerySet for WineOriginAddress model.""" + + +class WineOriginAddress(WineOriginAddressMixin): + """Wine origin address model.""" + product = models.ForeignKey('product.Product', on_delete=models.CASCADE, + related_name='wine_origins', + verbose_name=_('product')) + + objects = WineOriginAddressQuerySet.as_manager() + + class Meta: + """Meta class.""" + verbose_name = _('wine origin address') + verbose_name_plural = _('wine origin addresses') + + # todo: Make recalculate price levels @receiver(post_save, sender=Country) def run_recalculate_price_levels(sender, instance, **kwargs): diff --git a/apps/location/serializers/common.py b/apps/location/serializers/common.py index fadb6f6c..6255e67f 100644 --- a/apps/location/serializers/common.py +++ b/apps/location/serializers/common.py @@ -191,10 +191,58 @@ class WineSubRegionBaseSerializer(serializers.ModelSerializer): ] -class WineRegionSerializer(WineRegionBaseSerializer): - """Wine region w/ subregion serializer""" +class EstablishmentWineRegionBaseSerializer(serializers.ModelSerializer): + """Establishment wine region origin serializer.""" - wine_sub_region = WineSubRegionBaseSerializer(allow_null=True, many=True) + id = serializers.IntegerField(source='wine_region.id') + name = serializers.CharField(source='wine_region.name') + country = CountrySerializer(source='wine_region.country') + + class Meta: + """Meta class.""" + model = models.EstablishmentWineOriginAddress + fields = [ + 'id', + 'name', + 'country', + ] + + +class EstablishmentWineOriginBaseSerializer(serializers.ModelSerializer): + """Serializer for intermediate model EstablishmentWineOrigin.""" + wine_region = WineRegionBaseSerializer() + wine_sub_region = WineSubRegionBaseSerializer(allow_null=True) + + class Meta: + """Meta class.""" + model = models.EstablishmentWineOriginAddress + fields = [ + 'wine_region', + 'wine_sub_region', + ] + + +class WineOriginRegionBaseSerializer(EstablishmentWineRegionBaseSerializer): + """Product wine region origin serializer.""" + + class Meta(EstablishmentWineRegionBaseSerializer.Meta): + """Meta class.""" + model = models.WineOriginAddress + + +class WineOriginBaseSerializer(EstablishmentWineOriginBaseSerializer): + """Serializer for intermediate model ProductWineOrigin.""" + + class Meta(EstablishmentWineOriginBaseSerializer.Meta): + """Meta class.""" + model = models.WineOriginAddress + + +class WineRegionSerializer(serializers.ModelSerializer): + """Wine region w/ sub region serializer""" + + wine_sub_region = WineSubRegionBaseSerializer(source='wine_region.wine_sub_region', + allow_null=True, many=True) class Meta(WineRegionBaseSerializer.Meta): fields = WineRegionBaseSerializer.Meta.fields + [ diff --git a/apps/product/admin.py b/apps/product/admin.py index 3becfb2d..182be5c3 100644 --- a/apps/product/admin.py +++ b/apps/product/admin.py @@ -1,5 +1,6 @@ """Product admin conf.""" from django.contrib import admin + from utils.admin import BaseModelAdminMixin from .models import Product, ProductType, ProductSubType, ProductGallery, Unit diff --git a/apps/product/management/commands/add_wine_origin_address.py b/apps/product/management/commands/add_wine_origin_address.py new file mode 100644 index 00000000..d47d7412 --- /dev/null +++ b/apps/product/management/commands/add_wine_origin_address.py @@ -0,0 +1,44 @@ +from django.core.management.base import BaseCommand +from tqdm import tqdm + +from location.models import WineOriginAddress +from product.models import Product +from transfer.models import Products +from transfer.serializers.product import ProductSerializer + + +class Command(BaseCommand): + help = 'Add to product wine origin object.' + + def handle(self, *args, **kwarg): + def get_product(old_id: int): + if old_id: + qs = Product.objects.filter(old_id=old_id) + if qs.exists(): + return qs.first() + + objects_to_create = [] + products = Products.objects.exclude(wine_region_id__isnull=True) \ + .values_list('id', 'wine_sub_region_id', 'wine_region_id') + for old_id, wine_sub_region_id, wine_region_id in tqdm(products): + product = get_product(old_id) + if product: + wine_sub_region = ProductSerializer.get_wine_sub_region(wine_sub_region_id) + wine_region = ProductSerializer.get_wine_region(wine_region_id) + if wine_region: + filters = { + 'product': product, + 'wine_region': wine_region} + wine_origin_address = WineOriginAddress( + product=product, + wine_region=wine_region) + + if wine_sub_region: + filters.update({'wine_sub_region': wine_sub_region}) + wine_origin_address.wine_sub_region = wine_sub_region + + if not WineOriginAddress.objects.filter(**filters).exists(): + objects_to_create.append(wine_origin_address) + + WineOriginAddress.objects.bulk_create(objects_to_create) + self.stdout.write(self.style.WARNING(f'COUNT CREATED OBJECTS: {len(objects_to_create)}')) diff --git a/apps/product/migrations/0019_auto_20191204_1420.py b/apps/product/migrations/0019_auto_20191204_1420.py new file mode 100644 index 00000000..514fdd61 --- /dev/null +++ b/apps/product/migrations/0019_auto_20191204_1420.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.7 on 2019-12-04 14:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('product', '0018_purchasedproduct'), + ] + + operations = [ + migrations.RemoveField( + model_name='product', + name='wine_region', + ), + migrations.RemoveField( + model_name='product', + name='wine_sub_region', + ), + ] diff --git a/apps/product/models.py b/apps/product/models.py index 561280fc..a6129c48 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -2,11 +2,12 @@ from django.contrib.contenttypes import fields as generic from django.contrib.gis.db import models as gis_models from django.core.exceptions import ValidationError +from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.db.models import Case, When from django.utils.translation import gettext_lazy as _ -from django.core.validators import MaxValueValidator, MinValueValidator +from location.models import WineOriginAddressMixin from utils.models import (BaseAttributes, ProjectBaseMixin, HasTagsMixin, TranslatedFieldsMixin, TJSONField, FavoritesMixin, GalleryModelMixin, IntermediateGalleryModelMixin) @@ -89,8 +90,8 @@ class ProductQuerySet(models.QuerySet): 'establishment__address__city', 'establishment__address__city__country', 'establishment__establishment_subtypes', 'product_gallery', 'gallery', 'product_type', 'subtypes', - 'classifications__classification_type', 'classifications__tags') \ - .select_related('wine_region', 'wine_sub_region') + 'classifications__classification_type', 'classifications__tags', + 'wine_origins__wine_region', 'wine_origins__wine_sub_region', ) def common(self): return self.filter(category=self.model.COMMON) @@ -176,14 +177,6 @@ class Product(GalleryModelMixin, TranslatedFieldsMixin, BaseAttributes, verbose_name=_('establishment')) public_mark = models.PositiveIntegerField(blank=True, null=True, default=None, verbose_name=_('public mark'), ) - wine_region = models.ForeignKey('location.WineRegion', on_delete=models.PROTECT, - related_name='wines', - blank=True, null=True, default=None, - verbose_name=_('wine region')) - wine_sub_region = models.ForeignKey('location.WineSubRegion', on_delete=models.PROTECT, - related_name='wines', - blank=True, null=True, default=None, - verbose_name=_('wine sub region')) classifications = models.ManyToManyField('ProductClassification', blank=True, verbose_name=_('classifications')) diff --git a/apps/product/serializers/common.py b/apps/product/serializers/common.py index 14eff642..983cc2ab 100644 --- a/apps/product/serializers/common.py +++ b/apps/product/serializers/common.py @@ -11,7 +11,7 @@ from review.serializers import ReviewShortSerializer from utils import exceptions as utils_exceptions from utils.serializers import TranslatedField, FavoritesCreateSerializer, ImageBaseSerializer from main.serializers import AwardSerializer -from location.serializers import WineRegionBaseSerializer, WineSubRegionBaseSerializer +from location.serializers import WineOriginRegionBaseSerializer, WineOriginBaseSerializer from tag.serializers import TagBaseSerializer, TagCategoryProductSerializer @@ -90,7 +90,7 @@ class ProductBaseSerializer(serializers.ModelSerializer): subtypes = ProductSubTypeBaseSerializer(many=True, read_only=True) establishment_detail = EstablishmentProductShortSerializer(source='establishment', read_only=True) tags = ProductTagSerializer(source='related_tags', many=True, read_only=True) - wine_region = WineRegionBaseSerializer(read_only=True) + wine_regions = WineOriginRegionBaseSerializer(many=True, source='wine_origins', read_only=True) wine_colors = TagBaseSerializer(many=True, read_only=True) preview_image_url = serializers.URLField(allow_null=True, read_only=True) @@ -110,7 +110,7 @@ class ProductBaseSerializer(serializers.ModelSerializer): 'vintage', 'tags', 'preview_image_url', - 'wine_region', + 'wine_regions', 'wine_colors', 'in_favorites', ] @@ -124,7 +124,7 @@ class ProductDetailSerializer(ProductBaseSerializer): awards = AwardSerializer(many=True, read_only=True) classifications = ProductClassificationBaseSerializer(many=True, read_only=True) standards = ProductStandardBaseSerializer(many=True, read_only=True) - wine_sub_region = WineSubRegionBaseSerializer(read_only=True) + wine_origins = WineOriginBaseSerializer(many=True, read_only=True) bottles_produced = TagBaseSerializer(many=True, read_only=True) sugar_contents = TagBaseSerializer(many=True, read_only=True) grape_variety = TagBaseSerializer(many=True, read_only=True) @@ -141,7 +141,7 @@ class ProductDetailSerializer(ProductBaseSerializer): 'awards', 'classifications', 'standards', - 'wine_sub_region', + 'wine_origins', 'bottles_produced', 'sugar_contents', 'image_url', diff --git a/apps/transfer/serializers/product.py b/apps/transfer/serializers/product.py index 86c6720a..90a3c884 100644 --- a/apps/transfer/serializers/product.py +++ b/apps/transfer/serializers/product.py @@ -340,6 +340,9 @@ class ProductSerializer(TransferSerializerMixin): def create(self, validated_data): qs = self.Meta.model.objects.filter(old_id=validated_data.get('old_id')) + wine_region = validated_data.pop('wine_region') + wine_sub_region = validated_data.pop('wine_sub_region') + # classifications classifications = [validated_data.pop('wine_classification', None)] # standards @@ -361,6 +364,11 @@ class ProductSerializer(TransferSerializerMixin): obj.standards.add(*[i for i in standards if i and i not in obj.standards.all()]) # adding tags obj.tags.add(*[i for i in tags if i and i not in obj.tags.all()]) + # checking wine origin address + wine_origin_address, _ = location_models.WineOriginAddress.objects.get_or_create( + product=obj, + wine_region=wine_region, + wine_sub_region=wine_sub_region) return obj def get_name(self, name, brand): @@ -390,17 +398,23 @@ class ProductSerializer(TransferSerializerMixin): if classification_qs.exists(): return classification_qs.first() - def get_wine_region(self, wine_region): + @staticmethod + def get_wine_region(wine_region): if wine_region: + old_id = wine_region if not isinstance(wine_region, transfer_models.WineLocations) \ + else wine_region.id wine_region_qs = location_models.WineRegion.objects.filter( - old_id=wine_region.id) + old_id=old_id) if wine_region_qs.exists(): return wine_region_qs.first() - def get_wine_sub_region(self, wine_sub_region_id): - if wine_sub_region_id: + @staticmethod + def get_wine_sub_region(wine_sub_region): + if wine_sub_region: + old_id = wine_sub_region if not isinstance(wine_sub_region, transfer_models.WineLocations) \ + else wine_sub_region.id sub_region_qs = location_models.WineSubRegion.objects.filter( - old_id=wine_sub_region_id) + old_id=old_id) if sub_region_qs.exists(): return sub_region_qs.first() From 1da7c40a9db257f21dc17a0b10873981462a75c8 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Wed, 4 Dec 2019 20:18:47 +0300 Subject: [PATCH 16/27] fix merge --- project/settings/local.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/project/settings/local.py b/project/settings/local.py index 5a89823d..c56f9042 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -113,6 +113,3 @@ TESTING = sys.argv[1:2] == ['test'] if TESTING: ELASTICSEARCH_INDEX_NAMES = {} ELASTICSEARCH_DSL_AUTOSYNC = False - -# INSTALLED APPS -INSTALLED_APPS.append('transfer.apps.TransferConfig') \ No newline at end of file From c0bc66c749f089b82a935654b90150a691540dc2 Mon Sep 17 00:00:00 2001 From: dormantman Date: Wed, 4 Dec 2019 21:02:08 +0300 Subject: [PATCH 17/27] Hide unusable tags --- apps/establishment/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index c5da4094..e19135e5 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -440,6 +440,7 @@ class Establishment(GalleryModelMixin, ProjectBaseMixin, URLImageMixin, return super().visible_tags \ .exclude(category__index_name__in=['guide', 'collection', 'purchased_item', 'business_tag', 'business_tags_de']) \ + .exclude(value__in=['rss', 'rss_selection']) \ \ # todo: recalculate toque_number From 1d9ee76a5c622c497630f6de775f1fe269fcba59 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Thu, 5 Dec 2019 09:43:55 +0300 Subject: [PATCH 18/27] fix es for establishments --- apps/collection/serializers/common.py | 3 - apps/product/serializers/back.py | 5 +- .../search_indexes/documents/establishment.py | 64 +++++++++++++------ apps/search_indexes/documents/product.py | 38 ++++++----- apps/search_indexes/serializers.py | 21 ++++++ apps/search_indexes/views.py | 36 ++++++++--- 6 files changed, 117 insertions(+), 50 deletions(-) diff --git a/apps/collection/serializers/common.py b/apps/collection/serializers/common.py index 846236d5..1b043f3c 100644 --- a/apps/collection/serializers/common.py +++ b/apps/collection/serializers/common.py @@ -56,7 +56,4 @@ class GuideSerializer(serializers.ModelSerializer): 'name', 'start', 'end', - 'parent', - 'advertorials', - 'collection' ] diff --git a/apps/product/serializers/back.py b/apps/product/serializers/back.py index ffbf690d..55dc5ebc 100644 --- a/apps/product/serializers/back.py +++ b/apps/product/serializers/back.py @@ -62,8 +62,9 @@ class ProductBackOfficeDetailSerializer(ProductDetailSerializer): 'available', 'product_type', 'establishment', - 'wine_region', - 'wine_sub_region', + # todo: need fix + # 'wine_region', + # 'wine_sub_region', 'wine_village', 'state', ] diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index aca81f3f..c6b68ed4 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -83,33 +83,59 @@ class EstablishmentDocument(Document): multi=True) products = fields.ObjectField( properties={ - 'wine_region': fields.ObjectField(properties={ - 'id': fields.IntegerField(), - 'name': fields.KeywordField(), - 'country': fields.ObjectField(properties={ - 'id': fields.IntegerField(), - 'name': fields.ObjectField(attr='name_indexing', - properties=OBJECT_FIELD_PROPERTIES), - 'code': fields.KeywordField(), - }), - # 'coordinates': fields.GeoPointField(), - 'description': fields.ObjectField(attr='description_indexing', properties=OBJECT_FIELD_PROPERTIES), - }), + 'wine_origins': fields.ListField( + fields.ObjectField( + properties={ + 'wine_region': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField(), + 'country': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.ObjectField(attr='name_indexing', + properties=OBJECT_FIELD_PROPERTIES), + 'code': fields.KeywordField(), + }), + # 'coordinates': fields.GeoPointField(), + 'description': fields.ObjectField(attr='description_indexing', + properties=OBJECT_FIELD_PROPERTIES) + + }), + 'wine_sub_region': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField(), + }), + })), 'wine_colors': fields.ObjectField( properties={ 'id': fields.IntegerField(), 'label': fields.ObjectField(attr='label_indexing', properties=OBJECT_FIELD_PROPERTIES), 'value': fields.KeywordField(), }, - multi=True, - ), - 'wine_sub_region': fields.ObjectField(properties={ - 'id': fields.IntegerField(), - 'name': fields.KeywordField(), - }), - }, + multi=True,)}, multi=True ) + wine_origins = fields.ListField( + fields.ObjectField( + properties={ + 'wine_region': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField(), + 'country': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.ObjectField(attr='name_indexing', + properties=OBJECT_FIELD_PROPERTIES), + 'code': fields.KeywordField(), + }), + # 'coordinates': fields.GeoPointField(), + 'description': fields.ObjectField(attr='description_indexing', + properties=OBJECT_FIELD_PROPERTIES) + + }), + 'wine_sub_region': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField(), + })}) + ) schedule = fields.ListField(fields.ObjectField( properties={ 'id': fields.IntegerField(attr='id'), diff --git a/apps/search_indexes/documents/product.py b/apps/search_indexes/documents/product.py index 853f72a2..c9459dd8 100644 --- a/apps/search_indexes/documents/product.py +++ b/apps/search_indexes/documents/product.py @@ -83,22 +83,28 @@ class ProductDocument(Document): }, multi=True, ) - wine_region = fields.ObjectField(properties={ - 'id': fields.IntegerField(), - 'name': fields.KeywordField(), - 'country': fields.ObjectField(properties={ - 'id': fields.IntegerField(), - 'name': fields.ObjectField(attr='name_indexing', - properties=OBJECT_FIELD_PROPERTIES), - 'code': fields.KeywordField(), - }), - # 'coordinates': fields.GeoPointField(), - 'description': fields.ObjectField(attr='description_indexing', properties=OBJECT_FIELD_PROPERTIES), - }) - wine_sub_region = fields.ObjectField(properties={ - 'id': fields.IntegerField(), - 'name': fields.KeywordField(), - }) + wine_origins = fields.ListField( + fields.ObjectField( + properties={ + 'wine_region': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField(), + 'country': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.ObjectField(attr='name_indexing', + properties=OBJECT_FIELD_PROPERTIES), + 'code': fields.KeywordField(), + }), + # 'coordinates': fields.GeoPointField(), + 'description': fields.ObjectField(attr='description_indexing', + properties=OBJECT_FIELD_PROPERTIES) + + }), + 'wine_sub_region': fields.ObjectField(properties={ + 'id': fields.IntegerField(), + 'name': fields.KeywordField(), + })}) + ) classifications = fields.ObjectField( # TODO properties={ 'classification_type': fields.ObjectField(properties={}), diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index 7bd3995a..4357d8c1 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -69,6 +69,16 @@ class WineRegionDocumentSerializer(serializers.Serializer): return instance.wine_region if instance and instance.wine_region else None +class WineSubRegionDocumentSerializer(serializers.Serializer): + """Wine region ES document serializer.""" + + id = serializers.IntegerField() + name = serializers.CharField() + + def get_attribute(self, instance): + return instance.wine_sub_region if instance and instance.wine_sub_region else None + + class TagDocumentSerializer(serializers.Serializer): """Tag ES document serializer,""" @@ -223,6 +233,13 @@ class NewsDocumentSerializer(InFavoritesMixin, DocumentSerializer): return get_translated_value(obj.subtitle) +class WineOriginSerializer(serializers.Serializer): + """Wine origin serializer.""" + + wine_region = WineRegionDocumentSerializer() + wine_sub_region = WineSubRegionDocumentSerializer(allow_null=True) + + class EstablishmentDocumentSerializer(InFavoritesMixin, DocumentSerializer): """Establishment document serializer.""" @@ -234,6 +251,7 @@ class EstablishmentDocumentSerializer(InFavoritesMixin, DocumentSerializer): restaurant_cuisine = TagsDocumentSerializer(many=True, allow_null=True) artisan_category = TagsDocumentSerializer(many=True, allow_null=True) schedule = ScheduleDocumentSerializer(many=True, allow_null=True) + wine_origins = WineOriginSerializer(many=True) class Meta: """Meta class.""" @@ -259,6 +277,7 @@ class EstablishmentDocumentSerializer(InFavoritesMixin, DocumentSerializer): 'works_evening', 'works_at_weekday', 'tz', + 'wine_origins', # 'works_now', # 'collections', # 'establishment_type', @@ -276,6 +295,7 @@ class ProductDocumentSerializer(InFavoritesMixin, DocumentSerializer): grape_variety = TagDocumentSerializer(many=True) product_type = ProductTypeDocumentSerializer(allow_null=True) establishment_detail = ProductEstablishmentDocumentSerializer(source='establishment', allow_null=True) + wine_origins = WineOriginSerializer(many=True) class Meta: """Meta class.""" @@ -302,4 +322,5 @@ class ProductDocumentSerializer(InFavoritesMixin, DocumentSerializer): 'establishment_detail', 'average_price', 'created', + 'wine_origins', ) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index 5fa845f2..0e93d8fc 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -158,7 +158,15 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): }, }, 'wine_region_id': { - 'field': 'products.wine_region.id', + 'field': 'wine_origins.wine_region.id', + 'facet': TermsFacet, + 'enabled': True, + 'options': { + 'size': utils.FACET_MAX_RESPONSE, + }, + }, + 'wine_sub_region_id': { + 'field': 'wine_origins.wine_sub_region.id', 'facet': TermsFacet, 'enabled': True, 'options': { @@ -213,14 +221,14 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): ], }, 'wine_region_id': { - 'field': 'products.wine_region.id', + 'field': 'wine_origins.wine_region.id', 'lookups': [ constants.LOOKUP_QUERY_IN, constants.LOOKUP_QUERY_EXCLUDE, ], }, 'wine_sub_region_id': { - 'field': 'products.wine_sub_region_id', + 'field': 'wine_origins.wine_sub_region.id', 'lookups': [ constants.LOOKUP_QUERY_IN, constants.LOOKUP_QUERY_EXCLUDE, @@ -354,13 +362,21 @@ class ProductDocumentViewSet(BaseDocumentViewSet): }, }, 'wine_region_id': { - 'field': 'wine_region.id', - 'enabled': True, + 'field': 'wine_origins.wine_region.id', 'facet': TermsFacet, + 'enabled': True, 'options': { 'size': utils.FACET_MAX_RESPONSE, }, }, + 'wine_sub_region_id': { + 'field': 'wine_origins.wine_sub_region.id', + 'facet': TermsFacet, + 'enabled': True, + 'options': { + 'size': utils.FACET_MAX_RESPONSE, + }, + } } translated_search_fields = ( @@ -384,14 +400,14 @@ class ProductDocumentViewSet(BaseDocumentViewSet): ], }, 'wine_region_id': { - 'field': 'wine_region.id', + 'field': 'wine_origins.wine_region.id', 'lookups': [ constants.LOOKUP_QUERY_IN, constants.LOOKUP_QUERY_EXCLUDE, ], }, 'wine_sub_region_id': { - 'field': 'wine_sub_region_id', + 'field': 'wine_origins.wine_sub_region.id', 'lookups': [ constants.LOOKUP_QUERY_IN, constants.LOOKUP_QUERY_EXCLUDE, @@ -404,9 +420,9 @@ class ProductDocumentViewSet(BaseDocumentViewSet): constants.LOOKUP_QUERY_EXCLUDE, ] }, - 'wine_from_country_code': { - 'field': 'wine_region.country.code', - }, + # 'wine_from_country_code': { + # 'field': 'wine_origins.wine_region.country.code', + # }, 'for_establishment': { 'field': 'establishment.slug', }, From d114b93b14445e2b313b0486a4106488ca60c65e Mon Sep 17 00:00:00 2001 From: Anatoly Date: Thu, 5 Dec 2019 09:48:09 +0300 Subject: [PATCH 19/27] fix es for products --- apps/search_indexes/serializers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index 4357d8c1..a43ebaf7 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -290,7 +290,6 @@ class ProductDocumentSerializer(InFavoritesMixin, DocumentSerializer): tags = TagsDocumentSerializer(many=True, source='related_tags') subtypes = ProductSubtypeDocumentSerializer(many=True, allow_null=True) - wine_region = WineRegionDocumentSerializer(allow_null=True) wine_colors = TagDocumentSerializer(many=True) grape_variety = TagDocumentSerializer(many=True) product_type = ProductTypeDocumentSerializer(allow_null=True) @@ -316,7 +315,6 @@ class ProductDocumentSerializer(InFavoritesMixin, DocumentSerializer): 'tags', 'product_type', 'subtypes', - 'wine_region', 'wine_colors', 'grape_variety', 'establishment_detail', From 1e0044cbd154729b29d52c7c56103a1d2cab1d48 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 12:11:44 +0300 Subject: [PATCH 20/27] artisans tags for favs (cherry picked from commit 7081e52) --- apps/establishment/serializers/common.py | 2 ++ apps/favorites/views.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 4a419fee..23fc24df 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -427,11 +427,13 @@ class EstablishmentSimilarSerializer(EstablishmentBaseSerializer): address = AddressDetailSerializer(read_only=True) schedule = ScheduleRUDSerializer(many=True, allow_null=True) establishment_type = EstablishmentTypeGeoSerializer() + artisan_category = TagBaseSerializer(many=True, allow_null=True) class Meta(EstablishmentBaseSerializer.Meta): fields = EstablishmentBaseSerializer.Meta.fields + [ 'schedule', 'establishment_type', + 'artisan_category', ] diff --git a/apps/favorites/views.py b/apps/favorites/views.py index 53a05469..bee25ced 100644 --- a/apps/favorites/views.py +++ b/apps/favorites/views.py @@ -29,7 +29,8 @@ class FavoritesEstablishmentListView(generics.ListAPIView): def get_queryset(self): """Override get_queryset method""" return Establishment.objects.filter(favorites__user=self.request.user) \ - .order_by('-favorites').with_base_related() + .order_by('-favorites').with_base_related() \ + .with_certain_tag_category_related('shop_category', 'artisan_category') class FavoritesProductListView(generics.ListAPIView): From ca80f7a7971b1f6b04233c29faa53f3854c8c153 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 13:14:27 +0300 Subject: [PATCH 21/27] possible fix center calculation --- apps/search_indexes/filters.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 644b47fd..cc2bb55a 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -13,24 +13,24 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @staticmethod def calculate_center(first, second): - if second[1] < 0 < first[1]: - reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) + if second[0] < 0 < first[0]: + reverse_first, reverse_second = 180 - abs(first[0]), 180 - abs(second[0]) diff = (reverse_first + reverse_second) / 2 if reverse_first < reverse_second: - result_part = -180 + (180 + second[1] - diff) + result_part = -180 + (180 + second[0] - diff) else: - result_part = 180 - (180 - first[1] - diff) + result_part = 180 - (180 - first[0] - diff) - elif second[1] < 0 > first[1] or second[1] > 0 < first[1]: + elif second[0] < 0 > first[0] or second[0] > 0 < first[0]: reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) result_part = ((reverse_first + reverse_second) / 2) * (-1 + (second[1] < 0) * 2) else: - result_part = (first[1] + second[1]) / 2 + result_part = (first[0] + second[0]) / 2 - return (first[0] + second[0]) / 2, result_part + return result_part, (first[1] + second[1]) / 2 def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view) From 66fc2fae995d3dc5a61d69a1414c732261dd2609 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 13:15:51 +0300 Subject: [PATCH 22/27] Revert "possible fix center calculation" This reverts commit ca80f7a --- apps/search_indexes/filters.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index cc2bb55a..644b47fd 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -13,24 +13,24 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @staticmethod def calculate_center(first, second): - if second[0] < 0 < first[0]: - reverse_first, reverse_second = 180 - abs(first[0]), 180 - abs(second[0]) + if second[1] < 0 < first[1]: + reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) diff = (reverse_first + reverse_second) / 2 if reverse_first < reverse_second: - result_part = -180 + (180 + second[0] - diff) + result_part = -180 + (180 + second[1] - diff) else: - result_part = 180 - (180 - first[0] - diff) + result_part = 180 - (180 - first[1] - diff) - elif second[0] < 0 > first[0] or second[0] > 0 < first[0]: + elif second[1] < 0 > first[1] or second[1] > 0 < first[1]: reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) result_part = ((reverse_first + reverse_second) / 2) * (-1 + (second[1] < 0) * 2) else: - result_part = (first[0] + second[0]) / 2 + result_part = (first[1] + second[1]) / 2 - return result_part, (first[1] + second[1]) / 2 + return (first[0] + second[0]) / 2, result_part def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view) From 3e7922e0fcae0a978979637f3194725f07bb0a68 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 13:30:43 +0300 Subject: [PATCH 23/27] temp fix --- apps/search_indexes/filters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 644b47fd..3d55bdb4 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -30,7 +30,8 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): else: result_part = (first[1] + second[1]) / 2 - return (first[0] + second[0]) / 2, result_part + # return (first[0] + second[0]) / 2, result_part + return (first[0] + second[0]) / 2, (first[1] + second[1]) / 2 def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view) From 3afb51d4728b9d760f31bdbd40c49899d03546a5 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Thu, 5 Dec 2019 14:11:46 +0300 Subject: [PATCH 24/27] fix establishment admin page --- apps/establishment/admin.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 2c94676f..81301ddc 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -35,6 +35,7 @@ class ContactPhoneInline(admin.TabularInline): class GalleryImageInline(admin.TabularInline): """Gallery image inline admin.""" model = models.EstablishmentGallery + raw_id_fields = ['image', ] extra = 0 @@ -61,17 +62,20 @@ class ProductInline(admin.TabularInline): class CompanyInline(admin.TabularInline): model = models.Company + raw_id_fields = ['establishment', 'address'] extra = 0 class EstablishmentNote(admin.TabularInline): model = models.EstablishmentNote extra = 0 + raw_id_fields = ['user', ] -class PurchasedProduct(admin.TabularInline): +class PurchasedProductInline(admin.TabularInline): model = PurchasedProduct extra = 0 + raw_id_fields = ['product', ] @admin.register(models.Establishment) @@ -80,13 +84,12 @@ class EstablishmentAdmin(BaseModelAdminMixin, admin.ModelAdmin): list_display = ['id', '__str__', 'image_tag', ] search_fields = ['id', 'name', 'index_name', 'slug'] list_filter = ['public_mark', 'toque_number'] - inlines = [GalleryImageInline, CompanyInline, EstablishmentNote, - PurchasedProduct] - + inlines = [CompanyInline, EstablishmentNote, GalleryImageInline, + PurchasedProductInline, ] # inlines = [ # AwardInline, ContactPhoneInline, ContactEmailInline, # ReviewInline, CommentInline, ProductInline] - raw_id_fields = ('address',) + raw_id_fields = ('address', 'collections', 'tags', 'schedule') @admin.register(models.Position) From fe101f3cb062945340175eabefff2ba41f2a4225 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 14:16:44 +0300 Subject: [PATCH 25/27] try another calc strategy (cherry picked from commit 9ab17f7) --- apps/search_indexes/filters.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 3d55bdb4..857d44d2 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -13,25 +13,13 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @staticmethod def calculate_center(first, second): - if second[1] < 0 < first[1]: - reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) - diff = (reverse_first + reverse_second) / 2 - - if reverse_first < reverse_second: - result_part = -180 + (180 + second[1] - diff) - - else: - result_part = 180 - (180 - first[1] - diff) - - elif second[1] < 0 > first[1] or second[1] > 0 < first[1]: - reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) - result_part = ((reverse_first + reverse_second) / 2) * (-1 + (second[1] < 0) * 2) - + if second[1] < first[1]: + res_longtitude = first[1] + (first[1] - second[1]) / 2 else: - result_part = (first[1] + second[1]) / 2 + res_longtitude = first[1] + (second[1] - first[1]) / 2 # return (first[0] + second[0]) / 2, result_part - return (first[0] + second[0]) / 2, (first[1] + second[1]) / 2 + return (first[0] + second[0]) / 2, res_longtitude def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view) From aa8304acaf4a9ebf2d5d3a9b3d65b959c344ef9c Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 14:19:56 +0300 Subject: [PATCH 26/27] Revert "try another calc strategy" This reverts commit fe101f3 --- apps/search_indexes/filters.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 857d44d2..3d55bdb4 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -13,13 +13,25 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @staticmethod def calculate_center(first, second): - if second[1] < first[1]: - res_longtitude = first[1] + (first[1] - second[1]) / 2 + if second[1] < 0 < first[1]: + reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) + diff = (reverse_first + reverse_second) / 2 + + if reverse_first < reverse_second: + result_part = -180 + (180 + second[1] - diff) + + else: + result_part = 180 - (180 - first[1] - diff) + + elif second[1] < 0 > first[1] or second[1] > 0 < first[1]: + reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) + result_part = ((reverse_first + reverse_second) / 2) * (-1 + (second[1] < 0) * 2) + else: - res_longtitude = first[1] + (second[1] - first[1]) / 2 + result_part = (first[1] + second[1]) / 2 # return (first[0] + second[0]) / 2, result_part - return (first[0] + second[0]) / 2, res_longtitude + return (first[0] + second[0]) / 2, (first[1] + second[1]) / 2 def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view) From f9479dd7a61e0761e3d8ab5322c5c932a76372c0 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Thu, 5 Dec 2019 14:30:41 +0300 Subject: [PATCH 27/27] calc center again --- apps/search_indexes/filters.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/apps/search_indexes/filters.py b/apps/search_indexes/filters.py index 3d55bdb4..63f1dbb8 100644 --- a/apps/search_indexes/filters.py +++ b/apps/search_indexes/filters.py @@ -13,25 +13,13 @@ class CustomGeoSpatialFilteringFilterBackend(GeoSpatialFilteringFilterBackend): @staticmethod def calculate_center(first, second): - if second[1] < 0 < first[1]: - reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) - diff = (reverse_first + reverse_second) / 2 - - if reverse_first < reverse_second: - result_part = -180 + (180 + second[1] - diff) - - else: - result_part = 180 - (180 - first[1] - diff) - - elif second[1] < 0 > first[1] or second[1] > 0 < first[1]: - reverse_first, reverse_second = 180 - abs(first[1]), 180 - abs(second[1]) - result_part = ((reverse_first + reverse_second) / 2) * (-1 + (second[1] < 0) * 2) - + if second[1] < first[1]: + res_longtitude = first[1] + (360 + abs(first[1]) - abs(second[1])) / 2 else: - result_part = (first[1] + second[1]) / 2 + res_longtitude = first[1] + (second[1] - first[1]) / 2 # return (first[0] + second[0]) / 2, result_part - return (first[0] + second[0]) / 2, (first[1] + second[1]) / 2 + return (first[0] + second[0]) / 2, res_longtitude def filter_queryset(self, request, queryset, view): ret = super().filter_queryset(request, queryset, view)