diff --git a/apps/advertisement/migrations/0004_auto_20191025_0903.py b/apps/advertisement/migrations/0004_auto_20191025_0903.py new file mode 100644 index 00000000..f633d67c --- /dev/null +++ b/apps/advertisement/migrations/0004_auto_20191025_0903.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('advertisement', '0003_auto_20190919_1344'), + ] + + operations = [ + migrations.AlterField( + model_name='advertisement', + name='block_level', + field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'), + ), + ] diff --git a/apps/advertisement/models.py b/apps/advertisement/models.py index fdc25988..62b41428 100644 --- a/apps/advertisement/models.py +++ b/apps/advertisement/models.py @@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin): url = models.URLField(verbose_name=_('Ad URL')) width = models.PositiveIntegerField(verbose_name=_('Block width')) height = models.PositiveIntegerField(verbose_name=_('Block height')) - block_level = models.CharField(verbose_name=_('Block level'), max_length=10) + block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True) target_languages = models.ManyToManyField(Language) class Meta: diff --git a/apps/advertisement/transfer_data.py b/apps/advertisement/transfer_data.py new file mode 100644 index 00000000..3af16286 --- /dev/null +++ b/apps/advertisement/transfer_data.py @@ -0,0 +1,20 @@ +from pprint import pprint +from django.db.models import Value, IntegerField, F +from transfer.models import Ads +from transfer.serializers.advertisement import AdvertisementSerializer + + +def transfer_advertisement(): + queryset = Ads.objects.filter(href__isnull=False) + + serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "commercial": [transfer_advertisement] +} \ No newline at end of file diff --git a/apps/establishment/migrations/0044_establishment_old_id.py b/apps/establishment/migrations/0044_establishment_old_id.py new file mode 100644 index 00000000..9f630c2b --- /dev/null +++ b/apps/establishment/migrations/0044_establishment_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 07:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0043_establishment_currency'), + ] + + operations = [ + migrations.AddField( + model_name='establishment', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 304ea2a6..3fc623a9 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -14,6 +14,8 @@ from django.db.models import When, Case, F, ExpressionWrapper, Subquery, Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ from phonenumber_field.modelfields import PhoneNumberField +from pytz import timezone as ptz +from timezone_field import TimeZoneField from collection.models import Collection from location.models import Address @@ -21,7 +23,6 @@ from main.models import Award, Currency from review.models import Review from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin, TranslatedFieldsMixin, BaseAttributes) -from timezone_field import TimeZoneField # todo: establishment type&subtypes check @@ -296,6 +297,7 @@ class EstablishmentQuerySet(models.QuerySet): class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): """Establishment model.""" + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) name = models.CharField(_('name'), max_length=255, default='') name_translated = models.CharField(_('Transliterated name'), max_length=255, default='') @@ -436,7 +438,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): @property def works_now(self): """ Is establishment working now """ - now_at_est_tz = datetime.now(tz=self.tz) + now_at_est_tz = datetime.now(tz=ptz(self.tz)) current_week = now_at_est_tz.weekday() schedule_for_today = self.schedule.filter(weekday=current_week).first() if schedule_for_today is None or schedule_for_today.closed_at is None or schedule_for_today.opening_at is None: diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index ebb13c4a..bc0ae034 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -7,15 +7,41 @@ from transfer.serializers.establishment import EstablishmentSerializer def transfer_establishment(): result = [] - old_establishments = Establishments.objects.all().prefetch_related('establishmentinfos_set', 'schedules_set') + old_establishments = Establishments.objects.exclude( + type='Wineyard', + ).prefetch_related( + 'establishmentinfos_set', + 'schedules_set', + 'descriptions_set', + ) for item in old_establishments: data = { + 'old_id': item.id, 'name': item.name, + 'name_translated': item.index_name, 'slug': item.slug, 'type': item.type, - 'location': item.location.id, - 'schedules': [], + 'phone': item.phone, + 'created': item.created_at, + 'description': {}, + 'tz': None, + 'website': None, + 'facebook': None, + 'twitter': None, + 'lafourchette': None, + 'booking': None, + 'schedules': None, + 'location': None, + 'email': None, } + + if item.location: + data.update({ + 'location': item.location.id, + 'tz': item.location.timezone, + }) + + # Инфо info = item.establishmentinfos_set.first() if info: data.update({ @@ -24,22 +50,30 @@ def transfer_establishment(): 'twitter': info.twitter, 'lafourchette': info.lafourchette, 'booking': info.booking_url, + 'email': info.email, }) - for schedule in item.schedules_set.all(): - data['schedules'].append({ - 'raw_timetable': schedule.timetable + + # Время работы + schedule = item.schedules_set.first() + if schedule: + data.update({ + 'schedules': schedule.timetable, }) + + # Описание + descriptions = item.descriptions_set.all() + for description in descriptions: + data['description'].update({ + description.locale: description.text, + }) + result.append(data) - print('-' * 30) - print(len(result)) - pprint(result[0]) - - # serialized_data = EstablishmentSerializer(data=result, many=True) - # if serialized_data.is_valid(): - # serialized_data.save() - # else: - # pprint(f"Establishment serializer errors: {serialized_data.errors}") + serialized_data = EstablishmentSerializer(data=result, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Establishment serializer errors: {serialized_data.errors}") data_types = { diff --git a/apps/gallery/migrations/0005_auto_20191027_0756.py b/apps/gallery/migrations/0005_auto_20191027_0756.py new file mode 100644 index 00000000..d60745f1 --- /dev/null +++ b/apps/gallery/migrations/0005_auto_20191027_0756.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-27 07:56 + +from django.db import migrations +import sorl.thumbnail.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0004_merge_20191025_0906'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'), + ), + ] diff --git a/apps/news/models.py b/apps/news/models.py index dbb2f5bf..306e4906 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -1,11 +1,12 @@ """News app models.""" -from django.db import models from django.contrib.contenttypes import fields as generic +from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework.reverse import reverse -from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin + from rating.models import Rating +from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin class NewsType(models.Model): @@ -217,7 +218,6 @@ class NewsGalleryQuerySet(models.QuerySet): class NewsGallery(models.Model): - news = models.ForeignKey(News, null=True, related_name='news_gallery', on_delete=models.CASCADE, diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 7de18f5d..e9e52712 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -15,8 +15,9 @@ class Command(BaseCommand): 'subscriber', 'recipe', 'partner', - 'gallery', 'establishment', + 'gallery', + 'commercial' ] def handle(self, *args, **options): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f2a634bf..896d478b 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -277,7 +277,6 @@ class Collections(MigrateMixin): db_table = 'collections' - # class CollectionEvents(MigrateMixin): # using = 'legacy' # @@ -296,7 +295,7 @@ class Collections(MigrateMixin): # class CollectionEventAvailabilities(MigrateMixin): # using = 'legacy' -#TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts +# TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts # collection_event = models.ForeignKey('CollectionEvents', models.DO_NOTHING, blank=True, null=True) # establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) @@ -369,6 +368,7 @@ class GuideFilters(MigrateMixin): managed = False db_table = 'guide_filters' + # # class GuideSections(MigrateMixin): # using = 'legacy' @@ -447,6 +447,20 @@ class Establishments(MigrateMixin): db_table = 'establishments' +class Descriptions(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + text = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'descriptions' + + # class EstablishmentAssets(MigrateMixin): # using = 'legacy' # @@ -540,7 +554,7 @@ class EstablishmentInfos(MigrateMixin): # # establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) - #TODO: модели Merchandises нету в гугл таблице Check Migrations +# TODO: модели Merchandises нету в гугл таблице Check Migrations # merchandise = models.ForeignKey('Merchandises', models.DO_NOTHING, blank=True, null=True) # gifted = models.IntegerField(blank=True, null=True) @@ -664,7 +678,7 @@ class Reviews(MigrateMixin): aasm_state = models.CharField(max_length=255, blank=True, null=True) reviewer_id = models.IntegerField() priority = models.IntegerField(blank=True, null=True) - #TODO: модель Products в postgres закомментирована + # TODO: модель Products в postgres закомментирована # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) received_at = models.DateTimeField(blank=True, null=True) reviewer_name = models.CharField(max_length=255, blank=True, null=True) @@ -788,3 +802,23 @@ class PageMetadata(MigrateMixin): class Meta: managed = False db_table = 'page_metadata' + + +class Ads(MigrateMixin): + using = 'legacy' + + site_id = models.IntegerField(blank=True, null=True) + href = models.CharField(max_length=255, blank=True, null=True) + start_at = models.DateTimeField(blank=True, null=True) + expire_at = models.DateTimeField(blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'ads' diff --git a/apps/transfer/serializers/advertisement.py b/apps/transfer/serializers/advertisement.py new file mode 100644 index 00000000..2e3a488f --- /dev/null +++ b/apps/transfer/serializers/advertisement.py @@ -0,0 +1,39 @@ +from rest_framework import serializers +from advertisement.models import Advertisement +import yaml + + +class AdvertisementSerializer(serializers.ModelSerializer): + href = serializers.CharField() + geometries = serializers.CharField(max_length=1024) + + class Meta: + model = Advertisement + fields = ( + "href", + "geometries" + ) + + def validate(self, data): + data["url"] = data["href"] + data["width"] = self.get_width(data["geometries"]) + data["height"] = self.get_height(data["geometries"]) + data.pop("href") + data.pop("geometries") + return data + + def create(self, validated_data): + return Advertisement.objects.create(**validated_data) + + def get_width(self, data): + data = self.parse_geometries(data) + return int(float(data["width"])) + + def get_height(self, data): + data = self.parse_geometries(data) + return int(float(data["height"])) + + def parse_geometries(self, geo_str): + clear_str = "!ruby/object:Paperclip::Geometry" + content_dict = yaml.safe_load(geo_str.replace(clear_str, '')) + return content_dict[':original'] \ No newline at end of file diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index fe81c835..f5babf7b 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -1,26 +1,147 @@ +from django.core.exceptions import MultipleObjectsReturned, ValidationError +from django.db import transaction from rest_framework import serializers -from establishment.models import Establishment +from establishment.models import Establishment, ContactEmail, ContactPhone, EstablishmentType +from location.models import Address +from timetable.models import Timetable +from utils.legacy_parser import parse_legacy_schedule_content +from utils.slug_generator import generate_unique_slug class EstablishmentSerializer(serializers.ModelSerializer): + slug = serializers.CharField(allow_null=True, allow_blank=True) + type = serializers.CharField() + description = serializers.DictField( + allow_null=True, + child=serializers.CharField(allow_null=True), + ) + schedules = serializers.CharField(allow_null=True, allow_blank=True) + location = serializers.IntegerField(allow_null=True) + email = serializers.CharField(allow_null=True, allow_blank=True) + phone = serializers.CharField(allow_null=True, allow_blank=True) + website = serializers.CharField(allow_null=True, allow_blank=True) + facebook = serializers.CharField(allow_null=True, allow_blank=True) + twitter = serializers.CharField(allow_null=True, allow_blank=True) + booking = serializers.CharField(allow_null=True, allow_blank=True) + tz = serializers.CharField(allow_null=True, allow_blank=True) + created = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') class Meta: model = Establishment - fields = '__all__' + fields = ( + 'created', + 'old_id', # + + 'name', # + + 'name_translated', # + + 'tz', # + + 'website', # + + 'facebook', # + + 'twitter', # + + 'lafourchette', # + + 'booking', # + + 'type', # + см в JIRA + 'slug', # + сгенерировать уникальный слаг + 'description', # + (разобрал в transfer_data) + 'schedules', # + разобрать RUBY словать (2 варианта) + 'location', # + получить новые объекты Address по old_id + 'email', # + создать объект для ContactEmail + 'phone', # + создать объект для ContactPhone + ) def validate(self, data): - pass - # data.update({ - # 'state': self.get_state(data), - # 'template': self.get_template(data), - # 'title': self.get_title(data), - # 'description': self.get_description(data), - # }) - # data.pop('body') - # data.pop('locale') - # return data + data.update({ + 'slug': generate_unique_slug(Establishment, data['slug'] if data['slug'] else data['name']), + 'address_id': self.get_address(data['location']), + 'establishment_type_id': self.get_type(data), + }) + data.pop('location') + data.pop('type') + return data + @transaction.atomic def create(self, validated_data): - pass - # return News.objects.create(**validated_data) + email = validated_data.pop('email') + phone = validated_data.pop('phone') + schedules = validated_data.pop('schedules') + + establishment = Establishment.objects.create(**validated_data) + if email: + ContactEmail.objects.get_or_create( + email=email, + establishment=establishment, + ) + if phone: + ContactPhone.objects.get_or_create( + phone=phone, + establishment=establishment, + ) + if schedules: + new_schedules = self.get_schedules(schedules) + for schedule in new_schedules: + establishment.schedule.add(schedule) + establishment.save() + + return establishment + + @staticmethod + def get_address(address): + # return Address.objects.filter(old_id=address).first() + return None + + @staticmethod + def get_type(data): + types = { + 'Restaurant': EstablishmentType.RESTAURANT, + 'Shop': EstablishmentType.ARTISAN, + } + obj, _ = EstablishmentType.objects.get_or_create(index_name=types[data['type']]) + return obj.id + + @staticmethod + def get_schedules(schedules): + result = [] + legacy_dict = parse_legacy_schedule_content(schedules) + + weekdays = { + 'su': Timetable.SUNDAY, + 'mo': Timetable.MONDAY, + 'tu': Timetable.THURSDAY, + 'we': Timetable.WEDNESDAY, + 'th': Timetable.THURSDAY, + 'fr': Timetable.FRIDAY, + 'sa': Timetable.SATURDAY, + } + + for key, val in legacy_dict.items(): + payload = { + 'weekday': weekdays[key], + 'lunch_start': None, + 'lunch_end': None, + 'dinner_start': None, + 'dinner_end': None, + 'opening_at': None, + 'closed_at': None, + } + if val['morning']: + payload.update({ + 'lunch_start': val['morning']['start'], + 'lunch_end': val['morning']['end'], + }) + if val['afternoon']: + payload.update({ + 'dinner_start': val['afternoon']['start'], + 'dinner_end': val['afternoon']['end'], + }) + + try: + obj, _ = Timetable.objects.get_or_create(**payload) + except ValidationError: + obj = None + except MultipleObjectsReturned: + obj = Timetable.objects.filter(**payload).first() + + if obj: + result.append(obj) + + return result diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index c6a70fa9..e46c5b4a 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -2,10 +2,12 @@ from rest_framework import serializers from news.models import News from utils.legacy_parser import parse_legacy_news_content +from utils.slug_generator import generate_unique_slug class NewsSerializer(serializers.ModelSerializer): locale = serializers.CharField() + slug = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() template = serializers.CharField() @@ -27,6 +29,7 @@ class NewsSerializer(serializers.ModelSerializer): def validate(self, data): data.update({ + 'slug': generate_unique_slug(News, data['slug']), 'state': self.get_state(data), 'template': self.get_template(data), 'title': self.get_title(data), diff --git a/apps/transfer/serializers/recipe.py b/apps/transfer/serializers/recipe.py index 3948bbdd..11698ba1 100644 --- a/apps/transfer/serializers/recipe.py +++ b/apps/transfer/serializers/recipe.py @@ -1,6 +1,6 @@ from rest_framework import serializers from recipe.models import Recipe -from utils.legacy_parser import parse_legacy_content +from utils.legacy_parser import parse_legacy_news_content class RecipeSerializer(serializers.ModelSerializer): @@ -51,5 +51,5 @@ class RecipeSerializer(serializers.ModelSerializer): # return {"en-GB": desc} content = None if obj['body']: - content = parse_legacy_content(obj['body']) + content = parse_legacy_news_content(obj['body']) return {obj['locale']: content} diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py index 7ec0747d..7fbbd133 100644 --- a/apps/utils/legacy_parser.py +++ b/apps/utils/legacy_parser.py @@ -1,3 +1,5 @@ +from pprint import pprint + import yaml @@ -13,8 +15,36 @@ def parse_legacy_news_content(legacy_content): def parse_legacy_schedule_content(legacy_content): - clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' - content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) - result = '' - # TODO: вернуть валидные данные расписания для новой модели - return result \ No newline at end of file + initial = legacy_content + + s1 = "!ruby/object:ActionController::Parameters" + s2 = "!ruby/hash:ActiveSupport::HashWithIndifferentAccess" + + for _ in (s1, s2): + legacy_content = legacy_content.replace(_, "") + content_dict = yaml.safe_load(legacy_content) + + if s1 not in initial or s2 not in initial: + return yaml.safe_load(legacy_content) + + result = {} + for k, v in content_dict.items(): + try: + if v["parameters"]["afternoon"]: + afternoon = v["parameters"]["afternoon"]["parameters"] + else: + afternoon = None + + if v["parameters"]["morning"]: + morning = v["parameters"]["morning"]["parameters"] + else: + morning = None + + data = {"afternoon": afternoon, "morning": morning} + result.update({k: data}) + except KeyError: + print('--------' * 7) + pprint(initial) + raise KeyError + + return result diff --git a/apps/utils/slug_generator.py b/apps/utils/slug_generator.py new file mode 100644 index 00000000..fa1ec772 --- /dev/null +++ b/apps/utils/slug_generator.py @@ -0,0 +1,15 @@ +from django.utils.text import slugify + + +def generate_unique_slug(klass, text): + """ + return unique slug if origin slug is exist. + eg: `foo-bar` => `foo-bar-1` + """ + origin_slug = slugify(text) + unique_slug = origin_slug + numb = 1 + while klass.objects.filter(slug=unique_slug).exists(): + unique_slug = f'{origin_slug}-{numb}' + numb += 1 + return unique_slug