phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/origin/feature/migrate-etablishment' into origin/feature/migrate-etablishment

Browse Source
This commit is contained in:
alex 2019-10-27 20:55:34 +03:00
commit f146829d10
16 changed files with 402 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/advertisement/migrations/0004_auto_20191025_0903.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-25 09:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('advertisement', '0003_auto_20190919_1344'),
]
operations = [
migrations.AlterField(
model_name='advertisement',
name='block_level',
field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'),
),
]

2
apps/advertisement/models.py
View File

@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin):
url = models.URLField(verbose_name=_('Ad URL'))
width = models.PositiveIntegerField(verbose_name=_('Block width'))
height = models.PositiveIntegerField(verbose_name=_('Block height'))
block_level = models.CharField(verbose_name=_('Block level'), max_length=10)
block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True)
target_languages = models.ManyToManyField(Language)
class Meta:

20
apps/advertisement/transfer_data.py Normal file
View File

@ -0,0 +1,20 @@
from pprint import pprint
from django.db.models import Value, IntegerField, F
from transfer.models import Ads
from transfer.serializers.advertisement import AdvertisementSerializer
def transfer_advertisement():
queryset = Ads.objects.filter(href__isnull=False)
serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"News serializer errors: {serialized_data.errors}")
data_types = {
"commercial": [transfer_advertisement]
}

18
apps/establishment/migrations/0044_establishment_old_id.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-27 07:56
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('establishment', '0043_establishment_currency'),
]
operations = [
migrations.AddField(
model_name='establishment',
name='old_id',
field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'),
),
]

6
apps/establishment/models.py
View File

@ -14,6 +14,8 @@ from django.db.models import When, Case, F, ExpressionWrapper, Subquery, Q
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from phonenumber_field.modelfields import PhoneNumberField
from pytz import timezone as ptz
from timezone_field import TimeZoneField
from collection.models import Collection
from location.models import Address
@ -21,7 +23,6 @@ from main.models import Award, Currency
from review.models import Review
from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin,
TranslatedFieldsMixin, BaseAttributes)
from timezone_field import TimeZoneField
# todo: establishment type&subtypes check
@ -296,6 +297,7 @@ class EstablishmentQuerySet(models.QuerySet):
class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin):
"""Establishment model."""
old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
name = models.CharField(_('name'), max_length=255, default='')
name_translated = models.CharField(_('Transliterated name'),
max_length=255, default='')
@ -436,7 +438,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin):
@property
def works_now(self):
""" Is establishment working now """
now_at_est_tz = datetime.now(tz=self.tz)
now_at_est_tz = datetime.now(tz=ptz(self.tz))
current_week = now_at_est_tz.weekday()
schedule_for_today = self.schedule.filter(weekday=current_week).first()
if schedule_for_today is None or schedule_for_today.closed_at is None or schedule_for_today.opening_at is None:

64
apps/establishment/transfer_data.py
View File

@ -7,15 +7,41 @@ from transfer.serializers.establishment import EstablishmentSerializer
def transfer_establishment():
result = []
old_establishments = Establishments.objects.all().prefetch_related('establishmentinfos_set', 'schedules_set')
old_establishments = Establishments.objects.exclude(
type='Wineyard',
).prefetch_related(
'establishmentinfos_set',
'schedules_set',
'descriptions_set',
)
for item in old_establishments:
data = {
'old_id': item.id,
'name': item.name,
'name_translated': item.index_name,
'slug': item.slug,
'type': item.type,
'location': item.location.id,
'schedules': [],
'phone': item.phone,
'created': item.created_at,
'description': {},
'tz': None,
'website': None,
'facebook': None,
'twitter': None,
'lafourchette': None,
'booking': None,
'schedules': None,
'location': None,
'email': None,
}
if item.location:
data.update({
'location': item.location.id,
'tz': item.location.timezone,
})
# Инфо
info = item.establishmentinfos_set.first()
if info:
data.update({
@ -24,22 +50,30 @@ def transfer_establishment():
'twitter': info.twitter,
'lafourchette': info.lafourchette,
'booking': info.booking_url,
'email': info.email,
})
for schedule in item.schedules_set.all():
data['schedules'].append({
'raw_timetable': schedule.timetable
# Время работы
schedule = item.schedules_set.first()
if schedule:
data.update({
'schedules': schedule.timetable,
})
# Описание
descriptions = item.descriptions_set.all()
for description in descriptions:
data['description'].update({
description.locale: description.text,
})
result.append(data)
print('-' * 30)
print(len(result))
pprint(result[0])
# serialized_data = EstablishmentSerializer(data=result, many=True)
# if serialized_data.is_valid():
# serialized_data.save()
# else:
# pprint(f"Establishment serializer errors: {serialized_data.errors}")
serialized_data = EstablishmentSerializer(data=result, many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"Establishment serializer errors: {serialized_data.errors}")
data_types = {

20
apps/gallery/migrations/0005_auto_20191027_0756.py Normal file
View File

@ -0,0 +1,20 @@
# Generated by Django 2.2.4 on 2019-10-27 07:56
from django.db import migrations
import sorl.thumbnail.fields
import utils.methods
class Migration(migrations.Migration):
dependencies = [
('gallery', '0004_merge_20191025_0906'),
]
operations = [
migrations.AlterField(
model_name='image',
name='image',
field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'),
),
]

6
apps/news/models.py
View File

@ -1,11 +1,12 @@
"""News app models."""
from django.db import models
from django.contrib.contenttypes import fields as generic
from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from rest_framework.reverse import reverse
from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin
from rating.models import Rating
from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin
class NewsType(models.Model):
@ -217,7 +218,6 @@ class NewsGalleryQuerySet(models.QuerySet):
class NewsGallery(models.Model):
news = models.ForeignKey(News, null=True,
related_name='news_gallery',
on_delete=models.CASCADE,

3
apps/transfer/management/commands/transfer.py
View File

@ -15,8 +15,9 @@ class Command(BaseCommand):
'subscriber',
'recipe',
'partner',
'gallery',
'establishment',
'gallery',
'commercial'
]
def handle(self, *args, **options):

42
apps/transfer/models.py
View File

@ -277,7 +277,6 @@ class Collections(MigrateMixin):
db_table = 'collections'
# class CollectionEvents(MigrateMixin):
# using = 'legacy'
#
@ -296,7 +295,7 @@ class Collections(MigrateMixin):
# class CollectionEventAvailabilities(MigrateMixin):
# using = 'legacy'
#TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts
# TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts
# collection_event = models.ForeignKey('CollectionEvents', models.DO_NOTHING, blank=True, null=True)
# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True)
@ -369,6 +368,7 @@ class GuideFilters(MigrateMixin):
managed = False
db_table = 'guide_filters'
#
# class GuideSections(MigrateMixin):
# using = 'legacy'
@ -447,6 +447,20 @@ class Establishments(MigrateMixin):
db_table = 'establishments'
class Descriptions(MigrateMixin):
using = 'legacy'
establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True)
locale = models.CharField(max_length=5, blank=True, null=True)
text = models.TextField(blank=True, null=True)
created_at = models.DateTimeField()
updated_at = models.DateTimeField()
class Meta:
managed = False
db_table = 'descriptions'
# class EstablishmentAssets(MigrateMixin):
# using = 'legacy'
#
@ -540,7 +554,7 @@ class EstablishmentInfos(MigrateMixin):
#
# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True)
#TODO: модели Merchandises нету в гугл таблице Check Migrations
# TODO: модели Merchandises нету в гугл таблице Check Migrations
# merchandise = models.ForeignKey('Merchandises', models.DO_NOTHING, blank=True, null=True)
# gifted = models.IntegerField(blank=True, null=True)
@ -664,7 +678,7 @@ class Reviews(MigrateMixin):
aasm_state = models.CharField(max_length=255, blank=True, null=True)
reviewer_id = models.IntegerField()
priority = models.IntegerField(blank=True, null=True)
#TODO: модель Products в postgres закомментирована
# TODO: модель Products в postgres закомментирована
# product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True)
received_at = models.DateTimeField(blank=True, null=True)
reviewer_name = models.CharField(max_length=255, blank=True, null=True)
@ -788,3 +802,23 @@ class PageMetadata(MigrateMixin):
class Meta:
managed = False
db_table = 'page_metadata'
class Ads(MigrateMixin):
using = 'legacy'
site_id = models.IntegerField(blank=True, null=True)
href = models.CharField(max_length=255, blank=True, null=True)
start_at = models.DateTimeField(blank=True, null=True)
expire_at = models.DateTimeField(blank=True, null=True)
attachment_file_name = models.CharField(max_length=255, blank=True, null=True)
attachment_content_type = models.CharField(max_length=255, blank=True, null=True)
attachment_file_size = models.IntegerField(blank=True, null=True)
attachment_updated_at = models.DateTimeField(blank=True, null=True)
geometries = models.CharField(max_length=1024, blank=True, null=True)
created_at = models.DateTimeField()
updated_at = models.DateTimeField()
class Meta:
managed = False
db_table = 'ads'

39
apps/transfer/serializers/advertisement.py Normal file
View File

@ -0,0 +1,39 @@
from rest_framework import serializers
from advertisement.models import Advertisement
import yaml
class AdvertisementSerializer(serializers.ModelSerializer):
href = serializers.CharField()
geometries = serializers.CharField(max_length=1024)
class Meta:
model = Advertisement
fields = (
"href",
"geometries"
)
def validate(self, data):
data["url"] = data["href"]
data["width"] = self.get_width(data["geometries"])
data["height"] = self.get_height(data["geometries"])
data.pop("href")
data.pop("geometries")
return data
def create(self, validated_data):
return Advertisement.objects.create(**validated_data)
def get_width(self, data):
data = self.parse_geometries(data)
return int(float(data["width"]))
def get_height(self, data):
data = self.parse_geometries(data)
return int(float(data["height"]))
def parse_geometries(self, geo_str):
clear_str = "!ruby/object:Paperclip::Geometry"
content_dict = yaml.safe_load(geo_str.replace(clear_str, ''))
return content_dict[':original']

149
apps/transfer/serializers/establishment.py
View File

@ -1,26 +1,147 @@
from django.core.exceptions import MultipleObjectsReturned, ValidationError
from django.db import transaction
from rest_framework import serializers
from establishment.models import Establishment
from establishment.models import Establishment, ContactEmail, ContactPhone, EstablishmentType
from location.models import Address
from timetable.models import Timetable
from utils.legacy_parser import parse_legacy_schedule_content
from utils.slug_generator import generate_unique_slug
class EstablishmentSerializer(serializers.ModelSerializer):
slug = serializers.CharField(allow_null=True, allow_blank=True)
type = serializers.CharField()
description = serializers.DictField(
allow_null=True,
child=serializers.CharField(allow_null=True),
)
schedules = serializers.CharField(allow_null=True, allow_blank=True)
location = serializers.IntegerField(allow_null=True)
email = serializers.CharField(allow_null=True, allow_blank=True)
phone = serializers.CharField(allow_null=True, allow_blank=True)
website = serializers.CharField(allow_null=True, allow_blank=True)
facebook = serializers.CharField(allow_null=True, allow_blank=True)
twitter = serializers.CharField(allow_null=True, allow_blank=True)
booking = serializers.CharField(allow_null=True, allow_blank=True)
tz = serializers.CharField(allow_null=True, allow_blank=True)
created = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
class Meta:
model = Establishment
fields = '__all__'
fields = (
'created',
'old_id', # +
'name', # +
'name_translated', # +
'tz', # +
'website', # +
'facebook', # +
'twitter', # +
'lafourchette', # +
'booking', # +
'type', # + см в JIRA
'slug', # + сгенерировать уникальный слаг
'description', # + (разобрал в transfer_data)
'schedules', # + разобрать RUBY словать (2 варианта)
'location', # + получить новые объекты Address по old_id
'email', # + создать объект для ContactEmail
'phone', # + создать объект для ContactPhone
)
def validate(self, data):
pass
# data.update({
# 'state': self.get_state(data),
# 'template': self.get_template(data),
# 'title': self.get_title(data),
# 'description': self.get_description(data),
# })
# data.pop('body')
# data.pop('locale')
# return data
data.update({
'slug': generate_unique_slug(Establishment, data['slug'] if data['slug'] else data['name']),
'address_id': self.get_address(data['location']),
'establishment_type_id': self.get_type(data),
})
data.pop('location')
data.pop('type')
return data
@transaction.atomic
def create(self, validated_data):
pass
# return News.objects.create(**validated_data)
email = validated_data.pop('email')
phone = validated_data.pop('phone')
schedules = validated_data.pop('schedules')
establishment = Establishment.objects.create(**validated_data)
if email:
ContactEmail.objects.get_or_create(
email=email,
establishment=establishment,
)
if phone:
ContactPhone.objects.get_or_create(
phone=phone,
establishment=establishment,
)
if schedules:
new_schedules = self.get_schedules(schedules)
for schedule in new_schedules:
establishment.schedule.add(schedule)
establishment.save()
return establishment
@staticmethod
def get_address(address):
# return Address.objects.filter(old_id=address).first()
return None
@staticmethod
def get_type(data):
types = {
'Restaurant': EstablishmentType.RESTAURANT,
'Shop': EstablishmentType.ARTISAN,
}
obj, _ = EstablishmentType.objects.get_or_create(index_name=types[data['type']])
return obj.id
@staticmethod
def get_schedules(schedules):
result = []
legacy_dict = parse_legacy_schedule_content(schedules)
weekdays = {
'su': Timetable.SUNDAY,
'mo': Timetable.MONDAY,
'tu': Timetable.THURSDAY,
'we': Timetable.WEDNESDAY,
'th': Timetable.THURSDAY,
'fr': Timetable.FRIDAY,
'sa': Timetable.SATURDAY,
}
for key, val in legacy_dict.items():
payload = {
'weekday': weekdays[key],
'lunch_start': None,
'lunch_end': None,
'dinner_start': None,
'dinner_end': None,
'opening_at': None,
'closed_at': None,
}
if val['morning']:
payload.update({
'lunch_start': val['morning']['start'],
'lunch_end': val['morning']['end'],
})
if val['afternoon']:
payload.update({
'dinner_start': val['afternoon']['start'],
'dinner_end': val['afternoon']['end'],
})
try:
obj, _ = Timetable.objects.get_or_create(**payload)
except ValidationError:
obj = None
except MultipleObjectsReturned:
obj = Timetable.objects.filter(**payload).first()
if obj:
result.append(obj)
return result

3
apps/transfer/serializers/news.py
View File

@ -2,10 +2,12 @@ from rest_framework import serializers
from news.models import News
from utils.legacy_parser import parse_legacy_news_content
from utils.slug_generator import generate_unique_slug
class NewsSerializer(serializers.ModelSerializer):
locale = serializers.CharField()
slug = serializers.CharField()
body = serializers.CharField(allow_null=True)
title = serializers.CharField()
template = serializers.CharField()
@ -27,6 +29,7 @@ class NewsSerializer(serializers.ModelSerializer):
def validate(self, data):
data.update({
'slug': generate_unique_slug(News, data['slug']),
'state': self.get_state(data),
'template': self.get_template(data),
'title': self.get_title(data),

4
apps/transfer/serializers/recipe.py
View File

@ -1,6 +1,6 @@
from rest_framework import serializers
from recipe.models import Recipe
from utils.legacy_parser import parse_legacy_content
from utils.legacy_parser import parse_legacy_news_content
class RecipeSerializer(serializers.ModelSerializer):
@ -51,5 +51,5 @@ class RecipeSerializer(serializers.ModelSerializer):
# return {"en-GB": desc}
content = None
if obj['body']:
content = parse_legacy_content(obj['body'])
content = parse_legacy_news_content(obj['body'])
return {obj['locale']: content}

40
apps/utils/legacy_parser.py
View File

@ -1,3 +1,5 @@
from pprint import pprint
import yaml
@ -13,8 +15,36 @@ def parse_legacy_news_content(legacy_content):
def parse_legacy_schedule_content(legacy_content):
clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess'
content_dict = yaml.safe_load(legacy_content.replace(clear_str, ''))
result = ''
# TODO: вернуть валидные данные расписания для новой модели
return result
initial = legacy_content
s1 = "!ruby/object:ActionController::Parameters"
s2 = "!ruby/hash:ActiveSupport::HashWithIndifferentAccess"
for _ in (s1, s2):
legacy_content = legacy_content.replace(_, "")
content_dict = yaml.safe_load(legacy_content)
if s1 not in initial or s2 not in initial:
return yaml.safe_load(legacy_content)
result = {}
for k, v in content_dict.items():
try:
if v["parameters"]["afternoon"]:
afternoon = v["parameters"]["afternoon"]["parameters"]
else:
afternoon = None
if v["parameters"]["morning"]:
morning = v["parameters"]["morning"]["parameters"]
else:
morning = None
data = {"afternoon": afternoon, "morning": morning}
result.update({k: data})
except KeyError:
print('--------' * 7)
pprint(initial)
raise KeyError
return result

15
apps/utils/slug_generator.py Normal file
View File

@ -0,0 +1,15 @@
from django.utils.text import slugify
def generate_unique_slug(klass, text):
"""
return unique slug if origin slug is exist.
eg: `foo-bar` => `foo-bar-1`
"""
origin_slug = slugify(text)
unique_slug = origin_slug
numb = 1
while klass.objects.filter(slug=unique_slug).exists():
unique_slug = f'{origin_slug}-{numb}'
numb += 1
return unique_slug