Merge branch 'develop' into fix-subquery

.gitignore

@@ -24,3 +24,4 @@ logs/
 ./docker-compose.override.yml
 
 celerybeat-schedule
+local_files

apps/account/migrations/0018_user_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-29 08:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('account', '0017_merge_20191024_1233'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='user',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/account/models.py

@@ -94,6 +94,8 @@ class User(AbstractUser):
     unconfirmed_email = models.EmailField(_('unconfirmed email'), blank=True, null=True, default=None)
     email_confirmed = models.BooleanField(_('email status'), default=False)
     newsletter = models.NullBooleanField(default=True)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
+
 
     EMAIL_FIELD = 'email'
     USERNAME_FIELD = 'username'
@@ -241,6 +243,20 @@ class User(AbstractUser):
             template_name=settings.CHANGE_EMAIL_TEMPLATE,
             context=context)
 
+    @property
+    def favorite_establishment_ids(self):
+        """Return establishment IDs that in favorites for current user."""
+        return self.favorites.by_content_type(app_label='establishment',
+                                              model='establishment')\
+                             .values_list('object_id', flat=True)
+
+    @property
+    def favorite_recipe_ids(self):
+        """Return recipe IDs that in favorites for current user."""
+        return self.favorites.by_content_type(app_label='recipe',
+                                              model='recipe')\
+                             .values_list('object_id', flat=True)
+
 
 class UserRole(ProjectBaseMixin):
     """UserRole model."""

apps/account/transfer.py

@@ -0,0 +1,53 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+"""
+
+card = {
+    "User": {
+        "data_type": "objects",
+        "dependencies": None,
+        "fields": {
+            "Accounts": {
+                "username": "nickname",
+                "email": "email",
+                "email_confirmed": ("confirmed_at", "django.db.models.BooleanField")
+            },
+            "relations": {
+                "Profiles": {
+                    "key": "account",
+                    "fields": {
+                        "first_name": "firstname",
+                        "last_name": "lastname"
+                    }
+                }
+            }
+        }
+    }
+
+}
+
+used_apps = None
+
+

apps/account/transfer_data.py

@@ -0,0 +1,28 @@
+from django.db.models import Value, IntegerField, F
+from pprint import pprint
+from transfer.models import Profiles, Accounts
+from transfer.serializers.account import UserSerializer
+
+
+def transfer_user():
+    # queryset = Profiles.objects.all()
+    # queryset = queryset.annotate(nickname=F("account__nickname"))
+    # queryset = queryset.annotate(email=F("account__email"))
+    stop_list = ['cyril@tomatic.net',
+                 'cyril2@tomatic.net',
+                 'd.sadykova@id-east.ru',
+                 'd.sadykova@octopod.ru',
+                 'n.yurchenko@id-east.ru']
+    queryset = Accounts.objects.filter(confirmed_at__isnull=False).exclude(email__in=stop_list)
+
+    serialized_data = UserSerializer(data=list(queryset.values()), many=True)
+
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "account": [transfer_user]
+}

apps/advertisement/migrations/0004_auto_20191025_0903.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('advertisement', '0003_auto_20190919_1344'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='advertisement',
+            name='block_level',
+            field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'),
+        ),
+    ]

apps/advertisement/models.py

@@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin):
     url = models.URLField(verbose_name=_('Ad URL'))
     width = models.PositiveIntegerField(verbose_name=_('Block width'))
     height = models.PositiveIntegerField(verbose_name=_('Block height'))
-    block_level = models.CharField(verbose_name=_('Block level'), max_length=10)
+    block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True)
     target_languages = models.ManyToManyField(Language)
 
     class Meta:

apps/advertisement/transfer_data.py

@@ -0,0 +1,20 @@
+from pprint import pprint
+from django.db.models import Value, IntegerField, F
+from transfer.models import Ads
+from transfer.serializers.advertisement import AdvertisementSerializer
+
+
+def transfer_advertisement():
+    queryset = Ads.objects.filter(href__isnull=False)
+
+    serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True)
+
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "commercial": [transfer_advertisement]
+}

apps/collection/transfer.py

@@ -0,0 +1,48 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+"""
+card = {
+    "Collection": {
+        "data_type": "objects",
+        "dependencies": ("Country", ),
+        "fields": {
+            "Collections": {
+                # нету аналогов для полей description, start и end
+                "name": "title",
+                "slug": "slug",
+                "block_size": ("geometries", "django.db.models.JSONField"),
+                "is_publish": ("active", "django.db.models.BooleanField"),
+                "image_url": ("attachment_file_name", "django.db.models.URLField")
+            }
+        },
+        "relations": {
+            # "country": "Country",
+        }
+    },
+    "Guide": {
+        # как работать с ForeignKey на самого себя(self), поле "parent"
+        "data_type": "objects",
+        "dependencies": ("Collection", "self"),
+        "fields":  {
+            "Guides": {
+                # нету аналогов для полей start и end
+                "name": "title"
+            }
+        },
+        "relations": {
+            # аналалог для поля "collection" не найдено
+            # "parent": "Guide",
+            # "collection": "Collection"
+        }
+    }
+}
+
+used_apps = ("location", )

apps/comment/migrations/0004_comment_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-29 12:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('comment', '0003_auto_20191015_0704'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='comment',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/comment/models.py

@@ -45,6 +45,8 @@ class Comment(ProjectBaseMixin):
     objects = CommentQuerySet.as_manager()
     country = models.ForeignKey(Country, verbose_name=_('Country'),
                                 on_delete=models.SET_NULL, null=True)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
+
 
     class Meta:
         """Meta class"""

apps/comment/transfer.py

@@ -0,0 +1,49 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+"""
+
+card = {
+    # как работать с GenericForeignKey(content_type) - ?
+    "Comment": {
+        "data_type": "objects",
+        "dependencies": ("User",),
+        "fields": {
+            "Comments": {
+                "text": "comment",
+                "mark": ("mark", "django.db.models.PositiveIntegerField")
+                # как работать с GenericForeignKey - ?
+                # "content_object" : ""
+            },
+        },
+        "relations": {
+            "Accounts": [
+                (("account", None),
+                 ("User", "user", None, None))
+            ]
+        }
+    }
+}
+
+used_apps = ("account", )

apps/comment/transfer_data.py

@@ -0,0 +1,21 @@
+from pprint import pprint
+from transfer.models import Comments
+from transfer.serializers.comments import CommentSerializer
+
+
+def transfer_comments():
+    queryset = Comments.objects.filter(account__isnull=False, mark__isnull=False)\
+        .only("id", "comment", "mark", "locale", "establishment_id", "account_id")
+
+    serialized_data = CommentSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(serialized_data.errors)
+
+
+data_types = {
+    "tmp": [
+        # transfer_comments
+    ]
+}

apps/establishment/admin.py

@@ -4,6 +4,7 @@ from django.contrib.contenttypes.admin import GenericTabularInline
 from django.utils.translation import gettext_lazy as _
 
 from comment.models import Comment
+from utils.admin import BaseModelAdminMixin
 from establishment import models
 from main.models import Award
 from product.models import Product
@@ -11,12 +12,12 @@ from review import models as review_models
 
 
 @admin.register(models.EstablishmentType)
-class EstablishmentTypeAdmin(admin.ModelAdmin):
+class EstablishmentTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """EstablishmentType admin."""
 
 
 @admin.register(models.EstablishmentSubType)
-class EstablishmentSubTypeAdmin(admin.ModelAdmin):
+class EstablishmentSubTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """EstablishmentSubType admin."""
 
 
@@ -37,7 +38,7 @@ class ContactEmailInline(admin.TabularInline):
     extra = 0
 
 
-class ReviewInline(GenericTabularInline):
+class ReviewInline(BaseModelAdminMixin, GenericTabularInline):
     model = review_models.Review
     extra = 0
 
@@ -53,16 +54,17 @@ class ProductInline(admin.TabularInline):
 
 
 @admin.register(models.Establishment)
-class EstablishmentAdmin(admin.ModelAdmin):
+class EstablishmentAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """Establishment admin."""
     list_display = ['id', '__str__', 'image_tag', ]
     inlines = [
         AwardInline, ContactPhoneInline, ContactEmailInline,
         ReviewInline, CommentInline, ProductInline]
+    raw_id_fields = ('address',)
 
 
 @admin.register(models.Position)
-class PositionAdmin(admin.ModelAdmin):
+class PositionAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """Position admin."""
 
 
@@ -73,7 +75,7 @@ class PlateInline(admin.TabularInline):
 
 
 @admin.register(models.Menu)
-class MenuAdmin(admin.ModelAdmin):
+class MenuAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """Menu admin."""
     list_display = ['id', 'category_translated']
     inlines = [

apps/establishment/management/commands/add_establishment_description.py

@@ -0,0 +1,31 @@
+from django.core.management.base import BaseCommand
+
+from establishment.models import Establishment
+from transfer.models import Descriptions
+
+
+class Command(BaseCommand):
+    help = 'Add description values from old db to new db'
+
+    def handle(self, *args, **kwargs):
+        count = 0
+
+        queryset = Descriptions.objects.all()
+        for obj in queryset:
+            try:
+                establishment = Establishment.objects.get(old_id=obj.establishment.id)
+            except Establishment.DoesNotExist:
+                continue
+            except Establishment.MultipleObjectsReturned:
+                establishment = Establishment.objects.filter(old_id=obj.establishment.id).first()
+            else:
+                description = establishment.description
+                description.update({
+                    obj.locale: obj.text
+                })
+                establishment.description = description
+                establishment.save()
+                count += 1
+                break
+
+        self.stdout.write(self.style.WARNING(f'Updated {count} objects.'))

apps/establishment/management/commands/add_establishment_image.py

@@ -0,0 +1,32 @@
+from django.core.management.base import BaseCommand
+
+from establishment.models import Establishment
+from transfer.models import EstablishmentAssets
+
+
+class Command(BaseCommand):
+    help = 'Add preview_image_url values from old db to new db'
+
+    def handle(self, *args, **kwargs):
+        count = 0
+        url = 'https://1dc3f33f6d-3.optimicdn.com/gaultmillau.com/'
+        raw_qs = EstablishmentAssets.objects.raw(
+            '''SELECT establishment_assets.id, establishment_id, attachment_suffix_url
+               FROM establishment_assets
+               WHERE attachment_file_name IS NOT NULL
+                 AND attachment_suffix_url IS NOT NULL
+                 AND scope = 'public'
+                 AND type = 'Photo'
+                 AND menu_id IS NULL
+               GROUP BY establishment_id;'''
+        )
+        queryset = [vars(query) for query in raw_qs]
+        for obj in queryset:
+            establishment = Establishment.objects.filter(old_id=obj['establishment_id']).first()
+            if establishment:
+                img = url + obj['attachment_suffix_url']
+                establishment.preview_image_url = img
+                establishment.image_url = img
+                establishment.save()
+                count += 1
+        self.stdout.write(self.style.WARNING(f'Updated {count} objects.'))

apps/establishment/management/commands/add_establishment_mark.py

@@ -0,0 +1,46 @@
+from pprint import pprint
+
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+
+from establishment.models import Establishment
+from transfer.models import Reviews
+
+
+class Command(BaseCommand):
+    help = 'Add description values from old db to new db'
+
+    def handle(self, *args, **kwargs):
+        count = 0
+        valid_data = {}
+
+        queryset = Reviews.objects.exclude(
+            Q(establishment_id__isnull=True) |
+            Q(mark__isnull=True)
+        ).filter(aasm_state='published').values_list('establishment_id', 'mark', 'updated_at')
+
+        print(queryset.count())
+
+        for es_id, new_mark, new_date in queryset:
+            try:
+                mark, date = valid_data[es_id]
+            except KeyError:
+                valid_data[es_id] = (new_mark, new_date)
+            else:
+                if new_date > date:
+                    valid_data[es_id] = (new_mark, new_date)
+
+        for key, value in valid_data.items():
+            try:
+                establishment = Establishment.objects.get(old_id=key)
+            except Establishment.DoesNotExist:
+                continue
+            except Establishment.MultipleObjectsReturned:
+                establishment = Establishment.objects.filter(old_id=key).first()
+            else:
+                establishment.public_mark = int(value[0])
+                establishment.save()
+                count += 1
+                break
+
+        self.stdout.write(self.style.WARNING(f'Updated {count} objects.'))

apps/establishment/management/commands/add_publish_data.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+
+from establishment.models import Establishment
+from transfer.models import Establishments
+
+
+class Command(BaseCommand):
+    help = 'Add publish values from old db to new db'
+
+    def handle(self, *args, **kwargs):
+        old_establishments = Establishments.objects.all()
+        count = 0
+        for item in old_establishments:
+            obj = Establishment.objects.filter(old_id=item.id).first()
+            if obj:
+                count += 1
+                obj.is_publish = item.state == 'published'
+                obj.save()
+        self.stdout.write(self.style.WARNING(f'Updated {count} objects.'))

apps/establishment/management/commands/rm_old_establishments.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+
+from establishment.models import Establishment
+
+
+class Command(BaseCommand):
+    help = 'Remove old establishments from new bd'
+
+    def handle(self, *args, **kwargs):
+        old_establishments = Establishment.objects.exclude(old_id__isnull=True)
+        count = old_establishments.count()
+        old_establishments.delete()
+        self.stdout.write(self.style.WARNING(f'Deleted {count} objects.'))

apps/establishment/migrations/0044_establishment_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 07:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('establishment', '0043_establishment_currency'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='establishment',
+            name='old_id',
+            field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'),
+        ),
+    ]

apps/establishment/migrations/0045_auto_20191029_0719.py

@@ -0,0 +1,43 @@
+# Generated by Django 2.2.4 on 2019-10-29 07:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('establishment', '0044_establishment_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='establishment',
+            name='booking',
+            field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Booking URL'),
+        ),
+        migrations.AlterField(
+            model_name='establishment',
+            name='facebook',
+            field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Facebook URL'),
+        ),
+        migrations.AlterField(
+            model_name='establishment',
+            name='lafourchette',
+            field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Lafourchette URL'),
+        ),
+        migrations.AlterField(
+            model_name='establishment',
+            name='preview_image_url',
+            field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Preview image URL path'),
+        ),
+        migrations.AlterField(
+            model_name='establishment',
+            name='twitter',
+            field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Twitter URL'),
+        ),
+        migrations.AlterField(
+            model_name='establishment',
+            name='website',
+            field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Web site URL'),
+        ),
+    ]

apps/establishment/migrations/0046_merge_20191030_0858.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 08:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('establishment', '0045_auto_20191029_0719'),
+        ('establishment', '0044_position_index_name'),
+    ]
+
+    operations = [
+    ]

apps/establishment/migrations/0046_merge_20191030_1618.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 16:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('establishment', '0045_auto_20191029_0719'),
+        ('establishment', '0044_position_index_name'),
+    ]
+
+    operations = [
+    ]

apps/establishment/migrations/0047_merge_20191030_1714.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 17:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('establishment', '0046_merge_20191030_1618'),
+        ('establishment', '0046_merge_20191030_0858'),
+    ]
+
+    operations = [
+    ]

apps/establishment/models.py

@@ -14,6 +14,8 @@ from django.db.models import When, Case, F, ExpressionWrapper, Subquery, Q
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from phonenumber_field.modelfields import PhoneNumberField
+from pytz import timezone as ptz
+from timezone_field import TimeZoneField
 
 from collection.models import Collection
 from location.models import Address
@@ -21,7 +23,6 @@ from main.models import Award, Currency
 from review.models import Review
 from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin,
                           TranslatedFieldsMixin, BaseAttributes)
-from timezone_field import TimeZoneField
 
 
 # todo: establishment type&subtypes check
@@ -248,14 +249,12 @@ class EstablishmentQuerySet(models.QuerySet):
 
     def annotate_in_favorites(self, user):
         """Annotate flag in_favorites"""
-        favorite_establishments = []
+        favorite_establishment_ids = []
         if user.is_authenticated:
-            favorite_establishments = user.favorites.by_content_type(app_label='establishment',
-                                                                     model='establishment') \
-                                                    .values_list('object_id', flat=True)
+            favorite_establishment_ids = user.favorite_establishment_ids
         return self.annotate(in_favorites=Case(
             When(
-                id__in=favorite_establishments,
+                id__in=favorite_establishment_ids,
                 then=True),
             default=False,
             output_field=models.BooleanField(default=False)))
@@ -301,6 +300,7 @@ class EstablishmentQuerySet(models.QuerySet):
 class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin):
     """Establishment model."""
 
+    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
     name = models.CharField(_('name'), max_length=255, default='')
     name_translated = models.CharField(_('Transliterated name'),
                                        max_length=255, default='')
@@ -327,19 +327,19 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin):
     price_level = models.PositiveIntegerField(blank=True, null=True,
                                               default=None,
                                               verbose_name=_('price level'))
-    website = models.URLField(blank=True, null=True, default=None,
+    website = models.URLField(blank=True, null=True, default=None, max_length=255,
                               verbose_name=_('Web site URL'))
-    facebook = models.URLField(blank=True, null=True, default=None,
+    facebook = models.URLField(blank=True, null=True, default=None, max_length=255,
                                verbose_name=_('Facebook URL'))
-    twitter = models.URLField(blank=True, null=True, default=None,
+    twitter = models.URLField(blank=True, null=True, default=None, max_length=255,
                               verbose_name=_('Twitter URL'))
-    lafourchette = models.URLField(blank=True, null=True, default=None,
+    lafourchette = models.URLField(blank=True, null=True, default=None, max_length=255,
                                    verbose_name=_('Lafourchette URL'))
     guestonline_id = models.PositiveIntegerField(blank=True, verbose_name=_('guestonline id'),
                                                  null=True, default=None,)
     lastable_id = models.TextField(blank=True, verbose_name=_('lastable id'), unique=True,
                                                  null=True, default=None,)
-    booking = models.URLField(blank=True, null=True, default=None,
+    booking = models.URLField(blank=True, null=True, default=None, max_length=255,
                               verbose_name=_('Booking URL'))
     is_publish = models.BooleanField(default=False, verbose_name=_('Publish status'))
     schedule = models.ManyToManyField(to='timetable.Timetable',
@@ -355,7 +355,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin):
                                          related_name='establishments',
                                          blank=True, default=None,
                                          verbose_name=_('Collections'))
-    preview_image_url = models.URLField(verbose_name=_('Preview image URL path'),
+    preview_image_url = models.URLField(verbose_name=_('Preview image URL path'), max_length=255,
                                         blank=True, null=True, default=None)
     slug = models.SlugField(unique=True, max_length=255, null=True,
                             verbose_name=_('Establishment slug'))

apps/establishment/serializers/common.py

@@ -5,15 +5,14 @@ from rest_framework import serializers
 from comment import models as comment_models
 from comment.serializers import common as comment_serializers
 from establishment import models
-from favorites.models import Favorites
 from location.serializers import AddressBaseSerializer
 from main.serializers import AwardSerializer, CurrencySerializer
 from review import models as review_models
 from tag.serializers import TagBaseSerializer
 from timetable.serialziers import ScheduleRUDSerializer
 from utils import exceptions as utils_exceptions
-from utils.serializers import ProjectModelSerializer
-from utils.serializers import TranslatedField
+from utils.serializers import (ProjectModelSerializer, TranslatedField,
+                               FavoritesCreateSerializer)
 
 
 class ContactPhonesSerializer(serializers.ModelSerializer):
@@ -118,6 +117,18 @@ class EstablishmentTypeBaseSerializer(serializers.ModelSerializer):
             'use_subtypes': {'write_only': True},
         }
 
+class EstablishmentTypeGeoSerializer(EstablishmentTypeBaseSerializer):
+    """Serializer for EstablishmentType model w/ index_name."""
+
+    class Meta(EstablishmentTypeBaseSerializer.Meta):
+        fields = EstablishmentTypeBaseSerializer.Meta.fields + [
+            'index_name'
+        ]
+        extra_kwargs = {
+            **EstablishmentTypeBaseSerializer.Meta.extra_kwargs,
+            'index_name': {'read_only': True},
+        }
+
 
 class EstablishmentSubTypeBaseSerializer(serializers.ModelSerializer):
     """Serializer for EstablishmentSubType models."""
@@ -185,6 +196,19 @@ class EstablishmentBaseSerializer(ProjectModelSerializer):
         ]
 
 
+class EstablishmentGeoSerializer(EstablishmentBaseSerializer):
+    """Serializer for Geo view."""
+
+    type = EstablishmentTypeGeoSerializer(source='establishment_type', read_only=True)
+
+    class Meta(EstablishmentBaseSerializer.Meta):
+        """Meta class."""
+
+        fields = EstablishmentBaseSerializer.Meta.fields + [
+            'type'
+        ]
+
+
 class EstablishmentDetailSerializer(EstablishmentBaseSerializer):
     """Serializer for Establishment model."""
 
@@ -281,26 +305,13 @@ class EstablishmentCommentRUDSerializer(comment_serializers.CommentSerializer):
         ]
 
 
-class EstablishmentFavoritesCreateSerializer(serializers.ModelSerializer):
-    """Create comment serializer"""
-
-    class Meta:
-        """Serializer for model Comment"""
-        model = Favorites
-        fields = [
-            'id',
-            'created',
-        ]
-
-    def get_user(self):
-        """Get user from request"""
-        return self.context.get('request').user
+class EstablishmentFavoritesCreateSerializer(FavoritesCreateSerializer):
+    """Serializer to favorite object w/ model Establishment."""
 
     def validate(self, attrs):
-        """Override validate method"""
+        """Overridden validate method"""
         # Check establishment object
-        establishment_slug = self.context.get('request').parser_context.get('kwargs').get('slug')
-        establishment_qs = models.Establishment.objects.filter(slug=establishment_slug)
+        establishment_qs = models.Establishment.objects.filter(slug=self.slug)
 
         # Check establishment obj by slug from lookup_kwarg
         if not establishment_qs.exists():
@@ -309,18 +320,16 @@ class EstablishmentFavoritesCreateSerializer(serializers.ModelSerializer):
             establishment = establishment_qs.first()
 
         # Check existence in favorites
-        if self.get_user().favorites.by_content_type(app_label='establishment',
-                                                     model='establishment')\
-                                    .by_object_id(object_id=establishment.id).exists():
+        if establishment.favorites.filter(user=self.user).exists():
             raise utils_exceptions.FavoritesError()
 
         attrs['establishment'] = establishment
         return attrs
 
     def create(self, validated_data, *args, **kwargs):
-        """Override create method"""
+        """Overridden create method"""
         validated_data.update({
-            'user': self.get_user(),
+            'user': self.user,
             'content_object': validated_data.pop('establishment')
         })
         return super().create(validated_data)

apps/establishment/transfer.py

@@ -0,0 +1,119 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+"""
+
+card = {
+    "EstablishmentType": {
+        "data_type": "dictionaries",
+        "dependencies": None,
+        "fields": {
+            "Establishments": {
+                # значения для поля "name" берутся из поля "type"  legacy модели "establishments"
+                # https://jira.spider.ru/browse/GM-199
+                "name": ("name", "django.db.models.TJSONField")
+            }
+        }
+    },
+    "Establishment": {
+        "data_type": "dictionaries",
+        "dependencies": ("Address", "Collection", "EstablishmentType"),
+        # нету аналогов для establishment_type, establishment_subtypes, schedule, comments, tags
+        "fields": {
+            "Establishments": {
+                "name": "name",
+                "slug": "slug",
+                "is_publish": ("state", "django.db.models.BooleanField")
+            },
+            "relations": {
+                "EstablishmentInfos": {
+                    "key": "establishment",
+                    "fields": {
+                        "website": "website",
+                        "facebook": "facebook",
+                        "twitter": "twitter",
+                        "lafourchette": "lafourchette",
+                        "booking": "booking_url"
+                    }
+                },
+            },
+        },
+        # как работать с GenericRelation - ?
+        # как работать с ManyToManyField - ? "EstablishmentSubType", "schedule"
+        "relations": {
+            "Locations": [
+                (("location", None),
+                 ("Address", "address", None, None)),
+            ],
+            "Establishments": [  # TODO правильно ли заполнена связь с EstablishmentType - ?
+                (("type", "type"),
+                 ("EstablishmentType", "establishment_type", "id", "django.db.models.PositiveIntegerField"))
+            ]
+            # # "establishment_subtypes": "EstablishmentSubType",
+            # "collections": "Collection",
+            # # TODO: нашел schedules в legacy
+            # # "schedule": "Timetable",
+            # "award": "Award",
+            # # "tags": "MetaDataContent",
+            # "reviews": "Review",
+            # # "comments": "Comment",
+            # # "favorites": "Favorites" # В legacy  этой таблицы не было
+
+        }
+    },
+    "Menu": {
+        "data_type": "objects",
+        "dependencies": ("Establishment",),
+        "fields": {
+            "Menus": {
+                "category": ("name", "django.db.models.TJSONField")
+            }
+        },
+        "relations": {
+            "Establishments": [
+                (("establishment", None),
+                 ("Establishment", "establishment", None, None))
+            ]
+        }
+    },
+    "ContactPhone": {
+        "data_type": "objects",
+        "dependencies": ("Establishment",),
+        "fields": {
+            "Establishments": {
+                "phone": "phone"
+            },
+            "relations": {  # TODO правильно ли заполнена связь с Establishment - ?
+                "Establishments": [
+                    (("id", "id"),
+                     ("Establishment", "establishment", "id", "django.db.models.PositiveIntegerField"))
+                ]
+            }
+        }
+    },
+
+    "ContactEmail": {
+        "data_type": "objects",
+        "dependencies": ("Establishment",),
+        "fields": {
+            "EstablishmentInfos": {
+                "email": "email"
+            },
+        },
+        "relations": {
+            "Establishments": [
+                (("establishment", None),
+                 ("Establishment", "establishment", None, None))
+            ]
+        }
+    }
+}
+
+used_apps = ("review", "location", "collection", "main", "timetable", "favorites", "comment",)

apps/establishment/transfer_data.py

@@ -0,0 +1,138 @@
+from pprint import pprint
+
+from django.db.models import Q, F
+
+from transfer.models import Establishments, Dishes
+from transfer.serializers.establishment import EstablishmentSerializer
+from establishment.models import Establishment
+from location.models import Address
+from transfer.serializers.plate import PlateSerializer
+
+
+def transfer_establishment():
+    result = []
+
+    old_establishments = Establishments.objects.filter(
+        location__city__name__icontains='paris',
+    ).exclude(
+        Q(type='Wineyard') |
+        Q(location__timezone__isnull=True)
+    ).prefetch_related(
+        'establishmentinfos_set',
+        'schedules_set',
+        'descriptions_set',
+    )
+
+    for item in old_establishments:
+        data = {
+            'old_id': item.id,
+            'name': item.name,
+            'name_translated': item.index_name,
+            'slug': item.slug,
+            'type': item.type,
+            'phone': item.phone,
+            'created': item.created_at,
+            'state': item.state,
+            'description': {},
+            'website': None,
+            'facebook': None,
+            'twitter': None,
+            'lafourchette': None,
+            'booking': None,
+            'schedules': None,
+            'email': None,
+        }
+
+        if item.location:
+            data.update({
+                'location': item.location.id,
+                'tz': item.location.timezone,
+            })
+
+        # Инфо
+        info = item.establishmentinfos_set.first()
+        if info:
+            data.update({
+                'website': info.website,
+                'facebook': info.facebook,
+                'twitter': info.twitter,
+                'lafourchette': info.lafourchette,
+                'booking': info.booking_url,
+                'email': info.email,
+            })
+
+        # Время работы
+        schedule = item.schedules_set.first()
+        if schedule:
+            data.update({
+                'schedules': schedule.timetable,
+            })
+
+        # Описание
+        descriptions = item.descriptions_set.all()
+        for description in descriptions:
+            data['description'].update({
+                description.locale: description.text,
+            })
+
+        result.append(data)
+
+    serialized_data = EstablishmentSerializer(data=result, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Establishment serializer errors: {serialized_data.errors}")
+
+
+def transfer_menu():
+    dishes = Dishes.objects.exclude(
+        Q(establishment_id__isnull=True) |
+        Q(dish_type__isnull=True) |
+        Q(price__isnull=True) |
+        Q(currency__isnull=True) |
+        Q(name__isnull=True) |
+        Q(name__exact='')
+    ).annotate(
+        country_code=F('establishment__location__country_code'),
+    )
+
+    serialized_data = PlateSerializer(data=list(dishes.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Menu serializer errors: {serialized_data.errors}")
+
+        
+def transfer_establishment_addresses():
+    old_establishments = Establishments.objects.only("id", "location_id").exclude(
+        Q(type='Wineyard') |
+        Q(location__timezone__isnull=True)
+    )
+
+    for old_establishment in old_establishments:
+        try:
+            establishment = Establishment.objects.get(old_id=old_establishment.id)
+        except Establishment.DoesNotExist:
+            continue
+
+        try:
+            location = Address.objects.get(old_id=old_establishment.location_id)
+        except Address.MultipleObjectsReturned:
+            location = Address.objects.filter(old_id=old_establishment.location_id).first()
+        except Address.DoesNotExist:
+            continue
+
+        establishment.address = location
+        establishment.save()
+
+
+data_types = {
+    "establishment": [
+        transfer_establishment,
+    ],
+    "location_establishment": [
+        transfer_establishment_addresses
+    ],
+
+    "menu": [transfer_menu],
+}

apps/establishment/urls/common.py

@@ -9,7 +9,6 @@ urlpatterns = [
     path('', views.EstablishmentListView.as_view(), name='list'),
     path('recent-reviews/', views.EstablishmentRecentReviewListView.as_view(),
          name='recent-reviews'),
-    # path('wineries/', views.WineriesListView.as_view(), name='wineries-list'),
     path('slug/<slug:slug>/', views.EstablishmentRetrieveView.as_view(), name='detail'),
     path('slug/<slug:slug>/similar/', views.EstablishmentSimilarListView.as_view(), name='similar'),
     path('slug/<slug:slug>/comments/', views.EstablishmentCommentListView.as_view(), name='list-comments'),
@@ -18,5 +17,5 @@ urlpatterns = [
     path('slug/<slug:slug>/comments/<int:comment_id>/', views.EstablishmentCommentRUDView.as_view(),
          name='rud-comment'),
     path('slug/<slug:slug>/favorites/', views.EstablishmentFavoritesCreateDestroyView.as_view(),
-         name='add-to-favorites')
+         name='create-destroy-favorites')
 ]

apps/establishment/views/web.py

@@ -138,21 +138,18 @@ class EstablishmentFavoritesCreateDestroyView(generics.CreateAPIView, generics.D
         """
         Returns the object the view is displaying.
         """
-        establishment_obj = get_object_or_404(models.Establishment,
-                                              slug=self.kwargs['slug'])
-        obj = get_object_or_404(
-            self.request.user.favorites.by_content_type(app_label='establishment',
-                                                        model='establishment')
-                                       .by_object_id(object_id=establishment_obj.pk))
+        establishment = get_object_or_404(models.Establishment,
+                                          slug=self.kwargs['slug'])
+        favorites = get_object_or_404(establishment.favorites.filter(user=self.request.user))
         # May raise a permission denied
-        self.check_object_permissions(self.request, obj)
-        return obj
+        self.check_object_permissions(self.request, favorites)
+        return favorites
 
 
 class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIView):
     """Resource for getting list of nearest establishments."""
 
-    serializer_class = serializers.EstablishmentBaseSerializer
+    serializer_class = serializers.EstablishmentGeoSerializer
     filter_class = filters.EstablishmentFilter
 
     def get_queryset(self):
@@ -170,14 +167,3 @@ class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIVi
             return qs.by_distance_from_point(**{k: v for k, v in filter_kwargs.items()
                                                 if v is not None})
         return qs
-
-
-# Wineries
-# todo: find out about difference between subtypes data
-# class WineriesListView(EstablishmentListView):
-#     """Return list establishments with type Wineries"""
-#
-#     def get_queryset(self):
-#         """Overridden get_queryset method."""
-#         qs = super(WineriesListView, self).get_queryset()
-#         return qs.with_type_related().wineries()

apps/favorites/urls.py

@@ -8,4 +8,6 @@ app_name = 'favorites'
 urlpatterns = [
     path('establishments/', views.FavoritesEstablishmentListView.as_view(),
          name='establishment-list'),
+    path('products/', views.FavoritesProductListView.as_view(),
+         name='product-list'),
 ]

apps/favorites/views.py

@@ -1,7 +1,11 @@
 """Views for app favorites."""
 from rest_framework import generics
 from establishment.models import Establishment
+from establishment.filters import EstablishmentFilter
 from establishment.serializers import EstablishmentBaseSerializer
+from product.models import Product
+from product.serializers import ProductBaseSerializer
+from product.filters import ProductFilterSet
 from .models import Favorites
 
 
@@ -14,11 +18,24 @@ class FavoritesBaseView(generics.GenericAPIView):
 
 
 class FavoritesEstablishmentListView(generics.ListAPIView):
-    """List views for favorites"""
+    """List views for establishments in favorites."""
 
     serializer_class = EstablishmentBaseSerializer
+    filter_class = EstablishmentFilter
 
     def get_queryset(self):
         """Override get_queryset method"""
         return Establishment.objects.filter(favorites__user=self.request.user)\
                                     .order_by('-favorites')
+
+
+class FavoritesProductListView(generics.ListAPIView):
+    """List views for products in favorites."""
+
+    serializer_class = ProductBaseSerializer
+    filter_class = ProductFilterSet
+
+    def get_queryset(self):
+        """Override get_queryset method"""
+        return Product.objects.filter(favorites__user=self.request.user)\
+                              .order_by('-favorites')

apps/gallery/migrations/0002_auto_20191023_1207.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.4 on 2019-10-23 12:07
+
+from django.db import migrations
+import easy_thumbnails.fields
+import utils.methods
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=easy_thumbnails.fields.ThumbnailerImageField(max_length=255, upload_to=utils.methods.image_path, verbose_name='Image file'),
+        ),
+    ]

apps/gallery/migrations/0004_merge_20191025_0906.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:06
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0002_auto_20191023_1207'),
+        ('gallery', '0003_auto_20191003_1228'),
+    ]
+
+    operations = [
+    ]

apps/gallery/migrations/0005_auto_20191025_0914.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:14
+
+from django.db import migrations
+import sorl.thumbnail.fields
+import utils.methods
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0004_merge_20191025_0906'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'),
+        ),
+    ]

apps/gallery/migrations/0005_auto_20191027_0756.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.4 on 2019-10-27 07:56
+
+from django.db import migrations
+import sorl.thumbnail.fields
+import utils.methods
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0004_merge_20191025_0906'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'),
+        ),
+    ]

apps/gallery/migrations/0006_merge_20191027_1758.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-27 17:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0005_auto_20191027_0756'),
+        ('gallery', '0005_auto_20191025_0914'),
+    ]
+
+    operations = [
+    ]

apps/gallery/transfer.py

@@ -0,0 +1,26 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+"""
+
+card = {
+    "Image": {
+        "data_type": "objects",
+        "dependencies": None,
+        "fields": {
+            "MercuryImages": {
+                "image": "attachment_file_name"
+            }
+        }
+    },
+
+}
+
+used_apps = None

apps/gallery/transfer_data.py

@@ -0,0 +1,21 @@
+from django.db.models import Value, IntegerField, F
+from pprint import pprint
+from gallery.models import Image
+from transfer.models import MercuryImages
+from transfer.serializers.gallery import ImageSerializer
+
+
+def transfer_gallery():
+    queryset = MercuryImages.objects.all()
+
+    serialized_data = ImageSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "gallery": [transfer_gallery]
+}
+

apps/location/migrations/0013_country_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 06:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0012_data_migrate'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='country',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0014_remove_country_old_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.4 on 2019-10-27 06:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0013_country_old_id'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='country',
+            name='old_id',
+        ),
+    ]

apps/location/migrations/0015_country_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 06:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0014_remove_country_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='country',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0016_region_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 10:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0015_country_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='region',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0017_city_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-28 04:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0016_region_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='city',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0018_address_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-28 05:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0017_city_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='address',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0019_merge_20191030_0858.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 08:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0018_address_old_id'),
+        ('location', '0013_wineappellation_wineregion'),
+    ]
+
+    operations = [
+    ]

apps/location/migrations/0019_merge_20191030_1618.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 16:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0018_address_old_id'),
+        ('location', '0013_wineappellation_wineregion'),
+    ]
+
+    operations = [
+    ]

apps/location/migrations/0020_merge_20191030_1714.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 17:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0019_merge_20191030_0858'),
+        ('location', '0019_merge_20191030_1618'),
+    ]
+
+    operations = [
+    ]

apps/location/models.py

@@ -21,6 +21,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
     low_price = models.IntegerField(default=25, verbose_name=_('Low price'))
     high_price = models.IntegerField(default=50, verbose_name=_('High price'))
     languages = models.ManyToManyField(Language, verbose_name=_('Languages'))
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     @property
     def country_id(self):
@@ -43,6 +44,7 @@ class Region(models.Model):
         blank=True, default=None, on_delete=models.CASCADE)
     country = models.ForeignKey(
         Country, verbose_name=_('country'), on_delete=models.CASCADE)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     class Meta:
         """Meta class."""
@@ -76,7 +78,7 @@ class City(models.Model):
         _('postal code'), max_length=10, default='', help_text=_('Ex.: 350018'))
 
     is_island = models.BooleanField(_('is island'), default=False)
-
+    old_id = models.IntegerField(null=True, blank=True, default=None)
     objects = CityQuerySet.as_manager()
 
     class Meta:
@@ -100,6 +102,7 @@ class Address(models.Model):
         default='', help_text=_('Ex.: 350018'))
     coordinates = models.PointField(
         _('Coordinates'), blank=True, null=True, default=None)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     class Meta:
         """Meta class."""

apps/location/transfer.py

@@ -0,0 +1,141 @@
+"""
+
+Структура записи в card:
+Название таблицы в postgresql: {
+    "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)",
+    "dependencies": кортеж с зависимостями от других таблиц в postgresql,
+    "fields": список полей для таблицы postgresql, пример:
+    {
+        "название legacy таблицы": {
+            список полей для переноса, пример структуры описан далее
+        },
+        "relations": список зависимостей legacy-таблицы, пример:
+        {
+            # имеет внешний ключ на "название legacy таблицы" из  "fields"
+            "название legacy таблицы": {
+                "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля
+                в дочерней таблице и названия поля в родительской таблице в ином случае
+                "fields": {
+                    список полей для переноса, пример структуры описан далее
+                }
+            }
+        }
+    },
+    "relations": список внешних ключей таблицы postgresql, пример структуры описан далее
+    {
+        "Cities": [(
+            (None, "region_code"),
+            ("Region", "region", "code", "CharField")),
+            ((None, "country_code_2"),
+             ("Country", "country", "code", "CharField"))
+        ]
+    }
+},
+
+
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy)
+"""
+
+card = {
+    "Country": {
+        "data_type": "dictionaries",
+        "dependencies": None,
+        "fields": {
+            "Cities": {
+                "code": "country_code_2",
+            }
+        }
+    },
+
+    "Region": {
+        # subregion_code -> code
+        # region_code -> parent_region
+        # MySQL: select distinct(country_code_2) from cities where subregion_code is not NULL;
+        # только у пяти стран есть понятие "subregion_code"(не NULL): fr, be, ma, aa, gr (country_code_2)
+        # Возможно получиться обойтись без изменений модели Region
+        "data_type": "dictionaries",
+        "dependencies": ("Country", "Region"),
+        "fields": {
+            # нету аналога для поля name
+            "Cities": {
+                "code": "region_code",
+            },
+        },
+        "relations": {
+            "Cities": [(
+                (None, "country_code_2"),
+                ("Country", "country", "code", "django.db.models.CharField")),
+                # ((None, "subregion_code"), #TODO: как сопоставлять parent_region из postgres с subregion_code из legacy ?
+                #  ("Region", "parent_region", "code", "CharField"))
+
+            ]
+        }
+    },
+
+    "City": {
+        "data_type": "dictionaries",
+        "dependencies": ("Country", "Region"),
+        "fields": {
+            "Cities": {
+                "name": "name",
+                "code": "country_code_2",
+                "postal_code": "zip_code",
+                "is_island": ("is_island", "is_island", "django.db.models.Boolean")
+            }
+        },
+        "relations": {
+            "Cities": [(
+                (None, "region_code"),
+                ("Region", "region", "code", "django.db.models.CharField")),
+                ((None, "country_code_2"),
+                 ("Country", "country", "code", "django.db.models.CharField"))
+            ]
+        }
+    },
+
+
+    "Address": {
+        "data_type": "dictionaries",
+        "dependencies": ("City",),
+        "fields": {
+            # нету аналога для поля number
+            "Locations": {
+                "postal_code": "zip_code",
+                "coordinates": (("lat", "latitude"), ("long", "longitude"))
+            },
+        },
+        "relations": {
+            "Cities": [ #TODO: Locations ссылается внешним ключом на Cities
+                (("city", None),
+                 ("City", "city", None, None))
+            ]
+        }
+    }
+}
+
+
+used_apps = None

apps/location/transfer_data.py

@@ -0,0 +1,123 @@
+from django.db.models import Q, QuerySet
+
+from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer, AddressSerializer
+from transfer.models import Cities, Locations
+from pprint import pprint
+
+
+def transfer_countries():
+    queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 
+                                  FROM cities WHERE 
+                                  country_code_2 IS NOT NULL AND 
+                                  country_code_2 != ""
+                                  GROUP BY cities.country_code_2""")
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = CountrySerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Country serializer errors: {serialized_data.errors}")
+
+
+def transfer_regions():
+    regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code,
+                                                            cities.country_code_2, cities.subregion_code
+                                                            FROM cities WHERE
+                                                            (subregion_code IS NULL OR 
+                                                            subregion_code = "") AND
+                                                            region_code IS NOT NULL AND 
+                                                            region_code != "" AND
+                                                            country_code_2 IS NOT NULL AND 
+                                                            country_code_2 != ""
+                                                            GROUP BY region_code""")
+
+    regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]
+
+    serialized_without_subregion = RegionSerializer(data=regions_without_subregion_queryset, many=True)
+    if serialized_without_subregion.is_valid():
+        serialized_without_subregion.save()
+    else:
+        pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")
+
+    regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code,
+                                                        cities.country_code_2, cities.subregion_code
+                                                        FROM cities WHERE
+                                                        subregion_code IS NOT NULL AND
+                                                        subregion_code != "" AND
+                                                        region_code IS NOT NULL AND
+                                                        region_code != "" AND
+                                                        country_code_2 IS NOT NULL AND
+                                                        country_code_2 != ""
+                                                        AND cities.subregion_code in (
+                                                        SELECT region_code FROM cities WHERE
+                                                            (subregion_code IS NULL OR
+                                                            subregion_code = "") AND
+                                                            region_code IS NOT NULL AND
+                                                            region_code != "" AND
+                                                            country_code_2 IS NOT NULL AND
+                                                            country_code_2 != ""
+                                                        )
+                                                        GROUP BY region_code""")
+
+    regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]
+
+    serialized_with_subregion = RegionSerializer(data=regions_with_subregion_queryset, many=True)
+    if serialized_with_subregion.is_valid():
+        serialized_with_subregion.save()
+    else:
+        pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")
+
+
+def transfer_cities():
+    queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
+                                    cities.is_island, cities.region_code, cities.subregion_code
+                                    FROM cities WHERE
+                                    region_code IS NOT NULL AND 
+                                    region_code != "" AND
+                                    country_code_2 IS NOT NULL AND 
+                                    country_code_2 != ""
+                                        """)
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = CitySerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"City serializer errors: {serialized_data.errors}")
+
+
+def transfer_addresses():
+    queryset = Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, 
+                                      locations.latitude, locations.address, locations.city_id 
+                                      FROM locations WHERE 
+                                      locations.address != "" AND 
+                                      locations.address IS NOT NULL AND 
+                                      locations.city_id IS NOT NULL AND 
+                                      locations.city_id IN (SELECT cities.id
+                                        FROM cities WHERE
+                                        region_code IS NOT NULL AND 
+                                        region_code != "" AND
+                                        country_code_2 IS NOT NULL AND 
+                                        country_code_2 != "")""")
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = AddressSerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Address serializer errors: {serialized_data.errors}")
+
+
+
+data_types = {
+    "dictionaries": [
+        transfer_countries,
+        transfer_regions,
+        transfer_cities,
+        transfer_addresses
+    ]
+}

apps/main/migrations/0020_merge_20191025_0423.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 04:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0019_auto_20191022_1359'),
+        ('main', '0019_award_image_url'),
+    ]
+
+    operations = [
+    ]

apps/main/migrations/0023_merge_20191028_0725.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-28 07:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0020_merge_20191025_0423'),
+        ('main', '0022_auto_20191023_1113'),
+    ]
+
+    operations = [
+    ]

apps/main/transfer.py

@@ -0,0 +1,146 @@
+"""
+
+Структура записи в card:
+Название таблицы в postgresql: {
+    "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)",
+    "dependencies": кортеж с зависимостями от других таблиц в postgresql,
+    "fields": список полей для таблицы postgresql, пример:
+    {
+        "название legacy таблицы": {
+            список полей для переноса, пример структуры описан далее
+        },
+        "relations": список зависимостей legacy-таблицы, пример:
+        {
+            # имеет внешний ключ на "название legacy таблицы" из  "fields"
+            "название legacy таблицы": {
+                "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля
+                в дочерней таблице и названия поля в родительской таблице в ином случае
+                "fields": {
+                    список полей для переноса, пример структуры описан далее
+                }
+            }
+        }
+    },
+    "relations": список внешних ключей таблицы postgresql, пример структуры описан далее
+    {
+        "Cities": [(
+            (None, "region_code"),
+            ("Region", "region", "code", "CharField")),
+            ((None, "country_code_2"),
+             ("Country", "country", "code", "CharField"))
+        ]
+    }
+},
+
+
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+"""
+
+card = {
+    "SiteSettings": {
+        "data_type": "objects",
+        "dependencies": ("Country",),
+        "fields": {
+            "Sites": {
+                "subdomain": "country_code_2",
+                "pinterest_page_url": "pinterest_page_url",
+                "twitter_page_url": "twitter_page_url",
+                "facebook_page_url": "facebook_page_url",
+                "instagram_page_url": "instagram_page_url",
+                "contact_email": "contact_email",
+                "config": ("config", "django.db.models.JSONField") #TODO в качесте ключа использовать  country_code_2 из legacy - ?
+            },
+            # "relations": {
+            #     # Как работать c отношение OneToOneField
+            # }
+        }
+
+    },
+    "Feature": {
+        "data_type": "objects",
+        "dependencies": ("SiteSettings",),
+        "fields": {
+            "Features": {
+                "slug": "slug"
+            }
+        },
+        # поле "site_settings" ManyToManyField  имеет through='SiteFeature' в postgres
+        # "relations": { # как работать с ManyToManyField
+        #
+        # }
+    },
+    "SiteFeature": {
+        "data_type": "objects",
+        "dependencies": ("SiteSettings", "Feature"),
+        "fields": {
+            "SiteFeatures": {
+                "published": ("state", "django.db.models.BooleanField")
+            }
+        },
+        "relations": {
+            "Sites": [(
+                ("site", None),
+                ("SiteSettings", "site_settings", None, None))
+            ],
+            "Features": [(
+                ("feature", None),
+                ("Feature", "feature", None, None))
+            ]
+        }
+    },
+    "AwardType": {
+        "data_type": "objects",
+        "dependencies": ("Country",),
+        "fields": {
+            "AwardTypes": {
+                "name": "title"
+            }
+        },
+        #  вопрос с ForeignKey на Country
+        # "relations": {
+        #
+        # }
+    },
+    "Award": {
+        "data_type": "objects",
+        "dependencies": ("AwardType", "ContentType"),
+        "fields": {
+            "Awards": {
+                "title": ("title", "django.db.models.JSONField"),
+                "vintage_year": "year",
+            }
+        },
+        #TODO вопрос с content_type
+        "relations": {
+            "AwardTypes": [(
+                ("award_type", None),
+                ("AwardType", "award_type", None, None))
+            ]
+        }
+    }
+}
+
+used_apps = ("locations", )

apps/news/admin.py

@@ -3,6 +3,7 @@ from django.conf import settings
 
 from news import models
 from .tasks import send_email_with_news
+from utils.admin import BaseModelAdminMixin
 
 
 @admin.register(models.NewsType)
@@ -24,9 +25,11 @@ send_email_action.short_description = "Send the selected news by email"
 
 
 @admin.register(models.News)
-class NewsAdmin(admin.ModelAdmin):
+class NewsAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """News admin."""
+    raw_id_fields = ('address',)
     actions = [send_email_action]
+    raw_id_fields = ('news_type', 'address', 'country')
 
 
 @admin.register(models.NewsGallery)

apps/news/management/commands/rm_all_news.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+
+from news.models import News
+
+
+class Command(BaseCommand):
+    help = 'Remove all news from new bd'
+
+    def handle(self, *args, **kwargs):
+        old_news = News.objects.all()
+        count = old_news.count()
+        old_news.delete()
+        self.stdout.write(self.style.WARNING(f'Deleted {count} objects.'))

apps/news/management/commands/rm_old_news.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand
+
+from news.models import News
+
+
+class Command(BaseCommand):
+    help = 'Remove old news from new bd'
+
+    def handle(self, *args, **kwargs):
+        old_news = News.objects.exclude(old_id__isnull=True)
+        count = old_news.count()
+        old_news.delete()
+        self.stdout.write(self.style.WARNING(f'Deleted {count} objects.'))

apps/news/migrations/0021_auto_20191021_1120.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-21 11:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0020_remove_news_author'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='news',
+            name='slug',
+            field=models.SlugField(max_length=255, unique=True, verbose_name='News slug'),
+        ),
+    ]

apps/news/migrations/0023_merge_20191025_0423.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 04:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0021_auto_20191021_1120'),
+        ('news', '0022_auto_20191021_1306'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0029_merge_20191025_0906.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:06
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0028_auto_20191024_1649'),
+        ('news', '0021_auto_20191021_1120'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0030_merge_20191028_0725.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-28 07:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0029_merge_20191025_0906'),
+        ('news', '0023_merge_20191025_0423'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0030_news_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-29 08:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0029_merge_20191025_0906'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='news',
+            name='old_id',
+            field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='odl id'),
+        ),
+    ]

apps/news/migrations/0031_merge_20191029_0858.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-29 08:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0030_news_old_id'),
+        ('news', '0030_merge_20191028_0725'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0032_auto_20191030_1149.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-30 11:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0031_merge_20191029_0858'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='news',
+            name='old_id',
+            field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'),
+        ),
+    ]

apps/news/migrations/0032_merge_20191030_0858.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 08:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0029_auto_20191025_1241'),
+        ('news', '0031_merge_20191029_0858'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0033_merge_20191030_1618.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 16:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0032_auto_20191030_1149'),
+        ('news', '0029_auto_20191025_1241'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0034_merge_20191030_1714.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-30 17:14
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0033_merge_20191030_1618'),
+        ('news', '0032_merge_20191030_0858'),
+    ]
+
+    operations = [
+    ]

apps/news/models.py

@@ -1,11 +1,12 @@
 """News app models."""
-from django.db import models
 from django.contrib.contenttypes import fields as generic
+from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from rest_framework.reverse import reverse
-from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin
+
 from rating.models import Rating
+from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin
 
 
 class NewsType(models.Model):
@@ -125,6 +126,7 @@ class News(BaseAttributes, TranslatedFieldsMixin):
         (PUBLISHED_EXCLUSIVE, _('Published exclusive')),
     )
 
+    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
     news_type = models.ForeignKey(NewsType, on_delete=models.PROTECT,
                                   verbose_name=_('news type'))
     title = TJSONField(blank=True, null=True, default=None,
@@ -217,7 +219,6 @@ class NewsGalleryQuerySet(models.QuerySet):
 
 
 class NewsGallery(models.Model):
-
     news = models.ForeignKey(News, null=True,
                              related_name='news_gallery',
                              on_delete=models.CASCADE,

apps/news/transfer.py

@@ -0,0 +1,92 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy)
+"""
+
+# "from django.db import models"
+
+# "models.PositiveIntegerField"
+
+# "django.db.models."
+
+card = {
+    # нету аналога для NewsType
+    "NewsType": {
+        "data_type": "news",
+        "dependencies": None,
+        "fields": {
+            "Pages": {
+                # будет только один тип новости "News"
+                # значения для поля "name" берутся из поля "type"  legacy модели "Pages", притом type="News"
+                # Mysql - select distinct(type) from pages;
+                "name": "type"
+            }
+        }
+    },
+
+    "News": {
+        "data_type": "news",
+        # "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"),
+        "dependencies": ("NewsType", ),
+        "fields": {
+            # нету аналогов для start, end, playlist
+            "Pages": {
+                "state": ("state", "django.db.models.PositiveSmallIntegerField"),
+                "template": ("template", "django.db.models.PositiveIntegerField"),
+                "image_url": ("attachment_file_name", "django.db.models.URLField"),
+                "preview_image_url": ("attachment_file_name", "django.db.models.URLField"),
+
+                # в NewsOlds нету аналога для поля subtitle модели News, также нет аналогов для полей start, end, playlist
+                # "subtitle": ""
+
+
+                # Поле "description" модели News имеет тип JSONField(где ключ - это язык, а значение - новость
+                # на языке который указан ключом), а поле "body" NewsOlds имеет тип html-разметки
+                # с вставками шаблонизатора Ruby
+                # "description" : "body"
+            },
+            "relations": {
+                "PageTexts": {
+                    "key": "page",
+                    "fields": {
+                        "title": ("title", "django.db.models.JSONField"),
+                        "description": ("body", "django.db.models.JSONField"),
+                        "slug": ("slug", "django.db.models.SlugField")
+                    }
+                }
+            },
+        },
+        # нету аналога для поля tags
+        "relations": {
+            "Pages": [
+                ((None, "type"),
+                 ("NewsType", "news_type", "name", "django.db.models.CharField"))
+            ]
+        }
+    },
+}
+
+used_apps = ("location", "main", )

apps/news/transfer_data.py

@@ -0,0 +1,52 @@
+from pprint import pprint
+
+from django.db.models import Aggregate, CharField, Value
+from django.db.models import IntegerField, F
+
+from news.models import NewsType
+from tag.models import TagCategory
+from transfer.models import PageTexts
+from transfer.serializers.news import NewsSerializer
+
+
+class GroupConcat(Aggregate):
+    function = 'GROUP_CONCAT'
+    template = '%(function)s(%(expressions)s)'
+
+    def __init__(self, expression, **extra):
+        output_field = extra.pop('output_field', CharField())
+        super().__init__(expression, output_field=output_field, **extra)
+
+    def as_postgresql(self, compiler, connection):
+        self.function = 'STRING_AGG'
+        return super().as_sql(compiler, connection)
+
+
+def transfer_news():
+    news_type, _ = NewsType.objects.get_or_create(name='News')
+    tag_cat, _ = TagCategory.objects.get_or_create(index_name='tag', public=True)
+    news_type.tag_categories.add(tag_cat)
+    news_type.save()
+
+    queryset = PageTexts.objects.filter(
+        page__type='News',
+    ).annotate(
+        tag_cat_id=Value(tag_cat.id, output_field=IntegerField()),
+        news_type_id=Value(news_type.id, output_field=IntegerField()),
+        country_code=F('page__site__country_code_2'),
+        news_title=F('page__root_title'),
+        image=F('page__attachment_suffix_url'),
+        template=F('page__template'),
+        tags=GroupConcat('page__tags__id'),
+    )
+
+    serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f'News serializer errors: {serialized_data.errors}')
+
+
+data_types = {
+    'news': [transfer_news]
+}

apps/notification/transfer.py

@@ -0,0 +1,48 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy)
+"""
+
+card = {
+    # нету аналога для NewsType
+    "Subscriber": {
+        "data_type": "objects",
+        # "dependencies": ("User", ),
+        "dependencies": None,
+        "fields": {
+            "EmailAddresses": {
+                "email": "email",
+                "state": ("state", "django.db.models.PositiveIntegerField") # из legacy брать только те записи у которых state=usable
+            },
+            # "relations": [
+            #     # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных)
+            #     # "user": ""
+            # ]
+        }
+    },
+}
+
+used_apps = ("account", )

apps/notification/transfer_data.py

@@ -0,0 +1,21 @@
+from transfer.serializers.notification import SubscriberSerializer
+from notification.models import Subscriber
+from transfer.models import EmailAddresses
+from django.db.models import Value, IntegerField, F
+from pprint import pprint
+
+
+def transfer_subscriber():
+    queryset = EmailAddresses.objects.filter(state="usable")
+
+    serialized_data = SubscriberSerializer(data=list(queryset.values()), many=True)
+
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "subscriber": [transfer_subscriber]
+}

apps/partner/transfer.py

@@ -0,0 +1,41 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+"""
+
+card = {
+    "Partner": {
+        "data_type": "objects",
+        "dependencies": None,
+        "fields": {
+            # из EstablishmentBacklinks выбирать только те у которых  type="Partner"
+            "EstablishmentBacklinks": {
+                "url": "backlink_url",
+            },
+        }
+    },
+}
+
+used_apps = None

apps/partner/transfer_data.py

@@ -0,0 +1,19 @@
+from django.db.models import Value, IntegerField, F
+from pprint import pprint
+from transfer.models import EstablishmentBacklinks
+from transfer.serializers.partner import PartnerSerializer
+
+
+def transfer_partner():
+    queryset = EstablishmentBacklinks.objects.filter(type="Partner")
+
+    serialized_data = PartnerSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "partner": [transfer_partner]
+}

apps/product/admin.py

@@ -1,10 +1,11 @@
 """Product admin conf."""
 from django.contrib import admin
+from utils.admin import BaseModelAdminMixin
 from .models import Product, ProductType, ProductSubType
 
 
 @admin.register(Product)
-class ProductAdmin(admin.ModelAdmin):
+class ProductAdmin(BaseModelAdminMixin, admin.ModelAdmin):
     """Admin page for model Product."""
 
 

apps/product/filters.py

@@ -5,7 +5,7 @@ from django_filters import rest_framework as filters
 from product import models
 
 
-class ProductListFilterSet(filters.FilterSet):
+class ProductFilterSet(filters.FilterSet):
     """Product filter set."""
 
     establishment_id = filters.NumberFilter()

apps/product/management/commands/add_product.py

@@ -0,0 +1,54 @@
+from django.core.management.base import BaseCommand
+from product.models import ProductType, ProductSubType
+
+
+def add_type():
+    product_type = ProductType.objects.create(
+        name={'"en-GB"': "Wine"},
+        index_name=ProductType.WINE
+    )
+    return product_type.save()
+
+
+def add_subtype(id_type):
+    subtypes = ProductSubType.objects.bulk_create([
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.EXTRA_BRUT},
+                       index_name=ProductSubType.EXTRA_BRUT),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.BRUT},
+                       index_name=ProductSubType.BRUT),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.BRUT_NATURE},
+                       index_name=ProductSubType.BRUT_NATURE),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.DEMI_SEC},
+                       index_name=ProductSubType.DEMI_SEC),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.EXTRA_DRY},
+                       index_name=ProductSubType.EXTRA_DRY),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.DOSAGE_ZERO},
+                       index_name=ProductSubType.DOSAGE_ZERO),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.SEC},
+                       index_name=ProductSubType.SEC),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.SEC},
+                       index_name=ProductSubType.SEC),
+        ProductSubType(product_type=id_type,
+                       name={"en-GB", ProductSubType.MOELLEUX},
+                       index_name=ProductSubType.MOELLEUX),
+        ])
+
+
+class Command(BaseCommand):
+    help = 'Add product data'
+
+    def handle(self, *args, **kwarg):
+        product_type = add_type()
+        add_subtype(product_type.id)
+
+
+
+

apps/product/migrations/0002_product_slug.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-29 14:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('product', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='product',
+            name='slug',
+            field=models.SlugField(max_length=255, null=True, unique=True, verbose_name='Establishment slug'),
+        ),
+    ]

apps/product/models.py

@@ -1,5 +1,6 @@
 """Product app models."""
 from django.db import models
+from django.contrib.contenttypes import fields as generic
 from django.core.exceptions import ValidationError
 from django.contrib.postgres.fields import JSONField
 from django.utils.translation import gettext_lazy as _
@@ -45,10 +46,28 @@ class ProductSubType(TranslatedFieldsMixin, ProjectBaseMixin):
     # INDEX NAME CHOICES
     RUM = 'rum'
     OTHER = 'other'
+    EXTRA_BRUT = 'extra brut'
+    BRUT = 'brut'
+    BRUT_NATURE = 'brut nature'
+    DEMI_SEC = 'demi-sec'
+    EXTRA_DRY = 'Extra Dry'
+    DOSAGE_ZERO = 'dosage zero'
+    SEC = 'sec'
+    DOUX = 'doux'
+    MOELLEUX=  'moelleux'
 
     INDEX_NAME_TYPES = (
         (RUM, _('Rum')),
         (OTHER, _('Other')),
+        (EXTRA_BRUT, _('extra brut')),
+        (BRUT, _('brut')),
+        (BRUT_NATURE, _('brut nature')),
+        (DEMI_SEC, _('demi-sec')),
+        (EXTRA_DRY, _('Extra Dry')),
+        (DOSAGE_ZERO, _('dosage zero')),
+        (SEC, _('sec')),
+        (DOUX, _('doux')),
+        (MOELLEUX, _('moelleux'))
     )
 
     product_type = models.ForeignKey(ProductType, on_delete=models.CASCADE,
@@ -141,6 +160,9 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
     wine_appellation = models.ForeignKey('location.WineAppellation', on_delete=models.PROTECT,
                                          blank=True, null=True,
                                          verbose_name=_('wine appellation'))
+    slug = models.SlugField(unique=True, max_length=255, null=True,
+                            verbose_name=_('Establishment slug'))
+    favorites = generic.GenericRelation(to='favorites.Favorites')
 
     objects = ProductManager.from_queryset(ProductQuerySet)()
 

apps/product/serializers/common.py

@@ -1,9 +1,11 @@
 """Product app serializers."""
 from rest_framework import serializers
-from utils.serializers import TranslatedField
+from utils.serializers import TranslatedField, FavoritesCreateSerializer
 from product.models import Product, ProductSubType, ProductType
+from utils import exceptions as utils_exceptions
+from django.utils.translation import gettext_lazy as _
 from location.serializers import (WineRegionBaseSerializer, WineAppellationBaseSerializer,
-                                 CountrySimpleSerializer)
+                                  CountrySimpleSerializer)
 
 
 class ProductSubTypeBaseSerializer(serializers.ModelSerializer):
@@ -50,6 +52,7 @@ class ProductBaseSerializer(serializers.ModelSerializer):
         model = Product
         fields = [
             'id',
+            'slug',
             'name_translated',
             'category_display',
             'description_translated',
@@ -61,3 +64,33 @@ class ProductBaseSerializer(serializers.ModelSerializer):
             'wine_appellation',
             'available_countries',
         ]
+
+
+class ProductFavoritesCreateSerializer(FavoritesCreateSerializer):
+    """Serializer to create favorite object w/ model Product."""
+
+    def validate(self, attrs):
+        """Overridden validate method"""
+        # Check establishment object
+        product_qs = Product.objects.filter(slug=self.slug)
+
+        # Check establishment obj by slug from lookup_kwarg
+        if not product_qs.exists():
+            raise serializers.ValidationError({'detail': _('Object not found.')})
+        else:
+            product = product_qs.first()
+
+        # Check existence in favorites
+        if product.favorites.filter(user=self.user).exists():
+            raise utils_exceptions.FavoritesError()
+
+        attrs['product'] = product
+        return attrs
+
+    def create(self, validated_data, *args, **kwargs):
+        """Overridden create method"""
+        validated_data.update({
+            'user': self.user,
+            'content_object': validated_data.pop('product')
+        })
+        return super().create(validated_data)

apps/product/urls/common.py

@@ -6,5 +6,7 @@ from product import views
 app_name = 'product'
 
 urlpatterns = [
-    path('', views.ProductListView.as_view(), name='list')
+    path('', views.ProductListView.as_view(), name='list'),
+    path('slug/<slug:slug>/favorites/', views.CreateFavoriteProductView.as_view(),
+         name='create-destroy-favorites')
 ]

apps/product/views/common.py

@@ -1,5 +1,6 @@
 """Product app views."""
 from rest_framework import generics, permissions
+from django.shortcuts import get_object_or_404
 from product.models import Product
 from product import serializers
 from product import filters
@@ -17,4 +18,22 @@ class ProductListView(ProductBaseView, generics.ListAPIView):
     """List view for model Product."""
     permission_classes = (permissions.AllowAny, )
     serializer_class = serializers.ProductBaseSerializer
-    filter_class = filters.ProductListFilterSet
+    filter_class = filters.ProductFilterSet
+
+
+class CreateFavoriteProductView(generics.CreateAPIView,
+                                generics.DestroyAPIView):
+    """View for create/destroy product in favorites."""
+    serializer_class = serializers.ProductFavoritesCreateSerializer
+    lookup_field = 'slug'
+
+    def get_object(self):
+        """
+        Returns the object the view is displaying.
+        """
+        product = get_object_or_404(Product, slug=self.kwargs['slug'])
+        favorites = get_object_or_404(product.favorites.filter(user=self.request.user))
+
+        # May raise a permission denied
+        self.check_object_permissions(self.request, favorites)
+        return favorites

apps/recipe/models.py

@@ -15,14 +15,12 @@ class RecipeQuerySet(models.QuerySet):
 
     def annotate_in_favorites(self, user):
         """Annotate flag in_favorites"""
-        favorite_establishments = []
+        favorite_recipe_ids = []
         if user.is_authenticated:
-            favorite_establishments = user.favorites.by_content_type(app_label='recipe',
-                                                                     model='recipe') \
-                .values_list('object_id', flat=True)
+            favorite_recipe_ids = user.favorite_recipe_ids
         return self.annotate(in_favorites=models.Case(
             models.When(
-                id__in=favorite_establishments,
+                id__in=favorite_recipe_ids,
                 then=True),
             default=False,
             output_field=models.BooleanField(default=False)))

apps/recipe/transfer.py

@@ -0,0 +1,86 @@
+"""
+
+Структура записи в card:
+Название таблицы в postgresql: {
+    "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)",
+    "dependencies": кортеж с зависимостями от других таблиц в postgresql,
+    "fields": список полей для таблицы postgresql, пример:
+    {
+        "название legacy таблицы": {
+            список полей для переноса, пример структуры описан далее
+        },
+        "relations": список зависимостей legacy-таблицы, пример:
+        {
+            # имеет внешний ключ на "название legacy таблицы" из  "fields"
+            "название legacy таблицы": {
+                "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля
+                в дочерней таблице и названия поля в родительской таблице в ином случае
+                "fields": {
+                    список полей для переноса, пример структуры описан далее
+                }
+            }
+        }
+    },
+    "relations": список внешних ключей таблицы postgresql, пример структуры описан далее
+    {
+        "Cities": [(
+            (None, "region_code"),
+            ("Region", "region", "code", "CharField")),
+            ((None, "country_code_2"),
+             ("Country", "country", "code", "CharField"))
+        ]
+    }
+},
+
+
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy)
+"""
+
+# значения берутся из legacy модели "Pages", притом type="Recipe"
+card = {
+    "Recipe": {
+        "data_type": "objects",
+        "dependencies": None,
+        "fields": {
+            "Pages": {
+                "state": ("state", "django.db.models.PositiveSmallIntegerField")
+            },
+            "relations": {
+                "PageTexts": {
+                    "key": "page",
+                    "fields": {
+                        "title": ("title", "django.db.models.JSONField"),
+                        "description": ("body", "django.db.models.JSONField")
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+used_apps = None

apps/recipe/transfer_data.py

@@ -0,0 +1,19 @@
+from django.db.models import Value, IntegerField, F
+from pprint import pprint
+from transfer.models import PageTexts
+from transfer.serializers.recipe import RecipeSerializer
+
+
+def transfer_recipe():
+    queryset = PageTexts.objects.filter(page__type="Recipe")
+
+    serialized_data = RecipeSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "recipe": [transfer_recipe]
+}

apps/review/admin.py

@@ -1,8 +1,9 @@
 """Admin page for app Review"""
 from . import models
 from django.contrib import admin
+from utils.admin import BaseModelAdminMixin
 
 
 @admin.register(models.Review)
-class ReviewAdminModel(admin.ModelAdmin):
+class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin):
     """Admin model for model Review."""

apps/review/migrations/0005_review_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-31 06:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0004_review_country'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='review',
+            name='old_id',
+            field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'),
+        ),
+    ]

apps/review/models.py

@@ -68,6 +68,9 @@ class Review(BaseAttributes, TranslatedFieldsMixin):
     country = models.ForeignKey('location.Country', on_delete=models.CASCADE,
                                 related_name='country', verbose_name=_('Country'),
                                 null=True)
+
+    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
+
     objects = ReviewQuerySet.as_manager()
 
     class Meta:

apps/review/transfer.py

@@ -0,0 +1,64 @@
+"""
+Структура fields:
+key - поле в таблице postgres
+value - поле или группа полей в таблице legacy
+
+В случае передачи группы полей каждое поле представляет собой кортеж, где:
+field[0] - название аргумента
+field[1] - название поля в таблице legacy
+Опционально: field[2] - тип данных для преобразования
+
+Структура внешних ключей:
+"legacy_table" - спикок кортежей для сопоставления полей
+"legacy_table": [
+    (("legacy_key", "legacy_field"),
+    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
+], где:
+legacy_table - название модели legacy
+legacy_key - ForeignKey в legacy
+legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
+psql_table - название модели psql
+psql_key - ForeignKey в postgresql
+psql_field - уникальное поле в модели postgresql для сопоставления с legacy
+psql_field_type - тип уникального поля в postgresql
+
+
+"""
+
+card = {
+    # как работать с GenericForeignKey(content_type) - ?
+    # как работать с ForeignKey на самого себя(self), поле "child"
+    # вопрос с внешним ключом language на таблицу Language
+    "Review": {
+        "data_type": "objects",
+        "dependencies": ("User", "Language", "Review"),
+        "fields": {
+            "Reviews": {
+                "published_at": "published_at",
+                "vintage": "vintage",
+                "status": ("aasm_state", "django.db.models.PositiveSmallIntegerField")
+
+                # "content_object": ""
+            },
+            "relations": {
+                "ReviewTexts": {
+                    "key": "review",
+                    "fields": {
+                        # полу text в модели Review имеет тип TJSONField, а поле text в модели ReviewTexts имеет тип TextField
+                        # при их сопоставлении использовать поле locale модели ReviewTexts
+                        "text": ("text", "django.db.models.JSONField")
+                    }
+                }
+            },
+        },
+        "relations": {
+            "Accounts": [
+                (("account", None),
+                 ("User", "reviewer", None, None))
+            ]
+        }
+    }
+
+}
+
+used_apps = ("account", "translation")

apps/review/transfer_data.py

@@ -0,0 +1,101 @@
+from transfer.models import Reviews, ReviewTexts
+from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment
+from pprint import pprint
+import json
+
+
+def transfer_languages():
+    queryset = ReviewTexts.objects.raw("""SELECT id, locale 
+                                        FROM review_texts 
+                                        WHERE CHAR_LENGTH(locale) = 5
+                                        AND review_texts.locale IS NOT NULL GROUP BY locale""")
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = LanguageSerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Language serializer errors: {serialized_data.errors}")
+
+
+def transfer_reviews():
+    queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, 
+                                    reviews.reviewer_id, review_texts.text AS text, reviews.mark, reviews.published_at,
+                                    review_texts.created_at AS published, review_texts.locale AS locale, 
+                                    reviews.aasm_state
+                                    FROM reviews 
+                                    LEFT OUTER JOIN review_texts 
+                                    ON (reviews.id = review_texts.review_id) 
+                                    WHERE reviews.reviewer_id > 0 
+                                    AND reviews.reviewer_id IS NOT NULL 
+                                    AND review_texts.text IS NOT NULL
+                                    AND review_texts.locale IS NOT NULL
+                                    AND reviews.mark IS NOT NULL
+                                    AND reviews.reviewer_id IN (
+                                      SELECT accounts.id 
+                                      FROM accounts 
+                                      WHERE accounts.confirmed_at IS NOT NULL 
+                                      AND NOT accounts.email IN (
+                                        "cyril@tomatic.net", 
+                                        "cyril2@tomatic.net", 
+                                        "cyril2@tomatic.net", 
+                                        "d.sadykova@id-east.ru", 
+                                        "d.sadykova@octopod.ru", 
+                                        "n.yurchenko@id-east.ru"
+                                      ))
+                                    AND reviews.establishment_id IN (
+                                      SELECT establishments.id 
+                                      FROM establishments 
+                                      INNER JOIN locations 
+                                      ON (establishments.location_id = locations.id) 
+                                      INNER JOIN cities 
+                                      ON (locations.city_id = cities.id) 
+                                      WHERE UPPER(cities.name) LIKE UPPER("%%paris%%") 
+                                      AND NOT establishments.type = "Wineyard" 
+                                      AND establishments.type IS NOT NULL AND locations.timezone IS NOT NULL
+                                    )
+                                    ORDER BY review_texts.created_at DESC
+                                  """)
+
+    queryset_result = []
+    establishments_mark_list = {}
+
+    for query in queryset:
+        query = vars(query)
+        if query['establishment_id'] not in establishments_mark_list.keys():
+            if "aasm_state" in query and query['aasm_state'] is not None and query['aasm_state'] == "published":
+                establishments_mark_list[query['establishment_id']] = [
+                    int(query['mark']),
+                    json.dumps({query['locale']: query['text']})
+                ]
+            else:
+                establishments_mark_list[query['establishment_id']] = int(query['mark'])
+        del(query['mark'])
+        queryset_result.append(query)
+
+    serialized_data = ReviewSerializer(data=queryset_result, many=True)
+
+    if serialized_data.is_valid():
+        serialized_data.save()
+
+        for establishment_id, mark in establishments_mark_list.items():
+            try:
+                establishment = Establishment.objects.get(old_id=establishment_id)
+            except Establishment.DoesNotExist:
+                continue
+            if isinstance(mark, list):
+                mark, review_text = mark
+
+            establishment.public_mark = mark
+            establishment.save()
+    else:
+        pprint(serialized_data.errors)
+
+
+data_types = {
+    "overlook": [
+        transfer_languages,
+        transfer_reviews
+    ]
+}

apps/search_indexes/documents/establishment.py

@@ -10,16 +10,6 @@ EstablishmentIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__,
 EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1)
 
 
-# todo: check & refactor
-class ObjectField(fields.ObjectField):
-
-    def get_value_from_instance(self, *args, **kwargs):
-        value = super(ObjectField, self).get_value_from_instance(*args, **kwargs)
-        if value == {}:
-            return None
-        return value
-
-
 @EstablishmentIndex.doc_type
 class EstablishmentDocument(Document):
     """Establishment document."""
@@ -47,7 +37,7 @@ class EstablishmentDocument(Document):
     works_noon = fields.ListField(fields.IntegerField(
         attr='works_noon'
     ))
-    works_now = fields.BooleanField(attr='works_now')
+    # works_now = fields.BooleanField(attr='works_now')
     tags = fields.ObjectField(
         properties={
             'id': fields.IntegerField(attr='id'),
@@ -63,7 +53,7 @@ class EstablishmentDocument(Document):
             'closed_at': fields.KeywordField(attr='closed_at_str'),
         }
     ))
-    address = ObjectField(
+    address = fields.ObjectField(
         properties={
             'id': fields.IntegerField(),
             'street_name_1': fields.TextField(
@@ -93,13 +83,6 @@ class EstablishmentDocument(Document):
             ),
         },
     )
-    # todo: need to fix
-    # collections = fields.ObjectField(
-    #     properties={
-    #         'id': fields.IntegerField(attr='collection.id'),
-    #         'collection_type': fields.IntegerField(attr='collection.collection_type'),
-    #     },
-    #     multi=True)
 
     class Django:
 
@@ -108,6 +91,7 @@ class EstablishmentDocument(Document):
             'id',
             'name',
             'name_translated',
+            'is_publish',
             'price_level',
             'toque_number',
             'public_mark',
@@ -115,4 +99,4 @@ class EstablishmentDocument(Document):
         )
 
     def get_queryset(self):
-        return super().get_queryset().published().with_es_related()
+        return super().get_queryset().with_es_related()

apps/search_indexes/serializers.py

@@ -1,5 +1,6 @@
 """Search indexes serializers."""
 from rest_framework import serializers
+from elasticsearch_dsl import AttrDict
 from django_elasticsearch_dsl_drf.serializers import DocumentSerializer
 from news.serializers import NewsTypeSerializer
 from search_indexes.documents import EstablishmentDocument, NewsDocument
@@ -13,6 +14,8 @@ class TagsDocumentSerializer(serializers.Serializer):
     label_translated = serializers.SerializerMethodField()
 
     def get_label_translated(self, obj):
+        if isinstance(obj, dict):
+            return get_translated_value(obj.get('label'))
         return get_translated_value(obj.label)
 
 
@@ -29,6 +32,13 @@ class AddressDocumentSerializer(serializers.Serializer):
     geo_lon = serializers.FloatField(allow_null=True, source='coordinates.lon')
     geo_lat = serializers.FloatField(allow_null=True, source='coordinates.lat')
 
+    # todo: refator
+    def to_representation(self, instance):
+        if instance != AttrDict(d={}) or \
+                (isinstance(instance, dict) and len(instance) != 0):
+            return super().to_representation(instance)
+        return None
+
 
 class ScheduleDocumentSerializer(serializers.Serializer):
     """Schedule serializer for ES Document"""
@@ -97,7 +107,7 @@ class EstablishmentDocumentSerializer(DocumentSerializer):
             'schedule',
             'works_noon',
             'works_evening',
-            'works_now',
+            # 'works_now',
             # 'collections',
             # 'establishment_type',
             # 'establishment_subtypes',

apps/search_indexes/views.py

@@ -3,12 +3,13 @@ from rest_framework import permissions
 from django_elasticsearch_dsl_drf import constants
 from django_elasticsearch_dsl_drf.filter_backends import (
     FilteringFilterBackend,
-    GeoSpatialFilteringFilterBackend
+    GeoSpatialFilteringFilterBackend,
+    DefaultOrderingFilterBackend,
 )
 from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet
 from search_indexes import serializers, filters
 from search_indexes.documents import EstablishmentDocument, NewsDocument
-from utils.pagination import ProjectPageNumberPagination
+from utils.pagination import ProjectMobilePagination
 
 
 class NewsDocumentViewSet(BaseDocumentViewSet):
@@ -16,7 +17,7 @@ class NewsDocumentViewSet(BaseDocumentViewSet):
 
     document = NewsDocument
     lookup_field = 'slug'
-    pagination_class = ProjectPageNumberPagination
+    pagination_class = ProjectMobilePagination
     permission_classes = (permissions.AllowAny,)
     serializer_class = serializers.NewsDocumentSerializer
     ordering = ('id',)
@@ -53,15 +54,20 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet):
 
     document = EstablishmentDocument
     lookup_field = 'slug'
-    pagination_class = ProjectPageNumberPagination
+    pagination_class = ProjectMobilePagination
     permission_classes = (permissions.AllowAny,)
     serializer_class = serializers.EstablishmentDocumentSerializer
-    ordering = ('id',)
+
+    def get_queryset(self):
+        qs = super(EstablishmentDocumentViewSet, self).get_queryset()
+        qs = qs.filter('match', is_publish=True)
+        return qs
 
     filter_backends = [
         FilteringFilterBackend,
         filters.CustomSearchFilterBackend,
         GeoSpatialFilteringFilterBackend,
+        DefaultOrderingFilterBackend,
     ]
 
     search_fields = {
@@ -72,6 +78,7 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet):
     translated_search_fields = (
         'description',
     )
+    ordering = 'id'
     filter_fields = {
         'slug': 'slug',
         'tag': {
@@ -145,12 +152,12 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet):
                 constants.LOOKUP_QUERY_IN,
             ],
         },
-        'works_now': {
-            'field': 'works_now',
-            'lookups': [
-                constants.LOOKUP_FILTER_TERM,
-            ]
-        },
+        # 'works_now': {
+        #     'field': 'works_now',
+        #     'lookups': [
+        #         constants.LOOKUP_FILTER_TERM,
+        #     ]
+        # },
     }
 
     geo_spatial_filter_fields = {

apps/tag/management/commands/add_tags.py

@@ -0,0 +1,69 @@
+from django.core.management.base import BaseCommand
+
+from establishment.models import Establishment, EstablishmentType
+from transfer import models as legacy
+from tag.models import Tag, TagCategory
+
+
+class Command(BaseCommand):
+    help = 'Add tags values from old db to new db'
+
+    def handle(self, *args, **kwargs):
+
+        existing_establishment = Establishment.objects.filter(old_id__isnull=False)
+        ESTABLISHMENT = 1
+        SHOP = 2
+        RESTAURANT = 3
+        WINEYARD = 4
+
+        MAPPER = {
+            RESTAURANT: EstablishmentType.RESTAURANT,
+            WINEYARD: EstablishmentType.PRODUCER,
+        }
+
+        mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all()
+        for key, value in MAPPER.items():
+            values_meta_id_list = mapper_values_meta.filter(
+                key_value_metadatum_establishment_id=key
+            ).values_list('key_value_metadatum_id')
+
+            est_type, _ = EstablishmentType.objects.get_or_create(index_name=value)
+
+            key_value_metadata = legacy.KeyValueMetadata.objects.filter(
+                id__in=values_meta_id_list)
+
+            # create TagCategory
+            for key_value in key_value_metadata:
+                tag_category, created = TagCategory.objects.get_or_create(
+                    index_name=key_value.key_name,
+                )
+
+                if created:
+                    tag_category.label = {
+                                             'en-GB': key_value.key_name,
+                                             'fr-FR': key_value.key_name,
+                                             'ru-RU': key_value.key_name,
+                                         },
+                    tag_category.value_type = key_value.value_type
+                    tag_category.save()
+                    est_type.tag_categories.add(
+                        tag_category
+                    )
+
+                # create Tag
+                for tag in key_value.metadata_set.filter(
+                        establishment__id__in=list(existing_establishment.values_list('old_id', flat=True))):
+
+                    new_tag, _ = Tag.objects.get_or_create(
+                        label={
+                               'en-GB': tag.value,
+                               'fr-FR': tag.value,
+                               'ru-RU': tag.value,
+                               },
+                        value=tag.value,
+                        category=tag_category,
+                    )
+                    est = existing_establishment.get(old_id=tag.establishment_id)
+                    est.tags.add(new_tag)
+                    est.save()
+