Merge branch 'develop' into feature/guides

# Conflicts:
#	apps/collection/models.py

apps/account/migrations/0026_auto_20191210_1553.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.7 on 2019-12-10 15:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('account', '0025_auto_20191210_0623'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='role',
+            name='role',
+            field=models.PositiveIntegerField(choices=[(1, 'Standard user'), (2, 'Comments moderator'), (3, 'Country admin'), (4, 'Content page manager'), (5, 'Establishment manager'), (6, 'Reviewer manager'), (7, 'Restaurant reviewer'), (8, 'Sales man'), (9, 'Winery reviewer'), (10, 'Seller'), (11, 'Liquor reviewer'), (12, 'Product reviewer')], verbose_name='Role'),
+        ),
+    ]

apps/account/migrations/0028_merge_20191217_1127.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-17 11:27
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('account', '0027_auto_20191211_1444'),
+        ('account', '0026_auto_20191210_1553'),
+    ]
+
+    operations = [
+    ]

apps/account/models.py

@@ -36,6 +36,8 @@ class Role(ProjectBaseMixin):
     SALES_MAN = 8
     WINERY_REVIEWER = 9  # Establishments subtype "winery"
     SELLER = 10
+    LIQUOR_REVIEWER = 11
+    PRODUCT_REVIEWER = 12
 
     ROLE_CHOICES = (
         (STANDARD_USER, _('Standard user')),
@@ -47,7 +49,9 @@ class Role(ProjectBaseMixin):
         (RESTAURANT_REVIEWER, 'Restaurant reviewer'),
         (SALES_MAN, 'Sales man'),
         (WINERY_REVIEWER, 'Winery reviewer'),
-        (SELLER, 'Seller')
+        (SELLER, 'Seller'),
+        (LIQUOR_REVIEWER, 'Liquor reviewer'),
+        (PRODUCT_REVIEWER, 'Product reviewer'),
     )
     role = models.PositiveIntegerField(verbose_name=_('Role'), choices=ROLE_CHOICES,
                                        null=False, blank=False)

apps/collection/migrations/0027_auto_20191217_1852.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.7 on 2019-12-17 18:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('collection', '0026_merge_20191217_1151'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collection',
+            name='slug',
+            field=models.SlugField(max_length=255, null=True, unique=True, verbose_name='Collection slug'),
+        ),
+    ]

apps/collection/migrations/0028_merge_20191223_1415.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-23 14:15
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('collection', '0027_auto_20191217_1852'),
+        ('collection', '0027_auto_20191218_0753'),
+    ]
+
+    operations = [
+    ]

apps/collection/models.py

@@ -6,12 +6,12 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 from mptt.models import MPTTModel, TreeForeignKey
-from django.conf import settings
+from slugify import slugify
-from collection import tasks
 
 from location.models import Country, Region, WineRegion, WineSubRegion
 from review.models import Review
 from translation.models import Language
+from utils.models import IntermediateGalleryModelMixin
 from utils.models import (
     ProjectBaseMixin, TJSONField, TranslatedFieldsMixin,
     URLImageMixin, IntermediateGalleryModelMixin
@@ -84,7 +84,7 @@ class Collection(ProjectBaseMixin, CollectionDateMixin,
     description = TJSONField(
         _('description'), null=True, blank=True,
         default=None, help_text='{"en-GB":"some text"}')
-    slug = models.SlugField(max_length=50, unique=True,
+    slug = models.SlugField(max_length=255, unique=True,
                             verbose_name=_('Collection slug'), editable=True, null=True)
     old_id = models.IntegerField(null=True, blank=True)
 
@@ -146,6 +146,15 @@ class Collection(ProjectBaseMixin, CollectionDateMixin,
 
         return related_objects
 
+    def save(self, *args, **kwargs):
+        if not self.pk:
+            slugify_slug = slugify(
+                next(iter(self.name.values())),
+                word_boundary=True
+            )
+            self.slug = slugify_slug
+        super(Collection, self).save(*args, **kwargs)
+
 
 class GuideTypeQuerySet(models.QuerySet):
     """QuerySet for model GuideType."""

apps/collection/serializers/back.py

@@ -43,6 +43,9 @@ class CollectionBackOfficeSerializer(CollectionBaseSerializer):
             'related_object_names',
             'rank',
         ]
+        extra_kwargs = {
+            'slug': {'read_only': True},
+        }
 
 
 class CollectionBindObjectSerializer(serializers.Serializer):

apps/favorites/tests.py

@@ -39,7 +39,6 @@ class BaseTestCase(APITestCase):
             title={"en-GB": "Test news"},
             news_type=self.test_news_type,
             description={"en-GB": "Description test news"},
-            start=datetime.fromisoformat("2020-12-03 12:00:00"),
             end=datetime.fromisoformat("2020-12-03 12:00:00"),
             state=News.PUBLISHED,
             slugs={'en-GB': 'test-news'}

apps/gallery/migrations/0007_auto_20191110_1329.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.4 on 2019-11-10 13:29
+
+from django.db import migrations
+import sorl.thumbnail.fields
+import utils.methods
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0006_merge_20191027_1758'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='image',
+            name='image',
+            field=sorl.thumbnail.fields.ImageField(max_length=255, upload_to=utils.methods.image_path, verbose_name='image file'),
+        ),
+    ]

apps/gallery/migrations/0008_merge_20191212_0752.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-12 07:52
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0007_auto_20191110_1329'),
+        ('gallery', '0007_auto_20191211_1528'),
+    ]
+
+    operations = [
+    ]

apps/gallery/models.py

@@ -21,7 +21,7 @@ class Image(ProjectBaseMixin, SORLImageMixin, PlatformMixin):
         (VERTICAL, _('Vertical')),
     )
 
-    image = SORLImageField(upload_to=image_path,
+    image = SORLImageField(max_length=255, upload_to=image_path,
                            verbose_name=_('image file'))
     orientation = models.PositiveSmallIntegerField(choices=ORIENTATIONS,
                                                    blank=True, null=True, default=None,

apps/gallery/serializers.py

@@ -1,11 +1,10 @@
-from django.conf import settings
 from django.core.validators import MinValueValidator, MaxValueValidator
 from django.core.files.base import ContentFile
 from rest_framework import serializers
 from sorl.thumbnail import get_thumbnail
 from sorl.thumbnail.parsers import parse_crop, ThumbnailParseError
 from django.utils.translation import gettext_lazy as _
-
+from django.conf import settings
 from . import models
 
 
@@ -36,6 +35,15 @@ class ImageSerializer(serializers.ModelSerializer):
             'orientation': {'write_only': True}
         }
 
+    def validate(self, attrs):
+        """Overridden validate method."""
+        image = attrs.get('image')
+
+        if image and image.size >= settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
+            raise serializers.ValidationError({'detail': _('File size too large: %s bytes') % image.size})
+
+        return attrs
+
 
 class CropImageSerializer(ImageSerializer):
     """Serializers for image crops."""

apps/location/csv/aa_cities.csv

@@ -0,0 +1,122 @@
+old_id,name,zip_code,country,,region,subregion,is_island
+9854,Sainte Marie,97230,Antilles Guyane West Indies,11,null,null,1
+9846,La Guyane,,Antilles Guyane West Indies,,null,null,1
+9845,Macouba,97218,Antilles Guyane West Indies,,Martinique,Nord,0
+9844,La Martinique,,Antilles Guyane West Indies,,null,null,1
+9843,Guadeloupe,,Antilles Guyane West Indies,,Guadeloupe,null,1
+9693,Gustavia,97133,Antilles Guyane West Indies,,St Barthelemy,null,0
+9663,Saint Martin,97150,Antilles Guyane West Indies,,St Martin,null,0
+9662,Saint-Barthélemy,97133,Antilles Guyane West Indies,,St Barthelemy,null,0
+9661,Javouhey,97318,Antilles Guyane West Indies,,Guyane,null,0
+9660,Kaw,97353,Antilles Guyane West Indies,,Guyane,null,0
+9659,Apatou,97317,Antilles Guyane West Indies,,Guyane,null,0
+9658,La comte,97352,Antilles Guyane West Indies,,Guyane,null,0
+9657,St elie,97312,Antilles Guyane West Indies,,Guyane,null,0
+9656,Grand santi,97340,Antilles Guyane West Indies,,Guyane,null,0
+9655,Camopi,97330,Antilles Guyane West Indies,,Guyane,null,0
+9654,Maripasoula,97370,Antilles Guyane West Indies,,Guyane,null,0
+9653,Saul,97314,Antilles Guyane West Indies,,Guyane,null,0
+9652,Ouanary,97380,Antilles Guyane West Indies,,Guyane,null,0
+9651,Montsinery tonnegrande,97300,Antilles Guyane West Indies,,Guyane,null,0
+9650,Sinnamary,97315,Antilles Guyane West Indies,,Guyane,null,0
+9649,St Laurent du Maroni,97320,Antilles Guyane West Indies,,Guyane,null,0
+9648,Roura,97311,Antilles Guyane West Indies,,Guyane,null,0
+9647,Rémiré Montjoly,97354,Antilles Guyane West Indies,,Guyane,null,0
+9646,St georges,97313,Antilles Guyane West Indies,,Guyane,null,0
+9645,Matoury,97351,Antilles Guyane West Indies,,Guyane,null,0
+9644,Mana,97360,Antilles Guyane West Indies,,Guyane,null,0
+9643,Macouria,97355,Antilles Guyane West Indies,,Guyane,null,0
+9642,Kourou,97310,Antilles Guyane West Indies,,Guyane,null,0
+9641,Iracoubo,97350,Antilles Guyane West Indies,,Guyane,null,0
+9640,Cayenne,97300,Antilles Guyane West Indies,,Guyane,null,0
+9639,Regina,97390,Antilles Guyane West Indies,,Guyane,null,0
+9638,Vert Pré,97231,Antilles Guyane West Indies,,Martinique,Nord,0
+9637,Vauclin,97280,Antilles Guyane West Indies,,Martinique,Sud,0
+9636,Trois Ilets,97229,Antilles Guyane West Indies,,Martinique,Sud,0
+9635,La Trinité,97220,Antilles Guyane West Indies,,Martinique,Nord,0
+9634,Schoelcher,97233,Antilles Guyane West Indies,,Martinique,Nord,0
+9633,Saint-Pierre,97250,Antilles Guyane West Indies,,Martinique,Nord,0
+9632,Sainte Marie,97230,Antilles Guyane West Indies,,Martinique,Nord,0
+9631,Sainte-Luce,97228,Antilles Guyane West Indies,,Martinique,Sud,0
+9630,Saint Joseph,97212,Antilles Guyane West Indies,,Martinique,Nord,0
+9629,Saint Esprit,97270,Antilles Guyane West Indies,,Martinique,Sud,0
+9628,Sainte-Anne,97227,Antilles Guyane West Indies,,Martinique,Sud,0
+9627,Le Robert,97231,Antilles Guyane West Indies,,Martinique,Nord,0
+9626,Rivière Salée,97215,Antilles Guyane West Indies,,Martinique,Sud,0
+9625,Rivière Pilote,97211,Antilles Guyane West Indies,,Martinique,Nord,0
+9624,Prêcheur,97250,Antilles Guyane West Indies,,Martinique,Nord,0
+9623,Morne Vert,97226,Antilles Guyane West Indies,,Martinique,Nord,0
+9622,Morne Rouge,97260,Antilles Guyane West Indies,,Martinique,Nord,0
+9621,Le Marin,97290,Antilles Guyane West Indies,,Martinique,Sud,0
+9620,Marigot,97225,Antilles Guyane West Indies,,Martinique,Nord,0
+9619,Lorrain,97214,Antilles Guyane West Indies,,Martinique,Nord,0
+9618,Lamentin,97232,Antilles Guyane West Indies,,Martinique,Sud,0
+9617,Gros Morne,97213,Antilles Guyane West Indies,,Martinique,Nord,0
+9616,Grand Rivière,97218,Antilles Guyane West Indies,,Martinique,Nord,0
+9615,Le François,97240,Antilles Guyane West Indies,,Martinique,Sud,0
+9614,Fort de France,97200,Antilles Guyane West Indies,,Martinique,Nord,0
+9613,Fonds Saint Denis,97250,Antilles Guyane West Indies,,Martinique,Nord,0
+9612,Ducos,97224,Antilles Guyane West Indies,,Martinique,Sud,0
+9611,Le Diamant,97223,Antilles Guyane West Indies,,Martinique,Sud,0
+9610,Case Pilote,97222,Antilles Guyane West Indies,,Martinique,Nord,0
+9609,Le Carbet,97221,Antilles Guyane West Indies,,Martinique,Nord,0
+9608,Bellefontaine,97222,Antilles Guyane West Indies,,Martinique,Nord,0
+9607,Basse Pointe,97218,Antilles Guyane West Indies,,Martinique,Nord,0
+9606,Anses d'Arlet,97217,Antilles Guyane West Indies,,Martinique,Sud,0
+9605,Ajoupa Bouillon,97216,Antilles Guyane West Indies,,Martinique,Nord,0
+9604,Les mangles,97131,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9603,Douville,97180,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9602,Sainte-Marie,97230,Antilles Guyane West Indies,,Martinique,Nord,0
+9601,Pigeon,97132,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9600,Bananier,97130,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9599,Vieux-Habitants,97119,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9598,Vieux fort,97141,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9597,Trois rivieres,97114,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9596,Terre de Haut,97137,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+9595,Terre de bas,97136,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+9594,Sainte-Rose,97115,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9593,Sainte-Anne,97180,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9592,St louis,97134,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+9591,Saint-François,97118,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9590,St claude,97120,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9589,Port louis,97117,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9588,Pointe noire,97116,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9587,Pointe-A-Pitre,97110,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9586,Petit canal,97131,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9585,Petit bourg,97170,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9584,Le moule,97160,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9583,Morne a l eau,97111,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9582,Lamentin,97129,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9581,Goyave,97128,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9580,Le Gosier,97190,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9579,Grand-Bourg,97112,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+9578,Deshaies,97126,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9577,La desirade,97127,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+9576,Gourbeyre,97113,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9575,Capesterre de Marie-Galante,97140,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+9574,Capesterre belle eau,97130,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9573,Bouillante,97125,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9572,Basse terre,97100,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9571,Baillif,97123,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9570,Baie-Mahault,97122,Antilles Guyane West Indies,,Guadeloupe,Basse Terre,0
+9569,Anse-Bertrand,97121,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+9568,Les abymes,97139,Antilles Guyane West Indies,,Guadeloupe,Grande Terre,0
+11371,Cul-de-Sac,97150,Antilles Guyane West Indies,,St Martin,null,0
+11370,Ilet Pinel,97150,Antilles Guyane West Indies,,St Martin,null,0
+11369,Simpson Bay,97150,Antilles Guyane West Indies,,St Martin,null,0
+11368,Anse Marcel,97150,Antilles Guyane West Indies,,St Martin,null,0
+11367,Orient Bay,97150,Antilles Guyane West Indies,,St Martin,null,0
+11366,Marigot,97150,Antilles Guyane West Indies,,St Martin,null,0
+11365,Baie Longue,97150,Antilles Guyane West Indies,,St Martin,null,0
+11364,Baie Nettle,97150,Antilles Guyane West Indies,,St Martin,null,0
+11363,Baie Orientale,97150,Antilles Guyane West Indies,,St Martin,null,0
+11362,Oyster Pond,97150,Antilles Guyane West Indies,,St Martin,null,0
+11349,Rodney Bay,LC01 401,Antilles Guyane West Indies,,Sainte-Lucie,null,0
+11348,Good Lands,null,Antilles Guyane West Indies,,Sainte-Lucie,null,0
+11347,Soufriere,null,Antilles Guyane West Indies,,Sainte-Lucie,null,0
+11343,Gros Islet,LC0001,Antilles Guyane West Indies,,Sainte-Lucie,null,0
+11342,CAP ESTATE,null,Antilles Guyane West Indies,,Sainte-Lucie,null,0
+11341,Castries,null,Antilles Guyane West Indies,,Sainte-Lucie,null,0
+11303,Saint- Louis,97134,Antilles Guyane West Indies,,Guadeloupe,Autres les Îles de Guadeloupe,0
+11302,Grand Cul de Sac,97133,Antilles Guyane West Indies,,St Barthelemy,null,0
+11301,Grand Case,97150,Antilles Guyane West Indies,,St Martin,null,0

apps/location/migrations/0031_auto_20191121_1236.py

@@ -0,0 +1,98 @@
+# Generated by Django 2.2.7 on 2019-11-21 12:36
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+import django.db.models.deletion
+import utils.models
+
+
+def copy_country_code(apps, schema_editor):
+    Country = apps.get_model('location', 'Country')
+    to_update = []
+    for country in Country.objects.all():
+        if country.code:
+            country.code_2 = country.code
+            to_update.append(country)
+    Country.objects.bulk_update(to_update, ['code_2', ])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0030_auto_20191120_1010'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='city',
+            name='map1',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='map2',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='map_ref',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='mysql_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='name_translated',
+            field=utils.models.TJSONField(blank=True, default=None, help_text='{"en-GB":"some text"}', null=True, verbose_name='Translated name'),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='situation',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='country',
+            name='mysql_ids',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='region',
+            name='mysql_ids',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), blank=True, null=True, size=None),
+        ),
+        migrations.AlterField(
+            model_name='city',
+            name='gallery',
+            field=models.ManyToManyField(blank=True, through='location.CityGallery', to='gallery.Image'),
+        ),
+        migrations.AlterField(
+            model_name='city',
+            name='region',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='location.Region', verbose_name='parent region'),
+        ),
+        # -- fix incorrect removal of uniqueness constraint code field
+        migrations.AddField(
+            model_name='country',
+            name='code_2',
+            field=models.CharField(max_length=255, null=True)
+        ),
+        migrations.RunPython(copy_country_code, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name='country',
+            name='code',
+        ),
+        migrations.RenameField(
+            model_name='country',
+            old_name='code_2',
+            new_name='code',
+        ),
+        migrations.AlterField(
+            model_name='country',
+            name='code',
+            field=models.CharField(max_length=255, verbose_name='Code'),
+        ),
+        # fix incorrect removal of uniqueness constraint code field --
+    ]

apps/location/migrations/0032_merge_20191209_0832.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-09 08:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0031_establishmentwineoriginaddress_wineoriginaddress'),
+        ('location', '0031_auto_20191121_1236'),
+    ]
+
+    operations = [
+    ]

apps/location/migrations/0033_merge_20191224_0920.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-24 09:20
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0032_merge_20191209_0832'),
+        ('location', '0032_auto_20191220_1019'),
+    ]
+
+    operations = [
+    ]

apps/location/models.py

@@ -9,6 +9,8 @@ from functools import reduce
 from typing import List
 
 
+from django.contrib.postgres.fields import ArrayField
+
 from translation.models import Language
 from utils.models import (ProjectBaseMixin, SVGImageMixin, TJSONField,
                           TranslatedFieldsMixin, get_current_locale,
@@ -22,7 +24,8 @@ class CountryQuerySet(models.QuerySet):
         return self.filter(is_active=switcher)
 
 
-class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
+class Country(TranslatedFieldsMixin,
+              SVGImageMixin, ProjectBaseMixin):
     """Country model."""
 
     STR_FIELD_NAME = 'name'
@@ -36,25 +39,17 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
 
     name = TJSONField(null=True, blank=True, default=None,
                       verbose_name=_('Name'), help_text='{"en-GB":"some text"}')
-    code = models.CharField(max_length=255, unique=True, verbose_name=_('Code'))
+    code = models.CharField(max_length=255, verbose_name=_('Code'))
     low_price = models.IntegerField(default=25, verbose_name=_('Low price'))
     high_price = models.IntegerField(default=50, verbose_name=_('High price'))
     languages = models.ManyToManyField(Language, verbose_name=_('Languages'))
     is_active = models.BooleanField(_('is active'), default=True)
     old_id = models.IntegerField(null=True, blank=True, default=None)
 
+    mysql_ids = ArrayField(models.IntegerField(), blank=True, null=True)
+
     objects = CountryQuerySet.as_manager()
 
-    @property
-    def time_format(self):
-        if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES:
-            return 'HH:mm'
-        return 'hh:mmA'
-
-    @property
-    def country_id(self):
-        return self.id
-
     class Meta:
         """Meta class."""
 
@@ -69,6 +64,17 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
                 str_name = translated_name
         return str_name
 
+    @property
+    def time_format(self):
+        if self.code.lower() not in self.TWELVE_HOURS_FORMAT_COUNTRIES:
+            return 'HH:mm'
+        return 'hh:mmA'
+
+    @property
+    def country_id(self):
+        return self.id
+
+
 
 class RegionQuerySet(models.QuerySet):
     """QuerySet for model Region."""
@@ -101,6 +107,7 @@ class Region(models.Model):
     country = models.ForeignKey(
         Country, verbose_name=_('country'), on_delete=models.CASCADE)
     old_id = models.IntegerField(null=True, blank=True, default=None)
+    mysql_ids = ArrayField(models.IntegerField(), blank=True, null=True)
 
     objects = RegionQuerySet.as_manager()
 
@@ -137,9 +144,13 @@ class CityQuerySet(models.QuerySet):
 class City(GalleryMixin, models.Model):
     """Region model."""
     name = models.CharField(_('name'), max_length=250)
+    name_translated = TJSONField(blank=True, null=True, default=None,
+                                 verbose_name=_('Translated name'),
+                                 help_text='{"en-GB":"some text"}')
     code = models.CharField(_('code'), max_length=250)
-    region = models.ForeignKey(
+    region = models.ForeignKey(Region, on_delete=models.CASCADE,
-        Region, verbose_name=_('parent region'), on_delete=models.CASCADE)
+                               blank=True, null=True,
+                               verbose_name=_('parent region'))
     country = models.ForeignKey(
         Country, verbose_name=_('country'), on_delete=models.CASCADE)
 
@@ -148,7 +159,15 @@ class City(GalleryMixin, models.Model):
 
     is_island = models.BooleanField(_('is island'), default=False)
     old_id = models.IntegerField(null=True, blank=True, default=None)
-    gallery = models.ManyToManyField('gallery.Image', through='CityGallery')
+
+    map1 = models.CharField(max_length=255, blank=True, null=True)
+    map2 = models.CharField(max_length=255, blank=True, null=True)
+    map_ref = models.CharField(max_length=255, blank=True, null=True)
+    situation = models.CharField(max_length=255, blank=True, null=True)
+
+    gallery = models.ManyToManyField('gallery.Image', through='location.CityGallery', blank=True)
+
+    mysql_id = models.IntegerField(blank=True, null=True, default=None)
 
     objects = CityQuerySet.as_manager()
 

apps/location/transfer_data.py

@@ -1,9 +1,23 @@
-from transfer.serializers import location as location_serializers
+import csv
-from transfer import models as transfer_models
+import json
-from location.models import Country, CityGallery, City
-from gallery.models import Image
 from pprint import pprint
+
+from django.conf import settings
+from django.db.transaction import atomic
 from requests import get
+from tqdm import tqdm
+
+from account.models import Role
+from collection.models import Collection
+from gallery.models import Image
+from location.models import Country, Region, City, Address, CityGallery
+from main.models import AwardType
+from news.models import News
+from review.models import Review
+from tag.models import TagCategory, ChosenTagSettings
+from transfer import models as transfer_models
+from transfer.serializers import location as location_serializers
+from transfer.utils import clean_old_records, clean_old_country_records, clean_old_region_records
 
 
 def transfer_countries():
@@ -111,19 +125,12 @@ def transfer_cities():
         pprint(f"City serializer errors: {serialized_data.errors}")
 
 
+@atomic
 def transfer_addresses():
     queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, 
                                       locations.latitude, locations.address, locations.city_id 
                                       FROM locations WHERE 
-                                      locations.address != "" AND 
+                                      locations.city_id IS NOT NULL""")
-                                      locations.address IS NOT NULL AND 
-                                      locations.city_id IS NOT NULL AND 
-                                      locations.city_id IN (SELECT cities.id
-                                        FROM cities WHERE
-                                        region_code IS NOT NULL AND 
-                                        region_code != "" AND
-                                        country_code_2 IS NOT NULL AND 
-                                        country_code_2 != "")""")
 
     queryset = [vars(query) for query in queryset]
 
@@ -136,18 +143,18 @@ def transfer_addresses():
 
 def transfer_wine_region():
     queryset = transfer_models.WineLocations.objects.filter(type='WineRegion')
-    serialized_data = location_serializers.WineRegion(
+    serialized_data = location_serializers.WineRegionSerializer(
       data=list(queryset.values()),
       many=True)
     if serialized_data.is_valid():
         serialized_data.save()
     else:
-        pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
+        pprint(f"WineRegionSerializer errors: {serialized_data.errors}")
 
 
 def transfer_wine_sub_region():
     queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion')
-    serialized_data = location_serializers.WineSubRegion(
+    serialized_data = location_serializers.WineSubRegionSerializer(
       data=list(queryset.values()),
       many=True)
     if serialized_data.is_valid():
@@ -158,7 +165,7 @@ def transfer_wine_sub_region():
 
 def transfer_wine_village():
     queryset = transfer_models.WineLocations.objects.filter(type='Village')
-    serialized_data = location_serializers.WineVillage(
+    serialized_data = location_serializers.WineVillageSerializer(
       data=list(queryset.values()),
       many=True)
     if serialized_data.is_valid():
@@ -179,6 +186,374 @@ def update_flags():
             query.save()
 
 
+def update_fake_country_flag():
+    link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media/svg/country/10-31-2019/aa.svg"
+    resp = get(link_to_request)
+    if resp.status_code == 200:
+        country = Country.objects.get(code="aa")
+        country.svg_image = link_to_request
+        country.save()
+
+
+def migrate_city_map_situation(get_exists_cities=False):
+    if get_exists_cities:
+        ids = City.objects.values_list('mysql_id', flat=True)
+        queryset = transfer_models.Cities.objects.filter(id__in=list(ids))
+        queryset = list(queryset.values())
+    else:
+        queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.map1, cities.map2, cities.map_ref, cities.situation
+                                            FROM cities WHERE
+                                            region_code IS NOT NULL AND 
+                                            region_code != "" AND
+                                            country_code_2 IS NOT NULL AND 
+                                            country_code_2 != ""
+                                                """)
+
+        queryset = [vars(query) for query in queryset]
+
+    serialized_data = location_serializers.CityMapCorrectSerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"City info serializer errors: {serialized_data.errors}")
+
+
+def migrate_city_photos():
+    queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name 
+                                            FROM city_photos WHERE
+                                            city_photos.attachment_file_name IS NOT NULL AND
+                                            city_id IN(
+                                              SELECT cities.id
+                                                FROM cities WHERE
+                                                region_code IS NOT NULL AND 
+                                                region_code != "" AND
+                                                country_code_2 IS NOT NULL AND 
+                                                country_code_2 != ""
+                                            )
+                                            """)
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Address serializer errors: {serialized_data.errors}")
+
+
+# Update location models with ruby library
+# Utils functions defined before transfer functions
+def get_ruby_socket(params):
+    url = 'http://172.21.0.1:5678'  # docker host
+    response = get(url, params=params)
+    try:
+        data = json.loads(response.text)
+    except Exception as e:
+        print(f"{response.text} error: {e}")
+        return None
+    return data
+
+
+# Get data from ruby and save it to file
+# Save errors from ruby to another file
+def get_ruby_data():
+    cities = transfer_models.Cities.objects.all()
+    ruby_data = {}
+    error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w")
+
+    for mysql_city in cities:
+        # try:
+        #     mysql_city = transfer_models.Cities.objects.get(id=city.old_id)
+        # except transfer_models.Cities.DoesNotExist:
+        #     print(f"City with id {city.old_id} not found")
+        #     continue
+
+        ruby_params = {}
+
+        if mysql_city.country_code is not None:
+            ruby_params['country_code'] = mysql_city.country_code
+
+        if mysql_city.country_code_2 is not None:
+            ruby_params['country_code_2'] = mysql_city.country_code_2
+
+        if mysql_city.region_code is not None:
+            ruby_params['region_code'] = mysql_city.region_code
+
+        if mysql_city.subregion_code is not None:
+            ruby_params['subregion_code'] = mysql_city.subregion_code
+
+        ruby_response = get_ruby_socket(ruby_params)
+
+        if ruby_response is None:
+            continue
+
+        if "country" not in ruby_response.keys():
+            error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
+            continue
+
+        if mysql_city.country_code_2 is None and mysql_city.country_code is None:
+            error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
+            continue
+
+        city_name = mysql_city.name.strip()
+        ruby_data[mysql_city.id] = {
+            "mysql_id": mysql_city.id,
+            "name": city_name.strip(),
+            "name_translated": {'en-GB': city_name.strip()},
+
+            "map1": mysql_city.map1,
+            "map2": mysql_city.map2,
+            "map_ref": mysql_city.map_ref,
+            "situation": mysql_city.situation,
+
+            "is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False,
+
+            "postal_code": mysql_city.zip_code.strip() if mysql_city.zip_code is not None else '',
+            "code": ruby_response['country']['code'].lower().strip()
+
+        }
+
+        for ruby_data_key in ruby_response.keys():
+            if "error" in ruby_response[ruby_data_key]:
+                error_file.write(json.dumps(ruby_response) + '\n')
+            else:
+                ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key]
+
+    error_file.close()
+
+    with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file:
+        ruby_data_file.write(json.dumps(ruby_data))
+
+    return ruby_data
+
+
+def get_unused_data():
+    ruby_data = {}
+    error_file = open(f"{settings.PROJECT_ROOT}/ruby_unused_error.txt", "w")
+
+    countries = Country.objects.all()
+
+    for country in countries:
+
+        ruby_params = {}
+
+        ruby_response = get_ruby_socket({"country_regions": country.code})
+
+        if ruby_response is None:
+            continue
+
+        if "error" in ruby_response.keys():
+            error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
+            continue
+
+        ruby_data[country.code] = ruby_response
+    error_file.close()
+
+    with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_unused_data.py", "w") as ruby_data_file:
+        ruby_data_file.write(json.dumps(ruby_data))
+
+
+# Add correct objects of Country, Region and City with mysql_ids array (Country, Region) and mysql_id (City)
+def add_correct_location_models(ruby_data):
+    for mysql_id, city_object in tqdm(ruby_data.items()):
+        country_data = city_object["country"]
+        try:
+            country = Country.objects.get(code=country_data['code'], mysql_ids__isnull=False)
+            country.mysql_ids.append(mysql_id)
+            country.save()
+        except Country.DoesNotExist:
+            country_data['mysql_ids'] = [mysql_id]
+            country = Country.objects.create(**country_data)
+
+        city_object['country'] = country
+
+        if "region" in city_object:
+            region_data = city_object['region']
+            region_data['country'] = country
+            try:
+                region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False)
+                region.mysql_ids.append(mysql_id)
+                region.save()
+            except Region.DoesNotExist:
+                region_data['mysql_ids'] = [mysql_id]
+                region = Region.objects.create(**region_data)
+
+            if "subregion" in city_object:
+                subregion_data = city_object.pop('subregion')
+                subregion_data['country'] = country
+                try:
+                    subregion = Region.objects.get(code=subregion_data['code'],
+                                                   mysql_ids__isnull=False)
+                    subregion.mysql_ids.append(mysql_id)
+                    subregion.save()
+                except Region.DoesNotExist:
+                    subregion_data['parent_region'] = region
+                    subregion_data['mysql_ids'] = [mysql_id]
+                    subregion = Region.objects.create(**subregion_data)
+
+                city_object['region'] = subregion
+            else:
+                city_object['region'] = region
+
+        if "subregion" in city_object:
+            del(city_object['subregion'])
+
+        try:
+            City.objects.create(**city_object)
+        except Exception as e:
+            print(city_object)
+            print(e)
+            break
+
+
+def fix_location_address():
+    addresses = Address.objects.filter(old_id__isnull=False)
+    for address in addresses:
+        mysql_location = transfer_models.Locations.objects.get(id=address.old_id)
+        try:
+            correct_city = City.objects.get(mysql_id=mysql_location.city_id)
+        except City.DoesNotExist:
+            continue
+
+        address.city = correct_city
+        address.save()
+
+
+def fix_location_collection():
+    collections = Collection.objects.filter(old_id__isnull=False)
+    for collection in collections:
+        try:
+            mysql_collection = transfer_models.Collections.objects.\
+                raw(f"""select  
+                s.country_code_2, 
+                c.id 
+                from collections as c
+                join sites s on s.id = c.site_id where c.id={collection.old_id}""")[0]
+        except:
+            continue
+
+        try:
+            correct_country = Country.objects.get(code=mysql_collection.country_code_2, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        collection.country = correct_country
+        collection.save()
+
+
+def fix_award_type():
+    award_types = AwardType.objects.filter(old_id__isnull=False)
+    for award_type in award_types:
+        mysql_award_type = transfer_models.AwardTypes.objects.\
+            raw(f"""SELECT at.id, s.country_code_2 AS country_code
+                            FROM award_types as at
+                            JOIN sites s on s.id = at.site_id
+                            WHERE at.id={award_type.old_id}""")[0]
+
+        try:
+            correct_country = Country.objects.get(code=mysql_award_type.country_code, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        award_type.country = correct_country
+        award_type.save()
+
+
+def fix_role():
+    roles_list = Role.objects.filter(country__isnull=False)
+    for role in roles_list:
+        try:
+            correct_country = Country.objects.get(code=role.country.code, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        role.country = correct_country
+        role.save()
+
+
+def fix_news():
+    news = News.objects.filter(country__isnull=False)
+    for news_item in news:
+        try:
+            correct_country = Country.objects.get(code=news_item.country.code, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        news.country = correct_country
+        news_item.save()
+
+
+def fix_reviews():
+    reviews = Review.objects.filter(country__isnull=False)
+    for review in reviews:
+        try:
+            correct_country = Country.objects.get(code=review.country.code, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        review.country = correct_country
+        review.save()
+
+
+def fix_tag_category():
+    tags_categories = TagCategory.objects.filter(country__isnull=False)
+    for tag_category in tags_categories:
+        try:
+            correct_country = Country.objects.get(code=tag_category.country.code, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        tag_category.country = correct_country
+        tag_category.save()
+
+
+def fix_chosen_tag():
+    chosen_tags = ChosenTagSettings.objects.filter(country__isnull=False)
+    for chosen_tag in chosen_tags:
+        try:
+            correct_country = Country.objects.get(code=chosen_tag.country.code, mysql_ids__isnull=False)
+        except Country.DoesNotExist:
+            continue
+
+        chosen_tag.country = correct_country
+        chosen_tag.save()
+
+
+def fix_location_models():
+
+    try:
+        ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
+        ruby_data = json.loads(ruby_data_file.read())
+    except FileNotFoundError:
+        print('create ruby data')
+        ruby_data = get_ruby_data()
+
+    print('add_correct_location_models')
+    add_correct_location_models(ruby_data)
+    # print('fix_location_address')
+    # fix_location_address()
+    # print('fix_location_collection')
+    # fix_location_collection()
+    # print('fix_award_type')
+    # fix_award_type()
+    # print('fix_role')
+    # fix_role()
+    # print('fix_news')
+    # fix_news()
+    # print('fix_reviews')
+    # fix_reviews()
+    # print('fix_tag_category')
+    # fix_tag_category()
+    # print('fix_chosen_tag')
+    # fix_chosen_tag()
+
+
+def remove_old_records():
+    clean_old_records(City, {"mysql_id__isnull": True})
+    clean_old_country_records(Country, {"mysql_ids__isnull": True})
+    clean_old_region_records(Region, {"mysql_ids__isnull": True})
+
+
 def transfer_city_gallery():
     created_counter = 0
     cities_not_exists = {}
@@ -215,11 +590,218 @@ def transfer_city_gallery():
           f'Already added: {gallery_obj_exists_counter}')
 
 
+@atomic
+def add_fake_country():
+    # add country
+    country_data = {
+        "name": '{"en-GB": "Antilles Guyane West Indies"}',
+        "code": "aa",
+        "svg_image": "svg/country/11-02-2019/658714.svg",
+        "mysql_ids": []
+    }
+
+    country, _ = Country.objects.get_or_create(**country_data)
+
+    # add regions and subregions
+    regions_data = [
+        {
+            "name": "Guadeloupe",
+            "code": "gp",
+            "subregions": [
+                {
+                    "name": "Basse Terre",
+                    "code": "bat",
+                },
+                {
+                    "name": "Grande Terre",
+                    "code": "grt",
+                },
+                {
+                    "name": "Autres les Îles de Guadeloupe",
+                    "code": "aug",
+                },
+            ]
+        },
+        {
+            "name": "Martinique",
+            "code": "mq",
+            "subregions": [
+                {
+                    "name": "Nord",
+                    "code": "nor",
+                },
+                {
+                    "name": "Sud",
+                    "code": "sud",
+                },
+            ]
+        },
+        {
+            "name": "Guyane",
+            "code": "gy",
+        },
+        {
+            "name": "St Barthelemy",
+            "code": "bl",
+        },
+        {
+            "name": "St Martin",
+            "code": "mf",
+        },
+        {
+            "name": "Sainte-Lucie",
+            "code": "lc",
+        },
+    ]
+    regions = {}
+
+    for region_data in regions_data:
+        if "subregions" in region_data:
+            subregions = region_data['subregions']
+            del(region_data['subregions'])
+        else:
+            subregions = False
+
+        region_name = region_data['name']
+        region_data['name'] = '{"en-GB": "' + region_name + '"}'
+        region_data['country'] = country
+
+        region_data['mysql_ids'] = []
+
+        regions[region_name] = Region.objects.create(**region_data)
+
+        if subregions:
+            for subregion_data in subregions:
+                subregion_name = subregion_data['name']
+                subregion_data['name'] = '{"en-GB": "' + subregion_name + '"}'
+                subregion_data['country'] = country
+                subregion_data['parent_region'] = regions[region_name]
+                subregion_data['mysql_ids'] = []
+
+                regions[subregion_name] = Region.objects.create(**subregion_data)
+
+    # add cities
+    file = open(f"{settings.PROJECT_ROOT}/apps/location/csv/aa_cities.csv")
+    reader = csv.DictReader(file, delimiter=',')
+    for city_data in reader:
+        del(city_data[''])
+
+        city_data['mysql_id'] = city_data['old_id']
+        del(city_data['old_id'])
+
+        city_data['postal_code'] = city_data['zip_code']
+        del(city_data['zip_code'])
+
+        if city_data['postal_code'] == 'null' or city_data['postal_code'] == '':
+            del(city_data['postal_code'])
+
+        city_data["country"] = country
+
+        if city_data['subregion'] != 'null':
+            region = regions[city_data['subregion']]
+        elif city_data['region'] != 'null':
+            region = regions[city_data['region']]
+        else:
+            del(city_data['region'])
+            region = None
+
+        del(city_data['subregion'])
+
+        city_data['region'] = region
+
+        city_data['name_translated'] = '{"en-GB": "' + city_data['name'] + '"}'
+
+        city_data['is_island'] = True if int(city_data['is_island']) > 0 else False
+
+        if region is not None:
+            region.mysql_ids.append(city_data['mysql_id'])
+
+        country.mysql_ids.append(city_data['mysql_id'])
+
+        try:
+            mysql_data = transfer_models.Cities.objects.\
+                only("map1", "map2", "map_ref", "situation").get(id=city_data['mysql_id'])
+            city_data['map1'] = mysql_data.map1
+            city_data['map2'] = mysql_data.map2
+            city_data['map_ref'] = mysql_data.map_ref
+            city_data['situation'] = mysql_data.situation
+
+        except transfer_models.Cities.DoesNotExist:
+            pass
+
+        City.objects.create(**city_data)
+
+    country.save()
+
+    for region_name, region in regions.items():
+        if len(region.mysql_ids) > 0:
+            region.save()
+
+
+@atomic
+def setup_clean_db():
+    try:
+        ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
+        ruby_data = json.loads(ruby_data_file.read())
+    except FileNotFoundError:
+        ruby_data = get_ruby_data()
+
+    print('add_correct_location_models')
+    add_correct_location_models(ruby_data)
+
+    print('add_fake_country')
+    add_fake_country()
+
+    print('migrate_city_map_situation')
+    migrate_city_map_situation(True)
+
+    print('update_flags')
+    update_flags()
+
+    print('transfer_city_gallery')
+    transfer_city_gallery()
+
+
+def set_unused_regions():
+    ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_unused_data.py", "r")
+    ruby_data = json.loads(ruby_data_file.read())
+
+    for country_code, regions in ruby_data.items():
+        try:
+            country = Country.objects.get(code=country_code)
+        except Country.DoesNotExist:
+            print(f"Country with code {country_code} does not exists")
+            continue
+
+        for region_code, region_obj in regions.items():
+            try:
+                region = Region.objects.get(code=region_code, country=country)
+            except Region.DoesNotExist:
+                region = Region.objects.create(
+                    name=region_obj['name'],
+                    code=region_code,
+                    country=country
+                )
+
+            if "subregions" in region_obj:
+                for subregion_code, subregion in region_obj['subregions'].items():
+                    try:
+                        subregion = Region.objects.get(code=subregion, country=country)
+                    except Region.DoesNotExist:
+                        subregion = Region.objects.create(
+                            name=subregion,
+                            code=subregion_code,
+                            country=country,
+                            parent_region=region
+                        )
+
+
+
 data_types = {
     "dictionaries": [
-        transfer_countries,
+        # transfer_countries,
-        transfer_regions,
+        # transfer_regions,
-        transfer_cities,
+        # transfer_cities,
         transfer_addresses,
         transfer_wine_region,
         transfer_wine_sub_region,
@@ -228,5 +810,27 @@ data_types = {
     "update_country_flag": [
         update_flags
     ],
-    "fill_city_gallery": [transfer_city_gallery]
+    "update_city_info": [
+        migrate_city_map_situation
+    ],
+    "migrate_city_gallery": [
+        migrate_city_photos
+    ],
+    "fix_location": [
+        add_fake_country,
+        fix_location_models,
+    ],
+    "remove_old_locations": [
+        remove_old_records
+    ],
+    "fill_city_gallery": [
+        transfer_city_gallery
+    ],
+    "add_fake_country": [
+        add_fake_country,
+    ],
+
+    "setup_clean_db": [setup_clean_db],
+    "set_unused_regions": [set_unused_regions],
+    "update_fake_country_flag": [update_fake_country_flag]
 }

apps/main/admin.py

@@ -56,10 +56,17 @@ class PageAdmin(admin.ModelAdmin):
     date_hierarchy = 'created'
 
 
+class FooterLinkInline(admin.TabularInline):
+    model = models.Footer.links.through
+    extra = 1
+
+
 @admin.register(models.Footer)
 class FooterAdmin(admin.ModelAdmin):
     """Footer admin."""
-    list_display = ('id', 'site', )
+    list_display = ('id', 'site',)
+    exclude = ('links',)
+    inlines = [FooterLinkInline, ]
 
 
 @admin.register(models.FooterLink)
@@ -70,7 +77,6 @@ class FooterLinkAdmin(admin.ModelAdmin):
 @admin.register(models.Panel)
 class PanelAdmin(admin.ModelAdmin):
     """Panel admin."""
-    list_display = ('id', 'name', 'user', 'created', )
+    list_display = ('id', 'name', 'user', 'created',)
-    raw_id_fields = ('user', )
+    raw_id_fields = ('user',)
-    list_display_links = ('id', 'name', )
+    list_display_links = ('id', 'name',)
-

apps/main/migrations/0045_carousel_is_international.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.7 on 2019-12-23 14:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0044_auto_20191217_1125'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='carousel',
+            name='is_international',
+            field=models.BooleanField(default=False, verbose_name='is international'),
+        ),
+    ]

apps/main/models.py

@@ -214,6 +214,9 @@ class CarouselQuerySet(models.QuerySet):
         """Filter collection by country code."""
         return self.filter(country__code=code)
 
+    def get_international(self):
+        return self.filter(is_international=True)
+
 
 class Carousel(models.Model):
     """Carousel model."""
@@ -221,6 +224,7 @@ class Carousel(models.Model):
     object_id = models.PositiveIntegerField(blank=True, null=True, default=None)
     content_object = generic.GenericForeignKey('content_type', 'object_id')
 
+    is_international = models.BooleanField(default=False, verbose_name=_('is international'))
     old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
     title = models.CharField(_('old title'), max_length=255, blank=True, null=True, default=None)
     link = models.CharField(_('old link'), max_length=255, blank=True, null=True, default=None)

apps/main/serializers.py

@@ -39,6 +39,17 @@ class CurrencySerializer(ProjectModelSerializer):
         ]
 
 
+class _FooterLinkSerializer(serializers.ModelSerializer):
+    """FooterLink serializer"""
+
+    class Meta:
+        model = models.FooterLink
+        fields = [
+            'title',
+            'link',
+        ]
+
+
 class FooterSerializer(serializers.ModelSerializer):
     """Footer serializer."""
 
@@ -50,9 +61,15 @@ class FooterSerializer(serializers.ModelSerializer):
             'copyright',
             'created',
             'modified',
+            'links',
         ]
 
 
+class _FooterSerializer(FooterSerializer):
+    """Footer serializer."""
+    links = _FooterLinkSerializer(many=True, read_only=True)
+
+
 class FooterBackSerializer(FooterSerializer):
     site_id = serializers.PrimaryKeyRelatedField(
         queryset=models.SiteSettings.objects.all(),
@@ -98,7 +115,7 @@ class SiteSettingsSerializer(serializers.ModelSerializer):
 
     country_name = serializers.CharField(source='country.name_translated', read_only=True)
     time_format = serializers.CharField(source='country.time_format', read_only=True)
-    footers = FooterSerializer(many=True, read_only=True)
+    footers = _FooterSerializer(many=True, read_only=True)
 
     class Meta:
         """Meta class."""
@@ -300,6 +317,7 @@ class PanelSerializer(serializers.ModelSerializer):
 
 class PanelExecuteSerializer(serializers.ModelSerializer):
     """Panel execute serializer."""
+
     class Meta:
         model = models.Panel
         fields = [

apps/main/views/back.py

@@ -8,7 +8,7 @@ from rest_framework.response import Response
 from main import serializers
 from main import tasks
 from main.filters import AwardFilter
-from main.models import Award, Footer, PageType, Panel
+from main.models import Award, Footer, PageType, Panel, SiteFeature, Feature
 from main.views import SiteSettingsView, SiteListView
 
 
@@ -46,21 +46,29 @@ class ContentTypeView(generics.ListAPIView):
 class FeatureBackView(generics.ListCreateAPIView):
     """Feature list or create View."""
     serializer_class = serializers.FeatureSerializer
+    queryset = Feature.objects.all()
 
 
 class SiteFeatureBackView(generics.ListCreateAPIView):
     """Feature list or create View."""
     serializer_class = serializers.SiteFeatureSerializer
+    queryset = SiteFeature.objects.all()
+    pagination_class = None
+    permission_classes = [permissions.IsAdminUser]
 
 
 class FeatureRUDBackView(generics.RetrieveUpdateDestroyAPIView):
     """Feature RUD View."""
     serializer_class = serializers.FeatureSerializer
+    queryset = SiteFeature.objects.all()
+    permission_classes = [permissions.IsAdminUser]
 
 
 class SiteFeatureRUDBackView(generics.RetrieveUpdateDestroyAPIView):
     """Feature RUD View."""
     serializer_class = serializers.SiteFeatureSerializer
+    queryset = SiteFeature.objects.all()
+    permission_classes = [permissions.IsAdminUser]
 
 
 class SiteSettingsBackOfficeView(SiteSettingsView):

apps/news/migrations/0050_auto_20191223_1148.py

@@ -0,0 +1,34 @@
+# Generated by Django 2.2.7 on 2019-12-23 11:48
+
+from django.db import migrations, models
+import django.utils.timezone
+import utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0049_auto_20191223_0619'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='agenda',
+            name='event_datetime',
+        ),
+        migrations.AddField(
+            model_name='agenda',
+            name='end_datetime',
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='End datetime'),
+        ),
+        migrations.AddField(
+            model_name='agenda',
+            name='event_name',
+            field=utils.models.TJSONField(blank=True, default=None, help_text='{"en-GB":"some text"}', null=True, verbose_name='event name'),
+        ),
+        migrations.AddField(
+            model_name='agenda',
+            name='start_datetime',
+            field=models.DateTimeField(default=django.utils.timezone.now, verbose_name='Start datetime'),
+        ),
+    ]

apps/news/migrations/0050_auto_20191223_1238.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.7 on 2019-12-23 12:38
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0049_auto_20191223_0619'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='news',
+            name='news_type',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='news', to='news.NewsType', verbose_name='news type'),
+        ),
+    ]

apps/news/migrations/0051_merge_20191223_1405.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-23 14:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0050_auto_20191223_1148'),
+        ('news', '0050_auto_20191223_1238'),
+    ]
+
+    operations = [
+    ]

apps/news/models.py

@@ -22,12 +22,16 @@ from datetime import datetime
 
 class Agenda(ProjectBaseMixin, TranslatedFieldsMixin):
     """News agenda model"""
-
+    start_datetime = models.DateTimeField(default=timezone.now, editable=True,
-    event_datetime = models.DateTimeField(default=timezone.now, editable=False,
+                                          verbose_name=_('Start datetime'))
-                                          verbose_name=_('Event datetime'))
+    end_datetime = models.DateTimeField(default=timezone.now, editable=True,
+                                          verbose_name=_('End datetime'))
     address = models.ForeignKey('location.Address', blank=True, null=True,
                                 default=None, verbose_name=_('address'),
                                 on_delete=models.SET_NULL)
+    event_name = TJSONField(blank=True, null=True, default=None,
+                         verbose_name=_('event name'),
+                         help_text='{"en-GB":"some text"}')
     content = TJSONField(blank=True, null=True, default=None,
                          verbose_name=_('content'),
                          help_text='{"en-GB":"some text"}')
@@ -177,7 +181,7 @@ class News(GalleryMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixin,
 
     old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
     news_type = models.ForeignKey(NewsType, on_delete=models.PROTECT,
-                                  verbose_name=_('news type'))
+                                  verbose_name=_('news type'), related_name='news')
     title = TJSONField(blank=True, null=True, default=None,
                        verbose_name=_('title'),
                        help_text='{"en-GB":"some text"}')

apps/news/serializers.py

@@ -1,26 +1,29 @@
 """News app common serializers."""
+from django.shortcuts import get_object_or_404
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 from rest_framework.fields import SerializerMethodField
 
 from account.serializers.common import UserBaseSerializer
 from gallery.models import Image
-from main.models import SiteSettings, Carousel
 from location import models as location_models
-from location.serializers import CountrySimpleSerializer, AddressBaseSerializer
+from location.serializers import AddressBaseSerializer, CountrySimpleSerializer
+from main.models import SiteSettings
 from news import models
+from rating import models as rating_models
 from tag.serializers import TagBaseSerializer
 from utils import exceptions as utils_exceptions
-from utils.serializers import (TranslatedField, ProjectModelSerializer,
-                               FavoritesCreateSerializer, ImageBaseSerializer, CarouselCreateSerializer)
-from rating import models as rating_models
-from django.shortcuts import get_object_or_404
 from utils.models import get_current_locale, get_default_locale
+from utils.serializers import (
+    CarouselCreateSerializer, FavoritesCreateSerializer, ImageBaseSerializer, ProjectModelSerializer, TranslatedField,
+)
 
 
 class AgendaSerializer(ProjectModelSerializer):
-    event_datetime = serializers.DateTimeField()
+    start_datetime = serializers.DateTimeField()
+    end_datetime = serializers.DateTimeField()
     address = AddressBaseSerializer()
+    event_name_translated = TranslatedField()
     content_translated = TranslatedField()
 
     class Meta:
@@ -29,9 +32,11 @@ class AgendaSerializer(ProjectModelSerializer):
         model = models.Agenda
         fields = (
             'id',
-            'event_datetime',
+            'start_datetime',
+            'end_datetime',
             'address',
-            'content_translated'
+            'content_translated',
+            'event_name_translated'
         )
 
 
@@ -125,8 +130,7 @@ class NewsDetailSerializer(NewsBaseSerializer):
     description_translated = TranslatedField()
     country = CountrySimpleSerializer(read_only=True)
     author = UserBaseSerializer(source='created_by', read_only=True)
-    state_display = serializers.CharField(source='get_state_display',
+    state_display = serializers.CharField(source='get_state_display', read_only=True)
-                                          read_only=True)
     gallery = ImageBaseSerializer(read_only=True, source='crop_gallery', many=True)
     start = serializers.DateTimeField(source='publication_datetime', read_only=True)
 
@@ -196,17 +200,24 @@ class NewsBackOfficeBaseSerializer(NewsBaseSerializer):
             'created': {'read_only': True},
             'modified': {'read_only': True},
             'duplication_date': {'read_only': True},
-            'locale_to_description_is_active':  {'allow_null': False},
+            'locale_to_description_is_active': {'allow_null': False},
-            'must_of_the_week':  {'read_only': True},
+            'must_of_the_week': {'read_only': True},
         }
 
     def create(self, validated_data):
         slugs = validated_data.get('slugs')
+
         if slugs:
             if models.News.objects.filter(
                     slugs__values__contains=list(slugs.values())
             ).exists():
                 raise serializers.ValidationError({'slugs': _('News with this slug already exists.')})
+
+        request = self.context.get("request")
+        if request and hasattr(request, "user"):
+            user = request.user
+            validated_data['created_by'] = user
+
         return super().create(validated_data)
 
     def update(self, instance, validated_data):
@@ -375,11 +386,12 @@ class NewsCarouselCreateSerializer(CarouselCreateSerializer):
 
 
 class NewsCloneCreateSerializer(NewsBackOfficeBaseSerializer,
-                                     NewsDetailSerializer):
+                                NewsDetailSerializer):
     """Serializer for creating news clone."""
     template_display = serializers.CharField(source='get_template_display',
                                              read_only=True)
     duplicates = NewsBackOfficeDuplicationInfoSerializer(many=True, allow_null=True, read_only=True)
+
     class Meta(NewsBackOfficeBaseSerializer.Meta, NewsDetailSerializer.Meta):
         fields = NewsBackOfficeBaseSerializer.Meta.fields + NewsDetailSerializer.Meta.fields + (
             'template_display',
@@ -394,4 +406,3 @@ class NewsCloneCreateSerializer(NewsBackOfficeBaseSerializer,
         view_count_model = rating_models.ViewCount.objects.create(count=0)
         instance.create_duplicate(new_country, view_count_model)
         return get_object_or_404(models.News, pk=kwargs['pk'])
-

apps/news/tests.py

@@ -79,7 +79,6 @@ class NewsTestCase(BaseTestCase):
             "title": {"ru-RU": "Test news POST"},
             "news_type_id": self.test_news_type.id,
             "description": {"ru-RU": "Description test news"},
-            "start": datetime.now() + timedelta(hours=-2),
             "end": datetime.now() + timedelta(hours=2),
             "state": News.PUBLISHED,
             "slugs": {'en-GB': 'test-news-slug_post'},

apps/news/transfer_data.py

@@ -35,13 +35,14 @@ def clear_old_news():
 
     images.delete()
     news.delete()
+    # NewsType.objects.all().delete()
 
     print(f'Deleted {img_num} images')
     print(f'Deleted {news_num} news')
 
 
 def transfer_news():
-    news_type, _ = NewsType.objects.get_or_create(name='News')
+    news_type, _ = NewsType.objects.get_or_create(name='news')
 
     queryset = PageTexts.objects.filter(
         page__type='News',
@@ -116,7 +117,7 @@ def add_tags():
     Add news tags
     """
 
-    news_type, _ = NewsType.objects.get_or_create(name='News')
+    news_type, _ = NewsType.objects.get_or_create(name='news')
     tag_category, _ = TagCategory.objects.get_or_create(index_name='category')
     tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag')
     news_type.tag_categories.add(tag_category)

apps/news/views.py

@@ -125,7 +125,7 @@ class NewsBackOfficeGalleryCreateDestroyView(NewsBackOfficeMixinView,
         _ = super().create(request, *args, **kwargs)
         news_qs = self.filter_queryset(self.get_queryset())
         return response.Response(
-            data=serializers.NewsDetailSerializer(get_object_or_404(news_qs, pk=kwargs.get('pk'))).data
+            data=serializers.NewsBackOfficeDetailSerializer(get_object_or_404(news_qs, pk=kwargs.get('pk'))).data
         )
 
     def get_object(self):

apps/notification/migrations/0004_auto_20191118_1307.py

@@ -0,0 +1,35 @@
+# Generated by Django 2.2.7 on 2019-11-18 13:07
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+import utils.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('notification', '0003_auto_20191116_1248'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SubscriptionType',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Date created')),
+                ('modified', models.DateTimeField(auto_now=True, verbose_name='Date updated')),
+                ('index_name', models.CharField(max_length=255, unique=True, verbose_name='Index name')),
+                ('name', utils.models.TJSONField(blank=True, default=None, help_text='{"en-GB":"some text"}', null=True, verbose_name='name')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, utils.models.TranslatedFieldsMixin),
+        ),
+        migrations.AddField(
+            model_name='subscriber',
+            name='subscription_type',
+            field=models.ForeignKey(default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to='notification.SubscriptionType'),
+        ),
+    ]

apps/notification/models.py

@@ -4,7 +4,14 @@ from django.db import models
 from django.utils.translation import ugettext_lazy as _
 from account.models import User
 from utils.methods import generate_string_code
-from utils.models import ProjectBaseMixin
+from utils.models import ProjectBaseMixin, TranslatedFieldsMixin, TJSONField
+
+
+class SubscriptionType(ProjectBaseMixin, TranslatedFieldsMixin):
+    index_name = models.CharField(max_length=255, verbose_name=_('Index name'), unique=True)
+    name = TJSONField(blank=True, null=True, default=None,
+                      verbose_name=_('name'),
+                      help_text='{"en-GB":"some text"}')
 
 
 # todo: associate user & subscriber after users registration
@@ -12,7 +19,7 @@ class SubscriberManager(models.Manager):
     """Extended manager for Subscriber model."""
 
     def make_subscriber(self, email=None, user=None, ip_address=None, country_code=None,
-                        locale=None, *args, **kwargs):
+                        locale=None, subscription_type=None, *args, **kwargs):
         """Make subscriber and update info."""
         # search existing object
         if not user:
@@ -35,10 +42,12 @@ class SubscriberManager(models.Manager):
             obj.locale = locale
             obj.state = self.model.USABLE
             obj.update_code = generate_string_code()
+            obj.subscription_type = subscription_type
             obj.save()
         else:
             obj = self.model.objects.create(user=user, email=email, ip_address=ip_address,
-                                            country_code=country_code, locale=locale)
+                                            country_code=country_code, locale=locale,
+                                            subscription_type=subscription_type)
         return obj
 
     def associate_user(self, user):
@@ -98,6 +107,8 @@ class Subscriber(ProjectBaseMixin):
     )
     old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
 
+    subscription_type = models.ForeignKey(SubscriptionType, on_delete=models.CASCADE, null=True, default=None)
+
     objects = SubscriberManager.from_queryset(SubscriberQuerySet)()
 
     class Meta:

apps/notification/serializers/common.py

@@ -3,18 +3,39 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework import serializers
 from notification import models
 from utils.methods import get_user_ip
+from utils.serializers import TranslatedField
+
+
+class SubscriptionTypeSerializer(serializers.ModelSerializer):
+    """Subscription type serializer."""
+
+    name_translated = TranslatedField()
+    class Meta:
+        """Meta class."""
+
+        model = models.SubscriptionType
+        fields = (
+            'id',
+            'index_name',
+            'name_translated',
+        )
 
 
 class SubscribeSerializer(serializers.ModelSerializer):
     """Subscribe serializer."""
 
     email = serializers.EmailField(required=False, source='send_to')
+    subscription_type = SubscriptionTypeSerializer(read_only=True)
 
     class Meta:
         """Meta class."""
 
         model = models.Subscriber
-        fields = ('email', 'state',)
+        fields = (
+            'email',
+            'subscription_type',
+            'state',
+        )
         read_only_fields = ('state',)
 
     def validate(self, attrs):
@@ -38,9 +59,16 @@ class SubscribeSerializer(serializers.ModelSerializer):
         attrs['ip_address'] = get_user_ip(request)
         if user.is_authenticated:
             attrs['user'] = user
+
+        subscription_type_id = self.context.get('request').parser_context.get('kwargs').get("subscription_type_pk")
+        subscription_type_qs = models.SubscriptionType.objects.filter(id=subscription_type_id)
+        if not subscription_type_qs.exists():
+            raise serializers.ValidationError({'detail': _(f'SubscriptionType not found.')})
+        attrs["subscription_type"] = subscription_type_qs.first()
+
         return attrs
 
     def create(self, validated_data):
         """Create obj."""
-        obj = models.Subscriber.objects.make_subscriber(**validated_data)
+        subscriber = models.Subscriber.objects.make_subscriber(**validated_data)
-        return obj
+        return subscriber

apps/notification/urls/common.py

@@ -5,9 +5,10 @@ from notification.views import common
 app_name = "notification"
 
 urlpatterns = [
-    path('subscribe/', common.SubscribeView.as_view(), name='subscribe'),
+    path('subscribe/<int:subscription_type_pk>', common.SubscribeView.as_view(), name='subscribe'),
     path('subscribe-info/', common.SubscribeInfoAuthUserView.as_view(), name='check-code-auth'),
     path('subscribe-info/<code>/', common.SubscribeInfoView.as_view(), name='check-code'),
     path('unsubscribe/', common.UnsubscribeAuthUserView.as_view(), name='unsubscribe-auth'),
     path('unsubscribe/<code>/', common.UnsubscribeView.as_view(), name='unsubscribe'),
-]
+    path('subscription-types/', common.SubscriptionTypesView.as_view(), name='subscription-types'),
+]

apps/notification/views/common.py

@@ -30,20 +30,16 @@ class SubscribeInfoView(generics.RetrieveAPIView):
     serializer_class = serializers.SubscribeSerializer
 
 
-class SubscribeInfoAuthUserView(generics.RetrieveAPIView):
+class SubscribeInfoAuthUserView(generics.ListAPIView):
     """Subscribe info auth user view."""
 
     permission_classes = (permissions.IsAuthenticated, )
-    queryset = models.Subscriber.objects.all()
     serializer_class = serializers.SubscribeSerializer
 
-    def get_object(self):
+    def get_queryset(self):
         user = self.request.user
-        queryset = self.filter_queryset(self.get_queryset())
+        queryset = self.filter_queryset(models.Subscriber.objects.all())
-        filter_kwargs = {'user': user}
+        return queryset.filter(user=user)
-        obj = get_object_or_404(queryset, **filter_kwargs)
-        self.check_object_permissions(self.request, obj)
-        return obj
 
 
 class UnsubscribeView(generics.GenericAPIView):
@@ -76,3 +72,10 @@ class UnsubscribeAuthUserView(generics.GenericAPIView):
         serializer = self.get_serializer(instance=obj)
         return Response(data=serializer.data)
 
+
+class SubscriptionTypesView(generics.ListAPIView):
+    pagination_class = None
+    permission_classes = (permissions.AllowAny,)
+    queryset = models.SubscriptionType.objects.all()
+    serializer_class = serializers.SubscriptionTypeSerializer
+

apps/product/migrations/0021_product_site.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.7 on 2019-12-10 14:13
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0040_footer'),
+        ('product', '0020_merge_20191209_0911'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='product',
+            name='site',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='main.SiteSettings'),
+        ),
+    ]

apps/product/migrations/0022_auto_20191210_1517.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.7 on 2019-12-10 15:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('product', '0021_product_site'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='producttype',
+            name='index_name',
+            field=models.CharField(choices=[('food', 'food'), ('wine', 'wine'), ('liquor', 'liquor'), ('souvenir', 'souvenir'), ('book', 'book')], db_index=True, max_length=50, unique=True, verbose_name='Index name'),
+        ),
+    ]

apps/product/migrations/0023_merge_20191217_1127.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-17 11:27
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('product', '0022_auto_20191210_1517'),
+        ('product', '0021_auto_20191212_0926'),
+    ]
+
+    operations = [
+    ]

apps/product/migrations/0024_merge_20191223_1405.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.7 on 2019-12-23 14:05
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('product', '0023_merge_20191217_1127'),
+        ('product', '0022_auto_20191220_1007'),
+    ]
+
+    operations = [
+    ]

apps/product/models.py

@@ -30,10 +30,17 @@ class ProductType(TypeDefaultImageMixin, TranslatedFieldsMixin, ProjectBaseMixin
     SOUVENIR = 'souvenir'
     BOOK = 'book'
 
+    INDEX_CHOICES = (
+        (FOOD, 'food'),
+        (WINE, 'wine'),
+        (LIQUOR, 'liquor'),
+        (SOUVENIR, 'souvenir'),
+        (BOOK, 'book')
+    )
     name = TJSONField(blank=True, null=True, default=None,
                       verbose_name=_('Name'), help_text='{"en-GB":"some text"}')
     index_name = models.CharField(max_length=50, unique=True, db_index=True,
-                                  verbose_name=_('Index name'))
+                                  verbose_name=_('Index name'), choices=INDEX_CHOICES)
     use_subtypes = models.BooleanField(_('Use subtypes'), default=True)
     tag_categories = models.ManyToManyField('tag.TagCategory',
                                             related_name='product_types',
@@ -289,6 +296,8 @@ class Product(GalleryMixin, TranslatedFieldsMixin, BaseAttributes,
                                      default=None, null=True,
                                      verbose_name=_('Serial number'))
 
+    site = models.ForeignKey(to='main.SiteSettings', null=True, blank=True, on_delete=models.CASCADE)
+
     objects = ProductManager.from_queryset(ProductQuerySet)()
 
     class Meta:

apps/product/serializers/back.py

@@ -8,7 +8,7 @@ from product.serializers import ProductDetailSerializer, ProductTypeBaseSerializ
                                 ProductSubTypeBaseSerializer
 from tag.models import TagCategory
 from account.serializers.common import UserShortSerializer
-
+from main.serializers import SiteSettingsSerializer
 
 class ProductBackOfficeGallerySerializer(serializers.ModelSerializer):
     """Serializer class for model ProductGallery."""
@@ -55,6 +55,7 @@ class ProductBackOfficeGallerySerializer(serializers.ModelSerializer):
 
 class ProductBackOfficeDetailSerializer(ProductDetailSerializer):
     """Product back-office detail serializer."""
+    in_favorites = serializers.BooleanField(allow_null=True, read_only=True)
 
     class Meta(ProductDetailSerializer.Meta):
         """Meta class."""
@@ -68,9 +69,10 @@ class ProductBackOfficeDetailSerializer(ProductDetailSerializer):
             # 'wine_sub_region',
             'wine_village',
             'state',
+            'site',
+            'product_type'
         ]
 
-
 class ProductTypeBackOfficeDetailSerializer(ProductTypeBaseSerializer):
     """Product type back-office detail serializer."""
 

apps/product/serializers/common.py

@@ -101,7 +101,7 @@ class ProductBaseSerializer(serializers.ModelSerializer):
     wine_colors = TagBaseSerializer(many=True, read_only=True)
     preview_image_url = serializers.URLField(allow_null=True,
                                              read_only=True)
-    in_favorites = serializers.BooleanField(allow_null=True)
+    in_favorites = serializers.BooleanField(allow_null=True, read_only=True)
     wine_origins = EstablishmentWineOriginBaseSerializer(many=True, read_only=True)
 
     class Meta:

apps/product/tests.py

@@ -0,0 +1,121 @@
+from rest_framework.test import APITestCase
+from rest_framework import status
+from account.models import User
+from http.cookies import SimpleCookie
+from django.urls import reverse
+
+# Create your tests here.
+from translation.models import Language
+from account.models import Role, UserRole
+from location.models import Country, Address, City, Region
+from main.models import SiteSettings
+from product.models import Product, ProductType
+
+class BaseTestCase(APITestCase):
+    def setUp(self):
+        self.username = 'sedragurda'
+        self.password = 'sedragurdaredips19'
+        self.email = 'sedragurda@desoz.com'
+        self.newsletter = True
+        self.user = User.objects.create_user(
+            username=self.username,
+            email=self.email,
+            password=self.password,
+            is_staff=True,
+        )
+        # get tokens
+        tokens = User.create_jwt_tokens(self.user)
+        self.client.cookies = SimpleCookie(
+            {'access_token': tokens.get('access_token'),
+             'refresh_token': tokens.get('refresh_token')})
+
+
+        self.lang = Language.objects.create(
+            title='Russia',
+            locale='ru-RU'
+        )
+
+        self.country_ru = Country.objects.create(
+            name={'en-GB': 'Russian'},
+            code='RU',
+        )
+
+        self.region = Region.objects.create(name='Moscow area', code='01',
+                                            country=self.country_ru)
+        self.region.save()
+
+        self.city = City.objects.create(
+            name='Mosocow', code='01',
+            region=self.region,
+            country=self.country_ru)
+        self.city.save()
+
+        self.address = Address.objects.create(
+            city=self.city, street_name_1='Krasnaya',
+            number=2, postal_code='010100')
+        self.address.save()
+
+        self.site = SiteSettings.objects.create(
+            subdomain='ru',
+            country=self.country_ru
+        )
+
+        self.site.save()
+
+        self.role = Role.objects.create(role=Role.LIQUOR_REVIEWER,
+                                        site=self.site)
+        self.role.save()
+
+        self.user_role = UserRole.objects.create(
+             user=self.user, role=self.role)
+
+        self.user_role.save()
+
+        self.product_type = ProductType.objects.create(index_name=ProductType.LIQUOR)
+        self.product_type.save()
+
+        self.product = Product.objects.create(name='Product')
+        self.product.save()
+
+
+
+class LiquorReviewerTests(BaseTestCase):
+    def test_get(self):
+        self.product.product_type = self.product_type
+        self.product.save()
+
+        url = reverse("back:product:list-create")
+        response = self.client.get(url, format='json')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        url = reverse("back:product:rud", kwargs={'pk': self.product.id})
+        response = self.client.get(url, format='json')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+    def test_post_patch_put_delete(self):
+        data_post = {
+                "slug": None,
+                "public_mark": None,
+                "vintage": None,
+                "average_price": None,
+                "description": None,
+                "available": False,
+                "establishment": None,
+                "wine_village": None,
+                "state": Product.PUBLISHED,
+                "site_id": self.site.id,
+                "product_type_id": self.product_type.id
+        }
+        url = reverse("back:product:list-create")
+        response = self.client.post(url, data=data_post, format='json')
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        data_patch = {
+            'name': 'Test product'
+        }
+        url = reverse("back:product:rud", kwargs={'pk': self.product.id})
+        response = self.client.patch(url, data=data_patch, format='json')
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+
+

apps/product/views/back.py

@@ -7,6 +7,7 @@ from product import serializers, models
 from product.views import ProductBaseView
 from utils.serializers import ImageBaseSerializer
 from utils.views import CreateDestroyGalleryViewMixin
+from utils.permissions import IsLiquorReviewer, IsProductReviewer
 
 
 class ProductBackOfficeMixinView(ProductBaseView):
@@ -91,12 +92,14 @@ class ProductDetailBackOfficeView(ProductBackOfficeMixinView,
                                   generics.RetrieveUpdateDestroyAPIView):
     """Product back-office R/U/D view."""
     serializer_class = serializers.ProductBackOfficeDetailSerializer
+    permission_classes = [IsLiquorReviewer | IsProductReviewer]
 
 
 class ProductListCreateBackOfficeView(BackOfficeListCreateMixin, ProductBackOfficeMixinView,
                                       generics.ListCreateAPIView):
     """Product back-office list-create view."""
     serializer_class = serializers.ProductBackOfficeDetailSerializer
+    permission_classes = [IsLiquorReviewer | IsProductReviewer]
 
 
 class ProductTypeListCreateBackOfficeView(BackOfficeListCreateMixin,

apps/search_indexes/signals.py

@@ -55,9 +55,9 @@ def update_product(sender, **kwargs):
         ('product', 'productstandard'): 'standards',
         ('product', 'producttype'): 'product_type',
         ('tag', 'tag'): 'tags',
-        ('location', 'wineregion'): 'wine_region',
+        # ('location', 'wineregion'): 'wine_region',
-        ('location', 'winesubregion'): 'wine_sub_region',
+        # ('location', 'winesubregion'): 'wine_sub_region',
-        ('location', 'winevillage'): 'wine_village',
+        # ('location', 'winevillage'): 'wine_village',
         ('establishment', 'establishment'): 'establishment',
     }
     filter_name = app_label_model_name_to_filter.get((app_label, model_name))

apps/tag/serializers.py

@@ -49,6 +49,8 @@ class TagBaseSerializer(serializers.ModelSerializer):
 class TagBackOfficeSerializer(TagBaseSerializer):
     """Serializer for Tag model for Back office users."""
 
+    label = serializers.DictField(source='translation.text')
+
     class Meta(TagBaseSerializer.Meta):
         """Meta class."""
 
@@ -60,7 +62,6 @@ class TagBackOfficeSerializer(TagBaseSerializer):
 
 class TagCategoryProductSerializer(serializers.ModelSerializer):
     """SHORT Serializer for TagCategory"""
-    label_translated = serializers.SerializerMethodField(read_only=True, allow_null=True)
 
     def get_label_translated(self, obj):
         return translate_obj(obj)
@@ -177,6 +178,7 @@ class TagCategoryBackOfficeDetailSerializer(TagCategoryBaseSerializer):
     """Tag Category detail serializer for back-office users."""
 
     country_translated = TranslatedField(source='country.name_translated')
+    label = serializers.DictField(source='translation.text')
 
     class Meta(TagCategoryBaseSerializer.Meta):
         """Meta class."""

apps/transfer/management/commands/transfer.py

@@ -50,7 +50,15 @@ class Command(BaseCommand):
         'guide_element_advertorials',
         'guide_element_label_photo',
         'guide_complete',
+        'update_city_info',
+        'migrate_city_gallery',
+        'fix_location',
+        'remove_old_locations',
+        'add_fake_country',
+        'setup_clean_db',
         'languages',  # №4 - перенос языков
+        'set_unused_regions',
+        'update_fake_country_flag'
     ]
 
     def handle(self, *args, **options):

apps/transfer/serializers/location.py

@@ -6,9 +6,12 @@ from location import models
 from transfer.mixins import TransferSerializerMixin
 from utils.methods import get_point_from_coordinates
 from transfer.models import Cepages
-from tag.models import TagCategory
 from django.utils.text import slugify
 
+from django.contrib.gis.geos import Point
+from django.core.exceptions import MultipleObjectsReturned
+from gallery.models import Image
+
 
 class CountrySerializer(serializers.ModelSerializer):
     country_code_2 = serializers.CharField()
@@ -73,7 +76,6 @@ class RegionSerializer(serializers.ModelSerializer):
         return region
 
     def set_code(self, data):
-        print(data)
         if "subregion_code" in data and data["subregion_code"] is not None and data["subregion_code"].strip() != "":
             try:
                 parent_region = models.Region.objects.filter(code=str(data['region_code'])).first()
@@ -187,7 +189,7 @@ class AddressSerializer(serializers.ModelSerializer):
     zip_code = serializers.CharField(allow_null=True, allow_blank=True)
     latitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True)
     longitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True)
-    address = serializers.CharField()
+    address = serializers.CharField(allow_null=True, allow_blank=True)
 
     class Meta:
         model = models.Address
@@ -223,7 +225,7 @@ class AddressSerializer(serializers.ModelSerializer):
 
     def set_city(self, data):
         try:
-            city = models.City.objects.filter(old_id=data['city_id']).first()
+            city = models.City.objects.filter(mysql_id=data['city_id']).first()
         except models.City.DoesNotExist as e:
             raise ValueError(f"City not found with {data}: {e}")
 
@@ -232,25 +234,29 @@ class AddressSerializer(serializers.ModelSerializer):
         return data
 
     def set_address(self, data):
-        address_list = data.pop('address').split(' ')
+        if "address" in data and data['address'] is not None and data['address'] != "":
-        is_first_street = False
+            address_list = data.pop('address').split(' ')
-        data['street_name_1'] = []
+            is_first_street = False
-        data['street_name_2'] = []
+            data['street_name_1'] = []
-        while len(address_list) > 0:
+            data['street_name_2'] = []
-            address_part = address_list.pop()
+            while len(address_list) > 0:
-            try:
+                address_part = address_list.pop()
-                address_part = int(address_part)
+                try:
-                data['number'] = address_part
+                    address_part = int(address_part)
-                is_first_street = True
+                    data['number'] = address_part
-            except:
+                    is_first_street = True
-                if is_first_street:
+                except:
-                    data['street_name_1'].append(address_part)
+                    if is_first_street:
-                else:
+                        data['street_name_1'].append(address_part)
-                    data['street_name_2'].append(address_part)
+                    else:
+                        data['street_name_2'].append(address_part)
 
-        data['street_name_1'] = " ".join(data['street_name_1'])
+            data['street_name_1'] = " ".join(data['street_name_1'])
-        data['street_name_2'] = " ".join(data['street_name_2'])
+            data['street_name_2'] = " ".join(data['street_name_2'])
-        if "number" not in data:
+            if "number" not in data:
+                data['number'] = 0
+        else:
+            del(data['address'])
             data['number'] = 0
 
         return data
@@ -273,7 +279,7 @@ class AddressSerializer(serializers.ModelSerializer):
         return data
 
 
-class WineRegion(TransferSerializerMixin):
+class WineRegionSerializer(TransferSerializerMixin):
 
     id = serializers.IntegerField()
     name = serializers.CharField()
@@ -301,7 +307,8 @@ class WineRegion(TransferSerializerMixin):
         return attrs
 
 
-class WineSubRegion(WineRegion):
+
+class WineSubRegionSerializer(WineRegionSerializer):
 
     id = serializers.IntegerField()
     name = serializers.CharField()
@@ -328,7 +335,7 @@ class WineSubRegion(WineRegion):
             return qs.first()
 
 
-class WineVillage(TransferSerializerMixin):
+class WineVillageSerializer(TransferSerializerMixin):
 
     id = serializers.IntegerField()
     name = serializers.CharField()
@@ -349,6 +356,125 @@ class WineVillage(TransferSerializerMixin):
         return attrs
 
 
+class CityMapSerializer(serializers.ModelSerializer):
+    id = serializers.IntegerField()
+    map1 = serializers.CharField(allow_blank=True, allow_null=True)
+    map2 = serializers.CharField(allow_blank=True, allow_null=True)
+    map_ref = serializers.CharField(allow_blank=True, allow_null=True)
+    situation = serializers.CharField(allow_blank=True, allow_null=True)
+    
+    class Meta:
+        model = models.City
+        fields = (
+            "id",
+            "map1",
+            "map2",
+            "map_ref",
+            "situation",
+        )
+
+    def validate(self, data):
+        data = self.set_old_id(data)
+        data = self.set_city(data)
+        data = self.set_null_fields(data)
+        return data
+
+    def create(self, validated_data):
+        city = validated_data.pop('city')
+        city.update(
+            map1=validated_data['map1'],
+            map2=validated_data['map2'],
+            map_ref=validated_data['map_ref'],
+            situation=validated_data['situation']
+        )
+
+    def set_old_id(self, data):
+        data['old_id'] = data.pop("id")
+        return data
+
+    def set_city(self, data):
+        try:
+            city = models.City.objects.get(old_id=data['old_id'])
+        except models.City.DoesNotExist as e:
+            raise ValueError(f"Cannot find city with id = {data['old_id']}: {e}")
+        except MultipleObjectsReturned as e:
+            raise ValueError(f"Find multiple cities with id = {data['old_id']}: {e}")
+
+        data['city'] = city
+        return data
+
+    def set_null_fields(self, data):
+        for field in ["map1", "map2", "map_ref", "situation"]:
+            if field not in data or data[field] is None:
+                data[field] = ""
+        return data
+
+
+class CityMapCorrectSerializer(CityMapSerializer):
+    def set_city(self, data):
+        try:
+            city = models.City.objects.get(mysql_id=data['old_id'])
+        except models.City.DoesNotExist as e:
+            raise ValueError(f"Cannot find city with id = {data['old_id']}: {e}")
+        except MultipleObjectsReturned as e:
+            raise ValueError(f"Find multiple cities with id = {data['old_id']}: {e}")
+
+        data['city'] = city
+        return data
+
+    def create(self, validated_data):
+        city = validated_data.pop('city')
+        city.map1=validated_data['map1'],
+        city.map2=validated_data['map2'],
+        city.map_ref=validated_data['map_ref'],
+        city.situation=validated_data['situation']
+        city.save()
+
+
+class CityGallerySerializer(serializers.ModelSerializer):
+    id = serializers.IntegerField()
+    city_id = serializers.IntegerField()
+    attachment_file_name = serializers.CharField()
+
+    class Meta:
+        model = models.CityGallery
+        fields = ("id", "city_id", "attachment_file_name")
+
+    def validate(self, data):
+        data = self.set_old_id(data)
+        data = self.set_gallery(data)
+        data = self.set_city(data)
+        return data
+
+    def create(self, validated_data):
+        return models.CityGallery.objects.create(**validated_data)
+
+    def set_old_id(self, data):
+        data['old_id'] = data.pop('id')
+        return data
+
+    def set_gallery(self, data):
+        link_prefix = "city_photos/00baf486523f62cdf131fa1b19c5df2bf21fc9f8/"
+        try:
+            data['image'] = Image.objects.create(
+                image=f"{link_prefix}{data['attachment_file_name']}"
+            )
+        except Exception as e:
+            raise ValueError(f"Cannot create image with {data}: {e}")
+        del(data['attachment_file_name'])
+        return data
+
+    def set_city(self, data):
+        try:
+            data['city'] = models.City.objects.get(old_id=data.pop('city_id'))
+        except models.City.DoesNotExist as e:
+            raise ValueError(f"Cannot get city with {data}: {e}")
+        except MultipleObjectsReturned as e:
+            raise ValueError(f"Multiple cities find with {data}: {e}")
+
+        return data
+
+
 class CepageWineRegionSerializer(TransferSerializerMixin):
 
     CATEGORY_LABEL = 'Cepage'
@@ -358,7 +484,7 @@ class CepageWineRegionSerializer(TransferSerializerMixin):
         queryset=Cepages.objects.all())
     wine_region_id = serializers.IntegerField()
 
-    class Meta(WineRegion.Meta):
+    class Meta(WineRegionSerializer.Meta):
         fields = [
             'cepage_id',
             'wine_region_id',

apps/transfer/utils.py

@@ -1,10 +1,17 @@
 from os.path import exists
+from os import makedirs
 from importlib.machinery import SourceFileLoader
 from django.apps import apps
+from django.conf import settings
+from django.db.models import Count, Q
+from tqdm import tqdm
+import sys
+
+import timeit
 
 
 def transfer_objects(data_type):
-
+    start_t = timeit.default_timer()
     for app in apps.get_app_configs():
         if exists(f"{app.path}/transfer_data.py"):
             card_module = SourceFileLoader("transfer", f"{app.path}/transfer_data.py").load_module()
@@ -17,4 +24,164 @@ def transfer_objects(data_type):
                 if data_type == module_data_type:
                     for transfer_func in transfer_funcs:
                         print(f"========================== FUNCTION {transfer_func.__name__} ================================")
+                        transfer_func = file_log(transfer_func)
                         transfer_func()
+    end_t = timeit.default_timer()
+    print(f"Transfer time: {end_t - start_t}")
+
+
+def clean_old_records(model, conditions):
+    error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
+    non_existed_correct_city_ids = []
+    to_delete = []
+
+    try:
+        old_records = model.objects.filter(**conditions)
+    except Exception as e:
+        error_file.write(f"Cannot find model objects: {e}")
+        return
+
+    for old_record in tqdm(old_records):
+        correct_record_qs = model.objects.exclude(id=old_record.id) \
+                                         .exclude(**conditions) \
+                                         .filter(name=old_record.name,
+                                                 mysql_id=old_record.old_id)
+        if correct_record_qs.exists():
+            correct_record = correct_record_qs.first()
+            # check object dependencies
+            for rel_instance in old_record._related_instances:
+                field_name = [related_field.name
+                              for related_field in rel_instance._meta.fields
+                              if related_field.related_model == correct_record._meta.model][0]
+                # rebinding correct dependency instances
+                if getattr(rel_instance, field_name) != correct_record:
+                    setattr(rel_instance, field_name, correct_record)
+                    rel_instance.save()
+            to_delete.append(old_record.id)
+        else:
+            non_existed_correct_city_ids.append(old_record.id)
+
+    if non_existed_correct_city_ids:
+        print(f"Non existed correct city ids: {non_existed_correct_city_ids}")
+
+    # delete old records
+    counter = len(to_delete)
+    old_records.filter(id__in=to_delete).delete()
+    print(f'Deleted {counter} objects.')
+
+
+def clean_old_country_records(model, conditions):
+    error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
+    non_existed_correct_country_ids = []
+    to_delete = []
+
+    try:
+        unique_codes = model.objects.values_list('code', flat=True) \
+                                    .annotate(counter=Count('code')) \
+                                    .filter(counter=1) \
+                                    .values_list('code', flat=True)
+        old_records = model.objects.exclude(code__in=unique_codes) \
+                                   .filter(**conditions)
+    except Exception as e:
+        error_file.write(f"Cannot find model objects: {e}")
+        return
+
+    for old_record in tqdm(old_records):
+        correct_record_qs = model.objects.exclude(id=old_record.id) \
+                                         .exclude(**conditions) \
+                                         .filter(Q(code=old_record.code) |
+                                                 Q(mysql_ids__contains=[old_record.old_id, ]))
+        if correct_record_qs.exists():
+            correct_record = correct_record_qs.first()
+            # check object dependencies
+            for rel_instance in old_record._related_instances:
+                if not hasattr(rel_instance, '_meta'):
+                    for related in rel_instance.all():
+                        field_name = [related_field.name
+                                      for related_field in related._meta.fields
+                                      if related_field.related_model == correct_record._meta.model][0]
+                        # rebinding correct dependency instances
+                        if getattr(rel_instance, field_name) != correct_record:
+                            setattr(rel_instance, field_name, correct_record)
+                            rel_instance.save()
+                else:
+                    field_name = [related_field.name
+                                  for related_field in rel_instance._meta.fields
+                                  if related_field.related_model == correct_record._meta.model][0]
+                    # rebinding correct dependency instances
+                    if getattr(rel_instance, field_name) != correct_record:
+                        setattr(rel_instance, field_name, correct_record)
+                        rel_instance.save()
+            to_delete.append(old_record.id)
+        else:
+            non_existed_correct_country_ids.append(old_record.id)
+
+    if non_existed_correct_country_ids:
+        print(f"Non existed correct country ids: {non_existed_correct_country_ids}")
+
+    # delete old records
+    counter = len(to_delete)
+    old_records.filter(id__in=to_delete).delete()
+    print(f'Deleted {counter} objects.')
+
+
+def clean_old_region_records(model, conditions):
+    error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
+    non_existed_correct_country_ids = []
+    to_delete = []
+
+    try:
+        old_records = model.objects.filter(**conditions)
+    except Exception as e:
+        error_file.write(f"Cannot find model objects: {e}")
+        return
+
+    for old_record in tqdm(old_records):
+        correct_record_qs = model.objects.exclude(id=old_record.id) \
+                                         .exclude(**conditions) \
+                                         .filter(code__iexact=old_record.code,
+                                                 mysql_ids__contains=[old_record.old_id, ])
+        if correct_record_qs.exists():
+            correct_record = correct_record_qs.first()
+            # check object dependencies
+            for rel_instance in old_record._related_instances:
+                if not hasattr(rel_instance, '_meta'):
+                    for related in rel_instance.all():
+                        field_name = [related_field.name
+                                      for related_field in related._meta.fields
+                                      if related_field.related_model == correct_record._meta.model][0]
+                        # rebinding correct dependency instances
+                        if getattr(rel_instance, field_name) != correct_record:
+                            setattr(rel_instance, field_name, correct_record)
+                            rel_instance.save()
+                else:
+                    field_name = [related_field.name
+                                  for related_field in rel_instance._meta.fields
+                                  if related_field.related_model == correct_record._meta.model][0]
+                    # rebinding correct dependency instances
+                    if getattr(rel_instance, field_name) != correct_record:
+                        setattr(rel_instance, field_name, correct_record)
+                        rel_instance.save()
+            to_delete.append(old_record.id)
+        else:
+            non_existed_correct_country_ids.append(old_record.id)
+
+    if non_existed_correct_country_ids:
+        print(f"Non existed correct region ids: {non_existed_correct_country_ids}")
+
+    # delete old records
+    counter = len(to_delete)
+    old_records.filter(id__in=to_delete).delete()
+    print(f'Deleted {counter} objects.')
+
+
+def file_log(f):
+    directory = f"{settings.PROJECT_ROOT}/apps/transfer/log"
+    if not exists(directory):
+        makedirs(directory)
+    sys.stdout = open(f"{directory}/{f.__name__}.log","w+")
+
+    def res_func(*args, **kwargs):
+        return f(*args, **kwargs)
+
+    return res_func

apps/utils/models.py

@@ -88,6 +88,7 @@ def translate_field(self, field_name, toggle_field_name=None):
         return None
     return translate
 
+
 # todo: refactor this
 class IndexJSON:
 

apps/utils/permissions.py

@@ -8,7 +8,9 @@ from account.models import UserRole, Role
 from authorization.models import JWTRefreshToken
 from utils.tokens import GMRefreshToken
 from establishment.models import EstablishmentSubType
-from location.models import  Address
+from location.models import Address
+from product.models import Product, ProductType
+
 
 class IsAuthenticatedAndTokenIsValid(permissions.BasePermission):
     """
@@ -81,33 +83,21 @@ class IsStandardUser(IsGuest):
     """
 
     def has_permission(self, request, view):
-        rules = [
-            super().has_permission(request, view)
-        ]
 
-        # and request.user.email_confirmed,
+        rules = [super().has_permission(request, view),
-        if hasattr(request, 'user'):
+                 request.user.is_authenticated,
-            rules = [
+                 hasattr(request, 'user')
-                request.user.is_authenticated,
+                ]
-                super().has_permission(request, view)
-            ]
 
         return any(rules)
 
     def has_object_permission(self, request, view, obj):
         # Read permissions are allowed to any request
-        rules = [
-            super().has_object_permission(request, view, obj)
-        ]
 
-        if hasattr(obj, 'user'):
+        rules = [super().has_object_permission(request, view, obj),
-            rules = [
+                 request.user.is_authenticated,
-                obj.user == request.user
+                 hasattr(request, 'user')
-                and obj.user.email_confirmed
+                ]
-                and request.user.is_authenticated,
-
-                super().has_object_permission(request, view, obj)
-            ]
 
         return any(rules)
 
@@ -408,7 +398,7 @@ class IsWineryReviewer(IsStandardUser):
 
             est = EstablishmentSubType.objects.filter(establishment_type_id=request.data['type_id'])
             if est.exists():
-                role = Role.objects.filter(establishment_subtype_id__in=[type.id for type in est],
+                role = Role.objects.filter(establishment_subtype_id__in=[est_type.id for est_type in est],
                                            role=Role.WINERY_REVIEWER,
                                            country_id__in=[country.id for country in countries]) \
                     .first()
@@ -433,7 +423,7 @@ class IsWineryReviewer(IsStandardUser):
 
             est = EstablishmentSubType.objects.filter(establishment_type_id=type_id)
             role = Role.objects.filter(role=Role.WINERY_REVIEWER,
-                                       establishment_subtype_id__in=[id for type.id in est],
+                                       establishment_subtype_id__in=[est_type.id for est_type in est],
                                        country_id=obj.country_id).first()
 
             object_id: int
@@ -448,4 +438,160 @@ class IsWineryReviewer(IsStandardUser):
                                         ).exists(),
                 super().has_object_permission(request, view, obj)
             ]
-        return any(rules)
+        return any(rules)
+
+
+class IsWineryReviewer(IsStandardUser):
+
+    def has_permission(self, request, view):
+        rules = [
+            super().has_permission(request, view)
+        ]
+
+        if 'type_id' in request.data and 'address_id' in request.data and request.user:
+            countries = Address.objects.filter(id=request.data['address_id'])
+
+            est = EstablishmentSubType.objects.filter(establishment_type_id=request.data['type_id'])
+            if est.exists():
+                role = Role.objects.filter(establishment_subtype_id__in=[est_type.id for est_type in est],
+                                           role=Role.WINERY_REVIEWER,
+                                           country_id__in=[country.id for country in countries]) \
+                    .first()
+
+                rules.append(
+                    UserRole.objects.filter(user=request.user, role=role).exists()
+                )
+
+        return any(rules)
+
+    def has_object_permission(self, request, view, obj):
+        rules = [
+            super().has_object_permission(request, view, obj)
+        ]
+
+        if hasattr(obj, 'type_id') or hasattr(obj, 'establishment_type_id'):
+            type_id: int
+            if hasattr(obj, 'type_id'):
+                type_id = obj.type_id
+            else:
+                type_id = obj.establishment_type_id
+
+            est = EstablishmentSubType.objects.filter(establishment_type_id=type_id)
+            role = Role.objects.filter(role=Role.WINERY_REVIEWER,
+                                       establishment_subtype_id__in=[est_type.id for est_type in est],
+                                       country_id=obj.country_id).first()
+
+            object_id: int
+            if hasattr(obj, 'object_id'):
+                object_id = obj.object_id
+            else:
+                object_id = obj.establishment_id
+
+            rules = [
+                UserRole.objects.filter(user=request.user, role=role,
+                                        establishment_id=object_id
+                                        ).exists(),
+                super().has_object_permission(request, view, obj)
+            ]
+        return any(rules)
+
+
+class IsProductReviewer(IsStandardUser):
+
+    def has_permission(self, request, view):
+        rules = [
+            super().has_permission(request, view)
+        ]
+
+        pk_object = None
+        roles = None
+        permission = False
+
+        if 'site_id' in request.data:
+            if request.data['site_id'] is not None:
+                roles = Role.objects.filter(role=Role.PRODUCT_REVIEWER,
+                                            site_id=request.data['site_id'])
+
+        if 'pk' in view.kwargs:
+            pk_object = view.kwargs['pk']
+
+        if pk_object is not None:
+            product = Product.objects.get(pk=pk_object)
+            if product.site_id is not None:
+                roles = Role.objects.filter(role=Role.PRODUCT_REVIEWER,
+                                            site_id=product.site_id)
+
+        if roles is not None:
+            permission = UserRole.objects.filter(user=request.user, role__in=[role for role in roles])\
+                .exists()
+
+        rules.append(permission)
+        return any(rules)
+
+
+class IsLiquorReviewer(IsStandardUser):
+    def has_permission(self, request, view):
+        rules = [
+            super().has_permission(request, view)
+        ]
+
+        pk_object = None
+        roles = None
+        permission = False
+
+        if 'site_id' in request.data and 'product_type_id' in request.data:
+            if request.data['site_id'] is not None \
+                    and request.data['product_type_id'] is not None:
+
+                product_types = ProductType.objects. \
+                    filter(index_name=ProductType.LIQUOR,
+                           id=request.data['product_type_id'])
+
+                if product_types.exists():
+                    roles = Role.objects.filter(role=Role.LIQUOR_REVIEWER,
+                                                site_id=request.data['site_id'])
+
+        if 'pk' in view.kwargs:
+            pk_object = view.kwargs['pk']
+
+        if pk_object is not None:
+            product = Product.objects.get(pk=pk_object)
+            if product.site_id is not None \
+                    and product.product_type_id is not None:
+
+                product_types = ProductType.objects. \
+                    filter(index_name=ProductType.LIQUOR,
+                           id=product.product_type_id)
+
+                if product_types.exists():
+                    roles = Role.objects.filter(role=Role.LIQUOR_REVIEWER,
+                                                site_id=product.site_id)
+
+        if roles is not None:
+            permission = UserRole.objects.filter(user=request.user, role__in=[role for role in roles])\
+                .exists()
+
+        rules.append(permission)
+        return any(rules)
+
+    #
+    # def has_object_permission(self, request, view, obj):
+    #     rules = [
+    #         super().has_object_permission(request, view, obj)
+    #     ]
+    #     # pk_object = None
+    #     # product = None
+    #     # permission = False
+    #     #
+    #     # if 'pk' in view.kwargs:
+    #     #     pk_object = view.kwargs['pk']
+    #     #
+    #     # if pk_object is not None:
+    #     #     product = Product.objects.get(pk=pk_object)
+    #     #
+    #     # if product.sites.exists():
+    #     #     role = Role.objects.filter(role=Role.LIQUOR_REVIEWER, site__in=[site for site in product.sites])
+    #     #     permission = UserRole.objects.filter(user=request.user, role=role).exists()
+    #     #
+    #     # rules.append(permission)
+    #     return any(rules)

apps/utils/signals.py

@@ -0,0 +1,12 @@
+from functools import wraps
+
+
+def skip_signal():
+    def _skip_signal(signal_func):
+        @wraps(signal_func)
+        def _decorator(sender, instance, **kwargs):
+            if hasattr(instance, 'skip_signal'):
+                return None
+            return signal_func(sender, instance=instance, **kwargs)
+        return _decorator
+    return _skip_signal

apps/utils/tests/tests_translated.py

@@ -53,7 +53,6 @@ class TranslateFieldTests(BaseTestCase):
                 "ru-RU": "Тестовая новость"
             },
             description={"en-GB": "Test description"},
-            start=datetime.now(pytz.utc) + timedelta(hours=-13),
             end=datetime.now(pytz.utc) + timedelta(hours=13),
             news_type=self.news_type,
             slugs={'en-GB': 'test'},

make_data_migration.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 ./manage.py transfer -a
+./manage.py transfer --setup_clean_db
 ./manage.py transfer -d
 ./manage.py transfer -e
 ./manage.py transfer -n

project/settings/local.py

@@ -29,8 +29,7 @@ MEDIA_ROOT = os.path.join(PUBLIC_ROOT, MEDIA_LOCATION)
 # SORL thumbnails
 THUMBNAIL_DEBUG = True
 
-# ADDED TRANSFER APP
+
-INSTALLED_APPS.append('transfer.apps.TransferConfig')
 
 # DATABASES
 DATABASES = {

requirements/base.txt

@@ -64,6 +64,9 @@ pycountry==19.8.18
 # sql-tree
 django-mptt==0.9.1
 
+# slugify
+python-slugify==4.0.0
+
 # Export to Excel
 XlsxWriter==1.2.6