Merge branch 'origin/feature/migration' into migrate-location-app

# Conflicts:
#	apps/gallery/models.py
#	apps/location/transfer_data.py
#	apps/transfer/management/commands/transfer.py
#	project/settings/base.py
#	requirements/base.txt

apps/advertisement/migrations/0004_auto_20191025_0903.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('advertisement', '0003_auto_20190919_1344'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='advertisement',
+            name='block_level',
+            field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'),
+        ),
+    ]

apps/advertisement/models.py

@@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin):
     url = models.URLField(verbose_name=_('Ad URL'))
     width = models.PositiveIntegerField(verbose_name=_('Block width'))
     height = models.PositiveIntegerField(verbose_name=_('Block height'))
-    block_level = models.CharField(verbose_name=_('Block level'), max_length=10)
+    block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True)
     target_languages = models.ManyToManyField(Language)
 
     class Meta:

apps/advertisement/transfer_data.py

@@ -0,0 +1,20 @@
+from pprint import pprint
+from django.db.models import Value, IntegerField, F
+from transfer.models import Ads
+from transfer.serializers.advertisement import AdvertisementSerializer
+
+
+def transfer_advertisement():
+    queryset = Ads.objects.filter(href__isnull=False)
+
+    serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True)
+
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"News serializer errors: {serialized_data.errors}")
+
+
+data_types = {
+    "commercial": [transfer_advertisement]
+}

apps/gallery/migrations/0004_merge_20191025_0906.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:06
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0002_auto_20191023_1207'),
+        ('gallery', '0003_auto_20191003_1228'),
+    ]
+
+    operations = [
+    ]

apps/gallery/models.py

@@ -27,6 +27,7 @@ class Image(ProjectBaseMixin, SORLImageMixin, PlatformMixin):
                                                    blank=True, null=True, default=None,
                                                    verbose_name=_('image orientation'))
     title = models.CharField(_('title'), max_length=255, default='')
+
     objects = ImageQuerySet.as_manager()
 
     class Meta:

apps/news/migrations/0029_merge_20191025_0906.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 09:06
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0028_auto_20191024_1649'),
+        ('news', '0021_auto_20191021_1120'),
+    ]
+
+    operations = [
+    ]

apps/news/transfer_data.py

@@ -1,16 +1,18 @@
-from transfer.serializers.news import NewsSerializer
-from transfer.models import PageTexts
-from news.models import NewsType
-from django.db.models import Value, IntegerField, F
 from pprint import pprint
 
+from django.db.models import Value, IntegerField, F
+
+from news.models import NewsType
+from transfer.models import PageTexts
+from transfer.serializers.news import NewsSerializer
+
 
 def transfer_news():
     news_type, _ = NewsType.objects.get_or_create(name="News")
 
-    queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()),
+    queryset = PageTexts.objects.filter(page__type="News").annotate(
-                                                                    playlist=Value(1, output_field=IntegerField()))
+        news_type=Value(news_type.id, output_field=IntegerField()),
-    queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name'))
+    )
     queryset = queryset.annotate(template=F('page__template'))
 
     serialized_data = NewsSerializer(data=list(queryset.values()), many=True)

apps/transfer/management/commands/transfer.py

@@ -16,7 +16,7 @@ class Command(BaseCommand):
         'recipe',
         'partner',
         'gallery',
-
+        'commercial',
         'tmp'
     ]
 

apps/transfer/models.py

@@ -694,6 +694,7 @@ class ReviewTexts(MigrateMixin):
         managed = False
         db_table = 'review_texts'
 
+
 class Comments(MigrateMixin):
     using = 'legacy'
 
@@ -787,3 +788,23 @@ class PageMetadata(MigrateMixin):
     class Meta:
         managed = False
         db_table = 'page_metadata'
+
+
+class Ads(MigrateMixin):
+    using = 'legacy'
+
+    site_id = models.IntegerField(blank=True, null=True)
+    href = models.CharField(max_length=255, blank=True, null=True)
+    start_at = models.DateTimeField(blank=True, null=True)
+    expire_at = models.DateTimeField(blank=True, null=True)
+    attachment_file_name = models.CharField(max_length=255, blank=True, null=True)
+    attachment_content_type = models.CharField(max_length=255, blank=True, null=True)
+    attachment_file_size = models.IntegerField(blank=True, null=True)
+    attachment_updated_at = models.DateTimeField(blank=True, null=True)
+    geometries = models.CharField(max_length=1024, blank=True, null=True)
+    created_at = models.DateTimeField()
+    updated_at = models.DateTimeField()
+
+    class Meta:
+        managed = False
+        db_table = 'ads'

apps/transfer/serializers/advertisement.py

@@ -0,0 +1,39 @@
+from rest_framework import serializers
+from advertisement.models import Advertisement
+import yaml
+
+
+class AdvertisementSerializer(serializers.ModelSerializer):
+    href = serializers.CharField()
+    geometries = serializers.CharField(max_length=1024)
+
+    class Meta:
+        model = Advertisement
+        fields = (
+            "href",
+            "geometries"
+        )
+
+    def validate(self, data):
+        data["url"] = data["href"]
+        data["width"] = self.get_width(data["geometries"])
+        data["height"] = self.get_height(data["geometries"])
+        data.pop("href")
+        data.pop("geometries")
+        return data
+
+    def create(self, validated_data):
+        return Advertisement.objects.create(**validated_data)
+
+    def get_width(self, data):
+        data = self.parse_geometries(data)
+        return int(float(data["width"]))
+
+    def get_height(self, data):
+        data = self.parse_geometries(data)
+        return int(float(data["height"]))
+
+    def parse_geometries(self, geo_str):
+        clear_str = "!ruby/object:Paperclip::Geometry"
+        content_dict = yaml.safe_load(geo_str.replace(clear_str, ''))
+        return content_dict[':original']

apps/transfer/serializers/news.py

@@ -1,123 +1,70 @@
 from rest_framework import serializers
+
 from news.models import News
-from uuid import uuid4
+from utils.legacy_parser import parse_legacy_content
-import datetime
-from django.conf import settings
 
 
 class NewsSerializer(serializers.ModelSerializer):
-    # image_url = serializers.CharField()
+    locale = serializers.CharField()
-    # preview_image_url = serializers.CharField()
-    attachment_file_name = serializers.CharField(allow_null=True)
-    # body = serializers.CharField(allow_null=True, source="description")
     body = serializers.CharField(allow_null=True)
     title = serializers.CharField()
     template = serializers.CharField()
     state = serializers.CharField()
-    created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S')
+    created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S')
 
     class Meta:
         model = News
         fields = (
-            "created_at",
+            'created_at',
-            "state",
+            'state',
-            "template",
+            'template',
-            # "image_url",
+            'title',
-            # "preview_image_url",
+            'body',
-            "attachment_file_name",
+            'slug',
-            "title",
+            'news_type',
-            "body",
+            'locale',
-            "slug",
-            "news_type",
-            "playlist"
         )
 
     def validate(self, data):
-        data["state"] = self.get_state(data)
+        data.update({
-        data["template"] = self.get_template(data)
+            'state': self.get_state(data),
-        data["title"] = self.get_title(data)
+            'template': self.get_template(data),
-        data["description"] = self.get_description(data)
+            'title': self.get_title(data),
-        data["image_url"] = self.get_image_url(data)
+            'description': self.get_description(data),
-        data["preview_image_url"] = self.get_preview_image_url(data)
+        })
-        data.pop("attachment_file_name")
+        data.pop('body')
-        data.pop("body")
+        data.pop('locale')
         return data
 
     def create(self, validated_data):
-        # validated_data['state'] = self.get_state(validated_data)
         return News.objects.create(**validated_data)
 
-    def get_template(self, obj):
+    @staticmethod
-        if obj["template"] == "main":
+    def get_template(data):
-            return News.MAIN
+        templates = {
-        elif obj["template"] == "main.pdf.erb":
+            'main': News.MAIN,
-            return News.MAIN_PDF_ERB
+            'main.pdf.erb': News.MAIN_PDF_ERB,
-        else:
+        }
-            return News.NEWSPAPER
+        return templates.get(data['template'], News.MAIN)
 
-        # if obj.template == "main":
+    @staticmethod
-        #     return News.MAIN
+    def get_state(data):
-        # elif obj.template == "main.pdf.erb":
+        states = {
-        #     return News.MAIN_PDF_ERB
+            'new': News.WAITING,
-        # else:
+            'published': News.PUBLISHED,
-        #     return News.NEWSPAPER
+            'hidden': News.HIDDEN,
+            'published_exclusive': News.PUBLISHED_EXCLUSIVE,
+            'scheduled_exclusively': News.WAITING,
+        }
+        return states.get(data['state'], News.WAITING)
 
-    def get_state(self, obj):
+    @staticmethod
-        if obj["state"] == "published":
+    def get_title(data):
-            return News.PUBLISHED
+        return {data['locale']: data['title']}
-        elif obj["state"] == "hidden":
-            return News.HIDDEN
-        elif obj["state"] == "published_exclusive":
-            return News.PUBLISHED_EXCLUSIVE
-        else:
-            return News.WAITING
 
-        # if obj.state == "published":
+    @staticmethod
-        #     return News.PUBLISHED
+    def get_description(data):
-        # elif obj.state == "hidden":
+        content = None
-        #     return News.HIDDEN
+        if data['body']:
-        # elif obj.state == "published_exclusive":
+            content = parse_legacy_content(data['body'])
-        #     return News.PUBLISHED_EXCLUSIVE
+        return {data['locale']: content}
-        # else:
-        #     return News.WAITING
-
-    # django.db.utils.IntegrityError: "start" NOT NULL
-    # def get_start(self, obj):
-    #     # return obj.created_at
-    #     return datetime.datetime.now().isoformat()
-
-    def get_title(self, obj):
-        tit = obj.get("title")
-        return {"en-GB": tit}
-        # return f'{"en-GB": {obj.title}}'
-
-    def get_description(self, obj):
-        # return {"en-GB": {obj["body"]}}
-        desc = obj.get("body")
-        return {"en-GB": desc}
-        # if not hasattr(obj, "body"):
-        #     return ""
-        # else:
-        #     return obj.body
-
-    def get_image_url(self, obj):
-        if not obj.get("attachment_file_name"):
-            return ""
-
-        if len(obj.get("attachment_file_name"))>200:
-            return obj.get("attachment_file_name")[:200]
-        return obj.get("attachment_file_name")
-        # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None:
-        #     return ""
-        # return obj.attachment_file_name
-
-    def get_preview_image_url(self, obj):
-        if not obj.get("attachment_file_name"):
-            return ""
-
-        if len(obj.get("attachment_file_name"))>200:
-            return obj.get("attachment_file_name")[:200]
-        return obj.get("attachment_file_name")
-        # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None:
-        #     return ""
-        # return obj.attachment_file_name

apps/transfer/serializers/recipe.py

@@ -1,8 +1,10 @@
 from rest_framework import serializers
 from recipe.models import Recipe
+from utils.legacy_parser import parse_legacy_content
 
 
 class RecipeSerializer(serializers.ModelSerializer):
+    locale = serializers.CharField()
     body = serializers.CharField(allow_null=True)
     title = serializers.CharField()
     state = serializers.CharField()
@@ -14,7 +16,8 @@ class RecipeSerializer(serializers.ModelSerializer):
             "body",
             "title",
             "state",
-            "created_at"
+            "created_at",
+            'locale',
         )
 
     def validate(self, data):
@@ -22,6 +25,7 @@ class RecipeSerializer(serializers.ModelSerializer):
         data["title"] = self.get_title(data)
         data["description"] = self.get_description(data)
         data.pop("body")
+        data.pop("locale")
         return data
 
     def create(self, validated_data):
@@ -38,9 +42,14 @@ class RecipeSerializer(serializers.ModelSerializer):
             return Recipe.WAITING
 
     def get_title(self, obj):
-        tit = obj.get("title")
+        # tit = obj.get("title")
-        return {"en-GB": tit}
+        # return {"en-GB": tit}
+        return {obj['locale']: obj['title']}
 
     def get_description(self, obj):
-        desc = obj.get("body")
+        # desc = obj.get("body")
-        return {"en-GB": desc}
+        # return {"en-GB": desc}
+        content = None
+        if obj['body']:
+            content = parse_legacy_content(obj['body'])
+        return {obj['locale']: content}

apps/utils/legacy_parser.py

@@ -0,0 +1,12 @@
+import yaml
+
+
+def parse_legacy_content(legacy_content):
+    clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess'
+    content_dict = yaml.safe_load(legacy_content.replace(clear_str, ''))
+    result = ''
+    try:
+        result = content_dict['news_content']['value']
+    except KeyError:
+        pass
+    return result

docker-compose.mysql.yml

@@ -14,6 +14,8 @@ services:
     volumes:
       - .:/code
       - gm-mysql_db:/var/lib/mysql
+    networks:
+      - mysql_network
 
   # PostgreSQL database
   db:
@@ -29,6 +31,8 @@ services:
       - "5436:5432"
     volumes:
       - gm-db:/var/lib/postgresql/data/
+    networks:
+      - database_network
 
   elasticsearch:
     image: elasticsearch:7.3.1
@@ -42,6 +46,8 @@ services:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
       - discovery.type=single-node
       - xpack.security.enabled=false
+    networks:
+      - elasticsearch_network
 
   # Redis
   redis:
@@ -113,6 +119,10 @@ services:
       - gm-media:/media-data
     ports:
       - "8000:8000"
+    networks:
+      - database_network
+      - mysql_network
+      - elasticsearch_network
 
 volumes:
   gm-mysql_db:
@@ -125,3 +135,11 @@ volumes:
     name: gm-media
 
   gm-esdata:
+
+networks:
+  database_network:
+    driver: bridge
+  mysql_network:
+    driver: bridge
+  elasticsearch_network:
+    driver: bridge

project/settings/base.py

@@ -159,8 +159,9 @@ DATABASES = {
     },
     'legacy': {
         'ENGINE': 'django.db.backends.mysql',
-        'HOST': '172.20.0.1',
+        'HOST': '172.17.0.1',
-        # 'HOST': '127.0.0.1',
+        # 'HOST': '172.23.0.1',
+        # 'HOST': 'mysql_db',
         'PORT': 3306,
         'NAME': 'dev',
         'USER': 'dev',

project/settings/development.py

@@ -20,6 +20,7 @@ DOMAIN_URI = 'gm.id-east.ru'
 ELASTICSEARCH_DSL = {
     'default': {
         'hosts': 'localhost:9200'
+        # 'hosts': 'elasticsearch:9200'
     }
 }
 

requirements/base.txt

@@ -9,7 +9,7 @@ fcm-django
 django-easy-select2
 bootstrap-admin
 drf-yasg==1.16.0
-timezonefinder==4.1.0
+timezonefinder
 PySocks!=1.5.7,>=1.5.6;
 
 djangorestframework==3.9.4
@@ -41,9 +41,11 @@ django-storages==1.7.2
 
 sorl-thumbnail==12.5.0
 
+
+mysqlclient==1.4.4
+PyYAML==5.1.2
+
 # temp solution
 redis==3.2.0
 amqp>=2.4.0
 celery==4.3.0rc2
-
-mysqlclient==1.4.4