migrate from legacy news

apps/news/transfer_data.py

@@ -1,23 +1,23 @@
-from transfer.serializers.news import NewsSerializer
-from transfer.models import PageTexts
-from news.models import NewsType
 from django.db.models import Value, IntegerField, F
-from pprint import pprint
+
+from news.models import NewsType
+from transfer.models import PageTexts
+from transfer.serializers.news import NewsSerializer
 
 
 def transfer_news():
     news_type, _ = NewsType.objects.get_or_create(name="News")
 
-    queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()),
+    queryset = PageTexts.objects.filter(page__type="News").annotate(
-                                                                    playlist=Value(1, output_field=IntegerField()))
+        news_type=Value(news_type.id, output_field=IntegerField()),
-    queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name'))
+    )
     queryset = queryset.annotate(template=F('page__template'))
 
     serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
     if serialized_data.is_valid():
         serialized_data.save()
     else:
-        pprint(f"News serializer errors: {serialized_data.errors}")
+        print(f"News serializer errors: {serialized_data.errors}")
 
 
 data_types = {

apps/transfer/models.py

@@ -694,6 +694,7 @@ class ReviewTexts(MigrateMixin):
         managed = False
         db_table = 'review_texts'
 
+
 class Comments(MigrateMixin):
     using = 'legacy'
 

apps/transfer/serializers/news.py

@@ -1,123 +1,68 @@
 from rest_framework import serializers
+
 from news.models import News
-from uuid import uuid4
+from utils.legacy_parser import parse_legacy_content
-import datetime
-from django.conf import settings
 
 
 class NewsSerializer(serializers.ModelSerializer):
-    # image_url = serializers.CharField()
+    locale = serializers.CharField()
-    # preview_image_url = serializers.CharField()
-    attachment_file_name = serializers.CharField(allow_null=True)
-    # body = serializers.CharField(allow_null=True, source="description")
     body = serializers.CharField(allow_null=True)
     title = serializers.CharField()
     template = serializers.CharField()
     state = serializers.CharField()
-    created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S')
+    created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S')
 
     class Meta:
         model = News
         fields = (
-            "created_at",
+            'created_at',
-            "state",
+            'state',
-            "template",
+            'template',
-            # "image_url",
+            'title',
-            # "preview_image_url",
+            'body',
-            "attachment_file_name",
+            'slug',
-            "title",
+            'news_type',
-            "body",
+            'locale',
-            "slug",
-            "news_type",
-            "playlist"
         )
 
     def validate(self, data):
-        data["state"] = self.get_state(data)
+        data.update({
-        data["template"] = self.get_template(data)
+            'state': self.get_state(data),
-        data["title"] = self.get_title(data)
+            'template': self.get_template(data),
-        data["description"] = self.get_description(data)
+            'title': self.get_title(data),
-        data["image_url"] = self.get_image_url(data)
+            'description': self.get_description(data),
-        data["preview_image_url"] = self.get_preview_image_url(data)
+        })
-        data.pop("attachment_file_name")
+        data.pop('body')
-        data.pop("body")
+        data.pop('locale')
         return data
 
     def create(self, validated_data):
-        # validated_data['state'] = self.get_state(validated_data)
         return News.objects.create(**validated_data)
 
-    def get_template(self, obj):
+    @staticmethod
-        if obj["template"] == "main":
+    def get_template(data):
-            return News.MAIN
+        templates = {
-        elif obj["template"] == "main.pdf.erb":
+            'main': News.MAIN,
-            return News.MAIN_PDF_ERB
+            'main.pdf.erb': News.MAIN_PDF_ERB,
-        else:
+        }
-            return News.NEWSPAPER
+        return templates.get(data['template'], News.MAIN)
 
-        # if obj.template == "main":
+    @staticmethod
-        #     return News.MAIN
+    def get_state(data):
-        # elif obj.template == "main.pdf.erb":
+        states = {
-        #     return News.MAIN_PDF_ERB
+            'new': News.PUBLISHED,
-        # else:
+            'published': News.PUBLISHED,
-        #     return News.NEWSPAPER
+            'hidden': News.HIDDEN,
+        }
+        return states.get(data['state'], News.WAITING)
 
-    def get_state(self, obj):
+    @staticmethod
-        if obj["state"] == "published":
+    def get_title(data):
-            return News.PUBLISHED
+        return {data['locale']: data['title']}
-        elif obj["state"] == "hidden":
-            return News.HIDDEN
-        elif obj["state"] == "published_exclusive":
-            return News.PUBLISHED_EXCLUSIVE
-        else:
-            return News.WAITING
 
-        # if obj.state == "published":
+    @staticmethod
-        #     return News.PUBLISHED
+    def get_description(data):
-        # elif obj.state == "hidden":
+        content = None
-        #     return News.HIDDEN
+        if data['body']:
-        # elif obj.state == "published_exclusive":
+            content = parse_legacy_content(data['body'])
-        #     return News.PUBLISHED_EXCLUSIVE
+        return {data['locale']: content}
-        # else:
-        #     return News.WAITING
-
-    # django.db.utils.IntegrityError: "start" NOT NULL
-    # def get_start(self, obj):
-    #     # return obj.created_at
-    #     return datetime.datetime.now().isoformat()
-
-    def get_title(self, obj):
-        tit = obj.get("title")
-        return {"en-GB": tit}
-        # return f'{"en-GB": {obj.title}}'
-
-    def get_description(self, obj):
-        # return {"en-GB": {obj["body"]}}
-        desc = obj.get("body")
-        return {"en-GB": desc}
-        # if not hasattr(obj, "body"):
-        #     return ""
-        # else:
-        #     return obj.body
-
-    def get_image_url(self, obj):
-        if not obj.get("attachment_file_name"):
-            return ""
-
-        if len(obj.get("attachment_file_name"))>200:
-            return obj.get("attachment_file_name")[:200]
-        return obj.get("attachment_file_name")
-        # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None:
-        #     return ""
-        # return obj.attachment_file_name
-
-    def get_preview_image_url(self, obj):
-        if not obj.get("attachment_file_name"):
-            return ""
-
-        if len(obj.get("attachment_file_name"))>200:
-            return obj.get("attachment_file_name")[:200]
-        return obj.get("attachment_file_name")
-        # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None:
-        #     return ""
-        # return obj.attachment_file_name

apps/utils/legacy_parser.py

@@ -0,0 +1,12 @@
+import yaml
+
+
+def parse_legacy_content(legacy_content):
+    clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess'
+    content_dict = yaml.safe_load(legacy_content.replace(clear_str, ''))
+    result = ''
+    try:
+        result = content_dict['news_content']['value']
+    except IndexError:
+        pass
+    return result

docker-compose.mysql.yml

@@ -14,6 +14,8 @@ services:
     volumes:
       - .:/code
       - gm-mysql_db:/var/lib/mysql
+    networks:
+      - mysql_network
 
   # PostgreSQL database
   db:
@@ -29,6 +31,8 @@ services:
       - "5436:5432"
     volumes:
       - gm-db:/var/lib/postgresql/data/
+    networks:
+      - database_network
 
   elasticsearch:
     image: elasticsearch:7.3.1
@@ -42,6 +46,8 @@ services:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
       - discovery.type=single-node
       - xpack.security.enabled=false
+    networks:
+      - elasticsearch_network
 
   # Redis
   redis:
@@ -113,6 +119,10 @@ services:
       - gm-media:/media-data
     ports:
       - "8000:8000"
+    networks:
+      - database_network
+      - mysql_network
+      - elasticsearch_network
 
 volumes:
   gm-mysql_db:
@@ -125,3 +135,11 @@ volumes:
     name: gm-media
 
   gm-esdata:
+
+networks:
+  database_network:
+    driver: bridge
+  mysql_network:
+    driver: bridge
+  elasticsearch_network:
+    driver: bridge

project/settings/base.py

@@ -160,7 +160,7 @@ DATABASES = {
     'legacy': {
         'ENGINE': 'django.db.backends.mysql',
         # 'HOST': '172.23.0.1',
-        'HOST': '127.0.0.1',
+        'HOST': 'mysql_db',
         'PORT': 3306,
         'NAME': 'dev',
         'USER': 'dev',

project/settings/development.py

@@ -19,7 +19,7 @@ DOMAIN_URI = 'gm.id-east.ru'
 # ELASTICSEARCH SETTINGS
 ELASTICSEARCH_DSL = {
     'default': {
-        'hosts': 'localhost:9200'
+        'hosts': 'elasticsearch:9200'
     }
 }
 

requirements/base.txt

@@ -43,6 +43,7 @@ sorl-thumbnail==12.5.0
 
 
 mysqlclient==1.4.4
+PyYAML==5.1.2
 
 # temp solution
 redis==3.2.0