news refactor

apps/news/transfer_data.py

@@ -1,28 +1,46 @@
 from pprint import pprint
 
-from django.db.models import Value, IntegerField, F
+from django.db.models import Aggregate, CharField, Value
+from django.db.models import IntegerField, F
 
 from news.models import NewsType
 from transfer.models import PageTexts
 from transfer.serializers.news import NewsSerializer
 
 
-def transfer_news():
+class GroupConcat(Aggregate):
-    news_type, _ = NewsType.objects.get_or_create(name="News")
+    function = 'GROUP_CONCAT'
+    template = '%(function)s(%(expressions)s)'
 
-    queryset = PageTexts.objects.filter(page__type="News").annotate(
+    def __init__(self, expression, **extra):
+        output_field = extra.pop('output_field', CharField())
+        super().__init__(expression, output_field=output_field, **extra)
+
+    def as_postgresql(self, compiler, connection):
+        self.function = 'STRING_AGG'
+        return super().as_sql(compiler, connection)
+
+
+def transfer_news():
+    news_type, _ = NewsType.objects.get_or_create(name='News')
+    queryset = PageTexts.objects.filter(
+        page__type='News',
+    ).annotate(
         news_type=Value(news_type.id, output_field=IntegerField()),
         country_code=F('page__site__country_code_2'),
-    )
+        news_title=F('page__root_title'),
-    queryset = queryset.annotate(template=F('page__template'))
+        image=F('page__attachment_suffix_url'),
+        template=F('page__template'),
+        tags=GroupConcat('page__tags__id'),
+    )[:100]
 
     serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
     if serialized_data.is_valid():
         serialized_data.save()
     else:
-        pprint(f"News serializer errors: {serialized_data.errors}")
+        pprint(f'News serializer errors: {serialized_data.errors}')
 
 
 data_types = {
-    "news": [transfer_news]
+    'news': [transfer_news]
 }

apps/transfer/models.py

@@ -761,6 +761,7 @@ class Pages(MigrateMixin):
     attachment_content_type = models.CharField(max_length=255, blank=True, null=True)
     attachment_file_size = models.IntegerField(blank=True, null=True)
     attachment_updated_at = models.DateTimeField(blank=True, null=True)
+    attachment_suffix_url = models.TextField(blank=True, null=True)
     geometries = models.CharField(max_length=1024, blank=True, null=True)
     scheduled_at = models.DateTimeField(blank=True, null=True)
     created_at = models.DateTimeField()
@@ -785,7 +786,6 @@ class PageTexts(MigrateMixin):
     locale = models.CharField(max_length=255, blank=True, null=True)
     state = models.CharField(max_length=255, blank=True, null=True)
     page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True)
-    # page_id = models.IntegerField(blank=True, null=True)
     created_at = models.DateTimeField()
     updated_at = models.DateTimeField()
     summary = models.TextField(blank=True, null=True)
@@ -814,7 +814,7 @@ class PageMetadata(MigrateMixin):
 
     key = models.CharField(max_length=255, blank=True, null=True)
     value = models.CharField(max_length=255, blank=True, null=True)
-    page_id = models.IntegerField(blank=True, null=True)
+    page = models.ForeignKey('Pages', models.DO_NOTHING, blank=True, null=True, related_name='tags')
     created_at = models.DateTimeField()
     updated_at = models.DateTimeField()
 

apps/transfer/serializers/news.py

@@ -2,64 +2,67 @@ from rest_framework import serializers
 
 from location.models import Country
 from news.models import News
+from tag.models import TagCategory
 from utils.legacy_parser import parse_legacy_news_content
 from utils.slug_generator import generate_unique_slug
 
 
-class NewsSerializer(serializers.ModelSerializer):
+class NewsSerializer(serializers.Serializer):
-    locale = serializers.CharField()
+    old_id = serializers.IntegerField()
-    slug = serializers.CharField()
+    news_type = serializers.IntegerField()
-    body = serializers.CharField(allow_null=True)
+    news_title = serializers.CharField()
     title = serializers.CharField()
-    template = serializers.CharField()
+    body = serializers.CharField(allow_null=True)
+    created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
+    slug = serializers.CharField()
     state = serializers.CharField()
+    template = serializers.CharField()
     country_code = serializers.CharField(allow_null=True)
-    created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S')
+    locale = serializers.CharField()
-
+    image = serializers.CharField()
-    class Meta:
+    tags = serializers.CharField(allow_null=True)
-        model = News
-        fields = (
-            'old_id',
-            'created_at',
-            'state',
-            'template',
-            'title',
-            'body',
-            'slug',
-            'news_type',
-            'locale',
-            'country_code',
-        )
-
-    def validate(self, data):
-        data.update({
-            'slug': generate_unique_slug(News, data['slug']),
-            'state': self.get_state(data),
-            'template': self.get_template(data),
-            'title': self.get_title(data),
-            'description': self.get_description(data),
-            'subtitle': self.get_description(data),
-            'country': self.get_country(data),
-        })
-        data.pop('country_code')
-        data.pop('body')
-        data.pop('locale')
-        return data
 
     def create(self, validated_data):
-        return News.objects.create(**validated_data)
 
-    @staticmethod
+        payload = {
-    def get_country(data):
+            'old_id': validated_data['old_id'],
-        return Country.objects.filter(code__iexact=data['country_code']).first()
+            'news_type': validated_data['news_type'],
-
+            'title': {validated_data['locale']: validated_data['news_title']},
-    @staticmethod
+            'subtitle': {validated_data['locale']: validated_data['title']},
-    def get_template(data):
+            'description': self.get_description(validated_data),
-        templates = {
+            'start': validated_data['created_at'],
-            'main': News.MAIN,
+            'slug': generate_unique_slug(News, validated_data['slug']),
-            'main.pdf.erb': News.MAIN_PDF_ERB,
+            'state': self.get_state(validated_data),
+            'template': self.get_template(validated_data),
+            'country': self.get_country(validated_data),
         }
-        return templates.get(data['template'], News.MAIN)
+        obj = News.objects.create(**payload)
+
+        tags = self.get_tags(validated_data)
+
+        return obj
+
+    # TODO: теги
+    # TODO: галерея с картинкой
+
+    @staticmethod
+    def get_tags(data):
+        if not data['tags']:
+            return None
+        tag_cat, _ = TagCategory.objects.get_or_create(
+            public=True,
+            label={'en-GB': 'tag'},
+            index_name='tag',
+            country='tag',
+        )
+
+
+    @staticmethod
+    def get_description(data):
+        if data['body']:
+            content = parse_legacy_news_content(data['body'])
+            return {data['locale']: content}
+        return None
 
     @staticmethod
     def get_state(data):
@@ -73,12 +76,17 @@ class NewsSerializer(serializers.ModelSerializer):
         return states.get(data['state'], News.WAITING)
 
     @staticmethod
-    def get_title(data):
+    def get_template(data):
-        return {data['locale']: data['title']}
+        templates = {
+            'main': News.MAIN,
+            'main.pdf.erb': News.MAIN_PDF_ERB,
+        }
+        return templates.get(data['template'], News.MAIN)
 
     @staticmethod
-    def get_description(data):
+    def get_country(data):
-        content = None
+        return Country.objects.filter(code__iexact=data['country_code']).first()
-        if data['body']:
+
-            content = parse_legacy_news_content(data['body'])
+    @staticmethod
-        return {data['locale']: content}
+    def get_title(data):
+        return {data['locale']: data['title']}

docker-compose.mysql.yml

@@ -51,7 +51,7 @@ services:
 
   # Redis
   redis:
-    image: redis:2.8.23
+    image: redis:alpine
     networks:
       - redis_network