From c5218393894481ec4d6351d6cde847f43db3d763 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 30 Oct 2019 13:53:12 +0300 Subject: [PATCH] news refactor --- apps/news/transfer_data.py | 34 ++++++--- apps/transfer/models.py | 4 +- apps/transfer/serializers/news.py | 116 ++++++++++++++++-------------- docker-compose.mysql.yml | 2 +- 4 files changed, 91 insertions(+), 65 deletions(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 791b54ce..7524a6cc 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,28 +1,46 @@ from pprint import pprint -from django.db.models import Value, IntegerField, F +from django.db.models import Aggregate, CharField, Value +from django.db.models import IntegerField, F from news.models import NewsType from transfer.models import PageTexts from transfer.serializers.news import NewsSerializer -def transfer_news(): - news_type, _ = NewsType.objects.get_or_create(name="News") +class GroupConcat(Aggregate): + function = 'GROUP_CONCAT' + template = '%(function)s(%(expressions)s)' - queryset = PageTexts.objects.filter(page__type="News").annotate( + def __init__(self, expression, **extra): + output_field = extra.pop('output_field', CharField()) + super().__init__(expression, output_field=output_field, **extra) + + def as_postgresql(self, compiler, connection): + self.function = 'STRING_AGG' + return super().as_sql(compiler, connection) + + +def transfer_news(): + news_type, _ = NewsType.objects.get_or_create(name='News') + queryset = PageTexts.objects.filter( + page__type='News', + ).annotate( news_type=Value(news_type.id, output_field=IntegerField()), country_code=F('page__site__country_code_2'), - ) - queryset = queryset.annotate(template=F('page__template')) + news_title=F('page__root_title'), + image=F('page__attachment_suffix_url'), + template=F('page__template'), + tags=GroupConcat('page__tags__id'), + )[:100] serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + pprint(f'News serializer errors: {serialized_data.errors}') data_types = { - "news": [transfer_news] + 'news': [transfer_news] } diff --git a/apps/transfer/models.py b/apps/transfer/models.py index fbcea0d8..0f7d3348 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -761,6 +761,7 @@ class Pages(MigrateMixin): attachment_content_type = models.CharField(max_length=255, blank=True, null=True) attachment_file_size = models.IntegerField(blank=True, null=True) attachment_updated_at = models.DateTimeField(blank=True, null=True) + attachment_suffix_url = models.TextField(blank=True, null=True) geometries = models.CharField(max_length=1024, blank=True, null=True) scheduled_at = models.DateTimeField(blank=True, null=True) created_at = models.DateTimeField() @@ -785,7 +786,6 @@ class PageTexts(MigrateMixin): locale = models.CharField(max_length=255, blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) - # page_id = models.IntegerField(blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() summary = models.TextField(blank=True, null=True) @@ -814,7 +814,7 @@ class PageMetadata(MigrateMixin): key = models.CharField(max_length=255, blank=True, null=True) value = models.CharField(max_length=255, blank=True, null=True) - page_id = models.IntegerField(blank=True, null=True) + page = models.ForeignKey('Pages', models.DO_NOTHING, blank=True, null=True, related_name='tags') created_at = models.DateTimeField() updated_at = models.DateTimeField() diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 82853cad..cafbe994 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -2,64 +2,67 @@ from rest_framework import serializers from location.models import Country from news.models import News +from tag.models import TagCategory from utils.legacy_parser import parse_legacy_news_content from utils.slug_generator import generate_unique_slug -class NewsSerializer(serializers.ModelSerializer): - locale = serializers.CharField() - slug = serializers.CharField() - body = serializers.CharField(allow_null=True) +class NewsSerializer(serializers.Serializer): + old_id = serializers.IntegerField() + news_type = serializers.IntegerField() + news_title = serializers.CharField() title = serializers.CharField() - template = serializers.CharField() + body = serializers.CharField(allow_null=True) + created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + slug = serializers.CharField() state = serializers.CharField() + template = serializers.CharField() country_code = serializers.CharField(allow_null=True) - created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') - - class Meta: - model = News - fields = ( - 'old_id', - 'created_at', - 'state', - 'template', - 'title', - 'body', - 'slug', - 'news_type', - 'locale', - 'country_code', - ) - - def validate(self, data): - data.update({ - 'slug': generate_unique_slug(News, data['slug']), - 'state': self.get_state(data), - 'template': self.get_template(data), - 'title': self.get_title(data), - 'description': self.get_description(data), - 'subtitle': self.get_description(data), - 'country': self.get_country(data), - }) - data.pop('country_code') - data.pop('body') - data.pop('locale') - return data + locale = serializers.CharField() + image = serializers.CharField() + tags = serializers.CharField(allow_null=True) def create(self, validated_data): - return News.objects.create(**validated_data) - @staticmethod - def get_country(data): - return Country.objects.filter(code__iexact=data['country_code']).first() - - @staticmethod - def get_template(data): - templates = { - 'main': News.MAIN, - 'main.pdf.erb': News.MAIN_PDF_ERB, + payload = { + 'old_id': validated_data['old_id'], + 'news_type': validated_data['news_type'], + 'title': {validated_data['locale']: validated_data['news_title']}, + 'subtitle': {validated_data['locale']: validated_data['title']}, + 'description': self.get_description(validated_data), + 'start': validated_data['created_at'], + 'slug': generate_unique_slug(News, validated_data['slug']), + 'state': self.get_state(validated_data), + 'template': self.get_template(validated_data), + 'country': self.get_country(validated_data), } - return templates.get(data['template'], News.MAIN) + obj = News.objects.create(**payload) + + tags = self.get_tags(validated_data) + + return obj + + # TODO: теги + # TODO: галерея с картинкой + + @staticmethod + def get_tags(data): + if not data['tags']: + return None + tag_cat, _ = TagCategory.objects.get_or_create( + public=True, + label={'en-GB': 'tag'}, + index_name='tag', + country='tag', + ) + + + @staticmethod + def get_description(data): + if data['body']: + content = parse_legacy_news_content(data['body']) + return {data['locale']: content} + return None @staticmethod def get_state(data): @@ -73,12 +76,17 @@ class NewsSerializer(serializers.ModelSerializer): return states.get(data['state'], News.WAITING) @staticmethod - def get_title(data): - return {data['locale']: data['title']} + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + } + return templates.get(data['template'], News.MAIN) @staticmethod - def get_description(data): - content = None - if data['body']: - content = parse_legacy_news_content(data['body']) - return {data['locale']: content} + def get_country(data): + return Country.objects.filter(code__iexact=data['country_code']).first() + + @staticmethod + def get_title(data): + return {data['locale']: data['title']} diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index 213dabe8..4d4e553e 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -51,7 +51,7 @@ services: # Redis redis: - image: redis:2.8.23 + image: redis:alpine networks: - redis_network