Merge branch 'nes_transfer' into develop

apps/news/management/commands/add_author.py

@@ -1,29 +0,0 @@
-from django.core.management.base import BaseCommand
-from django.db.models import F
-from tqdm import tqdm
-
-from account.models import User
-from news.models import News
-from transfer.models import PageTexts
-
-
-class Command(BaseCommand):
-    help = 'Add author of News'
-
-    def handle(self, *args, **kwargs):
-        count = 0
-        news_list = News.objects.filter(created_by__isnull=True)
-
-        for news in tqdm(news_list, desc="Find author for exist news"):
-            old_news = PageTexts.objects.filter(id=news.old_id).annotate(
-                account_id=F('page__account_id'),
-            ).first()
-            if old_news:
-                user = User.objects.filter(old_id=old_news.account_id).first()
-                if user:
-                    news.created_by = user
-                    news.modified_by = user
-                    news.save()
-                    count += 1
-
-        self.stdout.write(self.style.WARNING(f'Update {count} objects.'))

apps/news/management/commands/add_news_tags.py

@@ -1,36 +0,0 @@
-from django.core.management.base import BaseCommand
-
-from news.models import News, NewsType
-from tag.models import Tag, TagCategory
-from transfer.models import PageMetadata, Pages, PageTexts
-
-
-class Command(BaseCommand):
-    help = 'Remove old news from new bd'
-
-    def handle(self, *args, **kwargs):
-        count = 0
-        news_type, _ = NewsType.objects.get_or_create(name='News')
-        tag_cat, _ = TagCategory.objects.get_or_create(index_name='category')
-        news_type.tag_categories.add(tag_cat)
-        news_type.save()
-
-        old_news_tag = PageMetadata.objects.filter(key='category', page__pagetexts__isnull=False)
-        for old_tag in old_news_tag:
-            old_id_list = old_tag.page.pagetexts_set.all().values_list('id', flat=True)
-
-            # Make Tag
-            new_tag, created = Tag.objects.get_or_create(category=tag_cat, value=old_tag.value)
-            if created:
-                text_value = ' '.join(new_tag.value.split('_'))
-                new_tag.label = {'en-GB': text_value}
-                new_tag.save()
-            for id in old_id_list:
-                if isinstance(id, int):
-                    news = News.objects.filter(old_id=id).first()
-                    if news:
-                        news.tags.add(new_tag)
-                        news.save()
-                        count += 1
-
-        self.stdout.write(self.style.WARNING(f'Create or update {count} objects.'))

apps/news/migrations/0049_auto_20191223_0619.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.7 on 2019-12-23 06:19
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0048_remove_news_must_of_the_week'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='news',
+            name='views_count',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='news', to='rating.ViewCount'),
+        ),
+    ]

apps/news/models.py

@@ -105,10 +105,10 @@ class NewsQuerySet(TranslationQuerysetMixin):
         date_now = now.date()
         time_now = now.time()
         return self.exclude(models.Q(publication_date__isnull=True) | models.Q(publication_time__isnull=True)). \
-                    filter(models.Q(models.Q(end__gte=now) |
+            filter(models.Q(models.Q(end__gte=now) |
-                                    models.Q(end__isnull=True)),
+                            models.Q(end__isnull=True)),
-                           state__in=self.model.PUBLISHED_STATES, publication_date__lte=date_now,
+                   state__in=self.model.PUBLISHED_STATES, publication_date__lte=date_now,
-                           publication_time__lte=time_now)
+                   publication_time__lte=time_now)
 
     # todo: filter by best score
     # todo: filter by country?
@@ -215,7 +215,8 @@ class News(GalleryMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixin,
     tags = models.ManyToManyField('tag.Tag', related_name='news',
                                   verbose_name=_('Tags'))
     gallery = models.ManyToManyField('gallery.Image', through='news.NewsGallery')
-    views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL)
+    views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL,
+                                       related_name='news')
     ratings = generic.GenericRelation(Rating)
     favorites = generic.GenericRelation(to='favorites.Favorites')
     carousels = generic.GenericRelation(to='main.Carousel')

apps/news/transfer_data.py

@@ -1,44 +1,61 @@
 from pprint import pprint
 
-from django.db.models import Aggregate, CharField, Value
 from django.db.models import IntegerField, F
+from django.db.models import Value
+from tqdm import tqdm
 
-from news.models import NewsType
+from gallery.models import Image
-from tag.models import TagCategory
+from news.models import NewsType, News
-from transfer.models import PageTexts
+from rating.models import ViewCount
+from tag.models import TagCategory, Tag
+from transfer.models import PageTexts, PageCounters, PageMetadata
 from transfer.serializers.news import NewsSerializer
 
 
-class GroupConcat(Aggregate):
+def add_locale(locale, data):
-    function = 'GROUP_CONCAT'
+    if isinstance(data, dict) and locale not in data:
-    template = '%(function)s(%(expressions)s)'
+        data.update({
+            locale: next(iter(data.values()))
+        })
+    return data
 
-    def __init__(self, expression, **extra):
-        output_field = extra.pop('output_field', CharField())
-        super().__init__(expression, output_field=output_field, **extra)
 
-    def as_postgresql(self, compiler, connection):
+def clear_old_news():
-        self.function = 'STRING_AGG'
+    """
-        return super().as_sql(compiler, connection)
+    Clear lod news and news images
+    """
+    images = Image.objects.filter(
+        news_gallery__isnull=False,
+        news__gallery__news__old_id__isnull=False
+    )
+    img_num = images.count()
+
+    news = News.objects.filter(old_id__isnull=False)
+    news_num = news.count()
+
+    images.delete()
+    news.delete()
+
+    print(f'Deleted {img_num} images')
+    print(f'Deleted {news_num} news')
 
 
 def transfer_news():
     news_type, _ = NewsType.objects.get_or_create(name='News')
-    tag_cat, _ = TagCategory.objects.get_or_create(index_name='tag')
-    news_type.tag_categories.add(tag_cat)
-    news_type.save()
 
     queryset = PageTexts.objects.filter(
         page__type='News',
     ).annotate(
-        tag_cat_id=Value(tag_cat.id, output_field=IntegerField()),
+        page__id=F('page__id'),
         news_type_id=Value(news_type.id, output_field=IntegerField()),
-        country_code=F('page__site__country_code_2'),
+        page__created_at=F('page__created_at'),
-        news_title=F('page__root_title'),
+        page__account_id=F('page__account_id'),
-        image=F('page__attachment_suffix_url'),
+        page__state=F('page__state'),
-        template=F('page__template'),
+        page__template=F('page__template'),
-        tags=GroupConcat('page__tags__id'),
+        page__site__country_code_2=F('page__site__country_code_2'),
-        account_id=F('page__account_id'),
+        page__root_title=F('page__root_title'),
+        page__attachment_suffix_url=F('page__attachment_suffix_url'),
+        page__published_at=F('page__published_at'),
     )
 
     serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
@@ -48,6 +65,102 @@ def transfer_news():
         pprint(f'News serializer errors: {serialized_data.errors}')
 
 
+def update_en_gb_locales():
+    """
+    Update default locales (en-GB)
+    """
+    news = News.objects.filter(old_id__isnull=False)
+
+    update_news = []
+    for news_item in tqdm(news):
+        news_item.slugs = add_locale('en-GB', news_item.slugs)
+        news_item.title = add_locale('en-GB', news_item.title)
+        news_item.locale_to_description_is_active = add_locale('en-GB', news_item.locale_to_description_is_active)
+        news_item.description = add_locale('en-GB', news_item.description)
+        news_item.subtitle = add_locale('en-GB', news_item.subtitle)
+        update_news.append(news_item)
+    News.objects.bulk_update(update_news, [
+        'slugs',
+        'title',
+        'locale_to_description_is_active',
+        'description',
+        'subtitle',
+    ])
+    print(f'Updated {len(update_news)} news locales')
+
+
+def add_views_count():
+    """
+    Add views count to news from page_counters
+    """
+
+    news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
+    counters = PageCounters.objects.filter(page_id__in=list(news))
+
+    update_counters = []
+    for counter in tqdm(counters):
+        news_item = News.objects.filter(old_id=counter.page_id).first()
+        if news_item:
+            obj, _ = ViewCount.objects.update_or_create(
+                news=news_item,
+                defaults={'count': counter.count},
+            )
+            news_item.views_count = obj
+            update_counters.append(news_item)
+    News.objects.bulk_update(update_counters, ['views_count', ])
+    print(f'Updated {len(update_counters)} news counters')
+
+
+def add_tags():
+    """
+    Add news tags
+    """
+
+    news_type, _ = NewsType.objects.get_or_create(name='News')
+    tag_category, _ = TagCategory.objects.get_or_create(index_name='category')
+    tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag')
+    news_type.tag_categories.add(tag_category)
+    news_type.tag_categories.add(tag_tag)
+    news_type.save()
+
+    tag_cat = {
+        'category': tag_category,
+        'tag': tag_tag,
+    }
+
+    news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
+    old_news_tag = PageMetadata.objects.filter(
+        key__in=('category', 'tag'),
+        page_id__in=list(news),
+    )
+
+    count = 0
+    for old_tag in tqdm(old_news_tag):
+        old_id = old_tag.page.id
+        new_tag, created = Tag.objects.get_or_create(
+            category=tag_cat.get(old_tag.key),
+            value=old_tag.value,
+        )
+        if created:
+            text_value = ' '.join(new_tag.value.split('_'))
+            new_tag.label = {'en-GB': text_value}
+            new_tag.save()
+
+        news = News.objects.filter(old_id=old_id).first()
+        if news:
+            news.tags.add(new_tag)
+            news.save()
+            count += 1
+
+    print(f'Updated {count} tags')
+
+
 data_types = {
-    'news': [transfer_news]
+    'news': [
+        clear_old_news,
+        transfer_news,
+        update_en_gb_locales,
+        add_views_count,
+        add_tags,
+    ]
 }

apps/rating/migrations/0005_auto_20191223_0850.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.7 on 2019-12-23 08:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rating', '0004_auto_20191114_2041'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='viewcount',
+            name='count',
+            field=models.PositiveIntegerField(),
+        ),
+    ]

apps/rating/models.py

@@ -23,4 +23,4 @@ class Rating(models.Model):
 
 
 class ViewCount(models.Model):
-    count = models.IntegerField()
+    count = models.PositiveIntegerField()

apps/recipe/models.py

@@ -10,6 +10,7 @@ class RecipeQuerySet(models.QuerySet):
 
     # todo: what records are considered published?
     def published(self):
+        # TODO: проверка по полю published_at
         return self.filter(state__in=[self.model.PUBLISHED,
                                       self.model.PUBLISHED_EXCLUSIVE])
 
@@ -67,3 +68,6 @@ class Recipe(TranslatedFieldsMixin, ImageMixin, BaseAttributes):
 
         verbose_name = _('Recipe')
         verbose_name_plural = _('Recipes')
+
+    # TODO: в save добавить обновление published_at если state в PUBLISHED или PUBLISHED_EXCLUSIVE
+    # TODO: в save добавить обновление published_at в None если state в WAITING или HIDDEN

apps/transfer/serializers/news.py

@@ -1,101 +1,92 @@
 from rest_framework import serializers
 
+from account.models import User
 from gallery.models import Image
 from location.models import Country
 from news.models import News, NewsGallery
-from tag.models import Tag
-from transfer.models import PageMetadata
 from utils.legacy_parser import parse_legacy_news_content
-from utils.slug_generator import generate_unique_slug
-from account.models import User
 
 
 class NewsSerializer(serializers.Serializer):
-    id = serializers.IntegerField()
-    account_id = serializers.IntegerField(allow_null=True)
-    tag_cat_id = serializers.IntegerField()
-    news_type_id = serializers.IntegerField()
-    news_title = serializers.CharField()
-    title = serializers.CharField()
-    summary = serializers.CharField(allow_null=True, allow_blank=True)
-    body = serializers.CharField(allow_null=True)
-    created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
-    slug = serializers.CharField()
-    state = serializers.CharField()
-    template = serializers.CharField()
-    country_code = serializers.CharField(allow_null=True)
     locale = serializers.CharField()
-    image = serializers.CharField()
+    page__id = serializers.IntegerField()
-    tags = serializers.CharField(allow_null=True)
+    news_type_id = serializers.IntegerField()
-
+    page__created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
-    def create(self, validated_data):
+    page__account_id = serializers.IntegerField(allow_null=True)
+    page__state = serializers.CharField()
+    page__template = serializers.CharField()
+    page__site__country_code_2 = serializers.CharField(allow_null=True)
+    slug = serializers.CharField()
+    body = serializers.CharField(allow_null=True)
+    title = serializers.CharField()
+    page__root_title = serializers.CharField()
+    summary = serializers.CharField(allow_null=True, allow_blank=True)
+    page__attachment_suffix_url = serializers.CharField()
+    page__published_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S', allow_null=True)
 
+    def create(self, data):
+        account = self.get_account(data)
         payload = {
-            'old_id': validated_data['id'],
+            'old_id': data['page__id'],
-            'news_type_id': validated_data['news_type_id'],
+            'news_type_id': data['news_type_id'],
-            'title': {validated_data['locale']: validated_data['news_title']},
+            'created': data['page__created_at'],
-            'subtitle': self.get_subtitle(validated_data),
+            'created_by': account,
-            'description': self.get_description(validated_data),
+            'modified_by': account,
-            'start': validated_data['created_at'],
+            'state': self.get_state(data),
-            'slug': generate_unique_slug(News, validated_data['slug']),
+            'template': self.get_template(data),
-            'state': self.get_state(validated_data),
+            'country': self.get_country(data),
-            'template': self.get_template(validated_data),
+            'slugs': {data['locale']: data['slug']},
-            'country': self.get_country(validated_data),
+            'description': self.get_description(data),
-            'created_by': self.get_account(validated_data),
+            'title': {data['locale']: data['title']},
-            'modified_by': self.get_account(validated_data),
+            'backoffice_title': data['page__root_title'],
+            'subtitle': self.get_subtitle(data),
+            'locale_to_description_is_active': {data['locale']: True},
+            'publication_date': self.get_publication_date(data),
+            'publication_time': self.get_publication_time(data),
         }
-        obj = News.objects.create(**payload)
 
-        tags = self.get_tags(validated_data)
+        obj, created = News.objects.get_or_create(
-        for tag in tags:
+            old_id=payload['old_id'],
-            obj.tags.add(tag)
+            defaults=payload,
-        obj.save()
+        )
+        if not created:
+            obj.slugs.update(payload['slugs'])
+            obj.title.update(payload['title'])
+            obj.locale_to_description_is_active.update(payload['locale_to_description_is_active'])
 
-        self.make_gallery(validated_data, obj)
+            if obj.description and payload['description']:
+                obj.description.update(payload['description'])
+            else:
+                obj.description = payload['description']
+
+            if obj.subtitle and payload['subtitle']:
+                obj.subtitle.update(payload['subtitle'])
+            else:
+                obj.subtitle = payload['subtitle']
+
+            obj.save()
+
+        self.make_gallery(data, obj)
         return obj
 
     @staticmethod
-    def make_gallery(data, obj):
+    def get_publication_date(data):
-        if not data['image'] or data['image'] == 'default/missing.png':
+        published_at = data.get('page__published_at')
-            return
+        if published_at:
-
+            return published_at.date()
-        img = Image.objects.create(
-            image=data['image'],
-            title=data['news_title'],
-        )
-        NewsGallery.objects.create(
-            news=obj,
-            image=img,
-            is_main=True,
-        )
-
-    @staticmethod
-    def get_tags(data):
-        results = []
-        if not data['tags']:
-            return results
-
-        meta_ids = (int(_id) for _id in data['tags'].split(','))
-        tags = PageMetadata.objects.filter(
-            id__in=meta_ids,
-            key='tag',
-            value__isnull=False,
-        )
-        for old_tag in tags:
-            tag, _ = Tag.objects.get_or_create(
-                category_id=data['tag_cat_id'],
-                label={data['locale']: old_tag.value},
-            )
-            results.append(tag)
-        return results
-
-    @staticmethod
-    def get_description(data):
-        if data['body']:
-            content = parse_legacy_news_content(data['body'])
-            return {data['locale']: content}
         return None
 
+    @staticmethod
+    def get_publication_time(data):
+        published_at = data.get('page__published_at')
+        if published_at:
+            return published_at.time()
+        return None
+
+    @staticmethod
+    def get_account(data):
+        return User.objects.filter(old_id=data['page__account_id']).first()
+
     @staticmethod
     def get_state(data):
         states = {
@@ -105,33 +96,47 @@ class NewsSerializer(serializers.Serializer):
             'published_exclusive': News.PUBLISHED_EXCLUSIVE,
             'scheduled_exclusively': News.WAITING,
         }
-        return states.get(data['state'], News.WAITING)
+        return states.get(data['page__state'], News.WAITING)
 
     @staticmethod
     def get_template(data):
         templates = {
             'main': News.MAIN,
             'main.pdf.erb': News.MAIN_PDF_ERB,
+            'newspaper': News.NEWSPAPER,
         }
-        return templates.get(data['template'], News.MAIN)
+        return templates.get(data['page__template'], News.MAIN)
 
     @staticmethod
     def get_country(data):
-        return Country.objects.filter(code__iexact=data['country_code']).first()
+        return Country.objects.filter(code__iexact=data['page__site__country_code_2']).first()
 
     @staticmethod
-    def get_title(data):
+    def get_description(data):
-        return {data['locale']: data['title']}
+        if data['body']:
+            content = parse_legacy_news_content(data['body'])
+            return {data['locale']: content}
+        return None
 
     @staticmethod
     def get_subtitle(data):
         if data.get('summary'):
-            content = {data['locale']: data['summary']}
+            return {data['locale']: data['summary']}
-        else:
+        return None
-            content = {data['locale']: data['title']}
-        return content
 
     @staticmethod
-    def get_account(data):
+    def make_gallery(data, obj):
-        """Get account"""
+        if not data['page__attachment_suffix_url'] or data['page__attachment_suffix_url'] == 'default/missing.png':
-        return User.objects.filter(old_id=data['account_id']).first()
+            return
+
+        img, _ = Image.objects.get_or_create(
+            image=data['page__attachment_suffix_url'],
+            title=data['page__root_title'],
+            created=data['page__created_at']
+        )
+
+        gal, _ = NewsGallery.objects.get_or_create(
+            news=obj,
+            image=img,
+            is_main=True,
+        )

docker-compose.mysql.yml

@@ -13,7 +13,6 @@ services:
       MYSQL_ROOT_PASSWORD: rootPassword
     volumes:
       - gm-mysql_db:/var/lib/mysql
-      - .:/code
 
 
   # PostgreSQL database
@@ -30,7 +29,6 @@ services:
       - "5436:5432"
     volumes:
       - gm-db:/var/lib/postgresql/data/
-      - .:/code
 
 
   elasticsearch: