gault-millau/apps/news/transfer_data.py

from pprint import pprint

from django.db.models import IntegerField, F
from django.db.models import Value
from tqdm import tqdm

from gallery.models import Image
from news.models import NewsType, News
from rating.models import ViewCount
from tag.models import TagCategory, Tag
from translation.models import SiteInterfaceDictionary
from transfer.models import PageTexts, PageCounters, PageMetadata
from transfer.serializers.news import NewsSerializer
from utils.methods import transform_camelcase_to_underscore


def add_locale(locale, data):
    if isinstance(data, dict) and locale not in data:
        data.update({
            locale: next(iter(data.values()))
        })
    return data


def clear_old_news():
    """
    Clear lod news and news images
    """
    images = Image.objects.filter(
        news_gallery__isnull=False,
        news__gallery__news__old_id__isnull=False
    )
    img_num = images.count()

    news = News.objects.filter(old_id__isnull=False)
    news_num = news.count()

    images.delete()
    news.delete()

    print(f'Deleted {img_num} images')
    print(f'Deleted {news_num} news')


def transfer_news():
    news_type, _ = NewsType.objects.get_or_create(name='news')

    queryset = PageTexts.objects.filter(
        page__type='News',
    ).annotate(
        page__id=F('page__id'),
        news_type_id=Value(news_type.id, output_field=IntegerField()),
        page__created_at=F('page__created_at'),
        page__account_id=F('page__account_id'),
        page__state=F('page__state'),
        page__template=F('page__template'),
        page__site__country_code_2=F('page__site__country_code_2'),
        page__root_title=F('page__root_title'),
        page__attachment_suffix_url=F('page__attachment_suffix_url'),
        page__published_at=F('page__published_at'),
    )

    serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f'News serializer errors: {serialized_data.errors}')


def update_en_gb_locales():
    """
    Update default locales (en-GB)
    """
    news = News.objects.filter(old_id__isnull=False)

    update_news = []
    for news_item in tqdm(news):
        news_item.slugs = add_locale('en-GB', news_item.slugs)
        news_item.title = add_locale('en-GB', news_item.title)
        news_item.locale_to_description_is_active = add_locale('en-GB', news_item.locale_to_description_is_active)
        news_item.description = add_locale('en-GB', news_item.description)
        news_item.subtitle = add_locale('en-GB', news_item.subtitle)
        update_news.append(news_item)
    News.objects.bulk_update(update_news, [
        'slugs',
        'title',
        'locale_to_description_is_active',
        'description',
        'subtitle',
    ])
    print(f'Updated {len(update_news)} news locales')


def add_views_count():
    """
    Add views count to news from page_counters
    """

    news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
    counters = PageCounters.objects.filter(page_id__in=list(news))

    update_counters = []
    for counter in tqdm(counters):
        news_item = News.objects.filter(old_id=counter.page_id).first()
        if news_item:
            obj, _ = ViewCount.objects.update_or_create(
                news=news_item,
                defaults={'count': counter.count},
            )
            news_item.views_count = obj
            update_counters.append(news_item)
    News.objects.bulk_update(update_counters, ['views_count', ])
    print(f'Updated {len(update_counters)} news counters')


def add_tags():
    """
    Add news tags
    """

    news_type, _ = NewsType.objects.get_or_create(name='news')
    tag_category, _ = TagCategory.objects.get_or_create(index_name='category')
    tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag')
    news_type.tag_categories.add(tag_category)
    news_type.tag_categories.add(tag_tag)
    news_type.save()

    tag_cat = {
        'category': tag_category,
        'tag': tag_tag,
    }

    news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
    old_news_tag = PageMetadata.objects.filter(
        key__in=('category', 'tag'),
        page_id__in=list(news),
    )

    count = 0
    for old_tag in tqdm(old_news_tag):
        old_id = old_tag.page.id
        new_tag, created = Tag.objects.get_or_create(
            category=tag_cat.get(old_tag.key),
            value=old_tag.value,
        )
        if created:
            text_value = ' '.join(new_tag.value.split('_'))
            translation = SiteInterfaceDictionary(text={'en-GB': text_value}, keywords=f'tag.{new_tag.category}.{new_tag.value}')
            translation.save()
            new_tag.translation = translation
            new_tag.save()

        news = News.objects.filter(old_id=old_id).first()
        if news:
            news.tags.add(new_tag)
            news.save()
            count += 1

    print(f'Updated {count} tags')


data_types = {
    'news': [
        clear_old_news,
        transfer_news,
        update_en_gb_locales,
        add_views_count,
        add_tags,
    ],
}