from pprint import pprint from django.db.models import IntegerField, F from django.db.models import Value from tqdm import tqdm from gallery.models import Image from news.models import NewsType, News from rating.models import ViewCount from tag.models import TagCategory, Tag from transfer.models import PageTexts, PageCounters, PageMetadata from transfer.serializers.news import NewsSerializer def add_locale(locale, data): if isinstance(data, dict) and locale not in data: data.update({ locale: next(iter(data.values())) }) return data def clear_old_news(): """ Clear lod news and news images """ images = Image.objects.filter( news_gallery__isnull=False, news__gallery__news__old_id__isnull=False ) img_num = images.count() news = News.objects.filter(old_id__isnull=False) news_num = news.count() images.delete() news.delete() print(f'Deleted {img_num} images') print(f'Deleted {news_num} news') def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name='News') queryset = PageTexts.objects.filter( page__type='News', ).annotate( page__id=F('page__id'), news_type_id=Value(news_type.id, output_field=IntegerField()), page__created_at=F('page__created_at'), page__account_id=F('page__account_id'), page__state=F('page__state'), page__template=F('page__template'), page__site__country_code_2=F('page__site__country_code_2'), page__root_title=F('page__root_title'), page__attachment_suffix_url=F('page__attachment_suffix_url'), page__published_at=F('page__published_at'), ) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f'News serializer errors: {serialized_data.errors}') def update_en_gb_locales(): """ Update default locales (en-GB) """ news = News.objects.filter(old_id__isnull=False) update_news = [] for news_item in tqdm(news): news_item.slugs = add_locale('en-GB', news_item.slugs) news_item.title = add_locale('en-GB', news_item.title) news_item.locale_to_description_is_active = add_locale('en-GB', news_item.locale_to_description_is_active) news_item.description = add_locale('en-GB', news_item.description) news_item.subtitle = add_locale('en-GB', news_item.subtitle) update_news.append(news_item) News.objects.bulk_update(update_news, [ 'slugs', 'title', 'locale_to_description_is_active', 'description', 'subtitle', ]) print(f'Updated {len(update_news)} news locales') def add_views_count(): """ Add views count to news from page_counters """ news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) counters = PageCounters.objects.filter(page_id__in=list(news)) update_counters = [] for counter in tqdm(counters): news_item = News.objects.filter(old_id=counter.page_id).first() if news_item: obj, _ = ViewCount.objects.update_or_create( news=news_item, defaults={'count': counter.count}, ) news_item.views_count = obj update_counters.append(news_item) News.objects.bulk_update(update_counters, ['views_count', ]) print(f'Updated {len(update_counters)} news counters') def add_tags(): """ Add news tags """ news_type, _ = NewsType.objects.get_or_create(name='News') tag_category, _ = TagCategory.objects.get_or_create(index_name='category') tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag') news_type.tag_categories.add(tag_category) news_type.tag_categories.add(tag_tag) news_type.save() tag_cat = { 'category': tag_category, 'tag': tag_tag, } news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) old_news_tag = PageMetadata.objects.filter( key__in=('category', 'tag'), page_id__in=list(news), ) count = 0 for old_tag in tqdm(old_news_tag): old_id = old_tag.page.id new_tag, created = Tag.objects.get_or_create( category=tag_cat.get(old_tag.key), value=old_tag.value, ) if created: text_value = ' '.join(new_tag.value.split('_')) new_tag.label = {'en-GB': text_value} new_tag.save() news = News.objects.filter(old_id=old_id).first() if news: news.tags.add(new_tag) news.save() count += 1 print(f'Updated {count} tags') data_types = { 'news': [ clear_old_news, transfer_news, update_en_gb_locales, add_views_count, add_tags, ] }