From 17fdc7705e47c74819f7e506591b83b44065ec3b Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 23 Dec 2019 11:53:34 +0300 Subject: [PATCH] news transfer fin --- apps/news/management/commands/add_author.py | 29 ---- .../news/management/commands/add_news_tags.py | 37 ----- .../migrations/0049_auto_20191223_0619.py | 19 +++ apps/news/models.py | 11 +- apps/news/transfer_data.py | 151 +++++++++++++++--- .../migrations/0005_auto_20191223_0850.py | 18 +++ apps/rating/models.py | 2 +- apps/recipe/models.py | 4 + apps/transfer/serializers/news.py | 72 +++------ docker-compose.mysql.yml | 2 - 10 files changed, 195 insertions(+), 150 deletions(-) delete mode 100644 apps/news/management/commands/add_author.py delete mode 100644 apps/news/management/commands/add_news_tags.py create mode 100644 apps/news/migrations/0049_auto_20191223_0619.py create mode 100644 apps/rating/migrations/0005_auto_20191223_0850.py diff --git a/apps/news/management/commands/add_author.py b/apps/news/management/commands/add_author.py deleted file mode 100644 index 0313d7b6..00000000 --- a/apps/news/management/commands/add_author.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.core.management.base import BaseCommand -from django.db.models import F -from tqdm import tqdm - -from account.models import User -from news.models import News -from transfer.models import PageTexts - - -class Command(BaseCommand): - help = 'Add author of News' - - def handle(self, *args, **kwargs): - count = 0 - news_list = News.objects.filter(created_by__isnull=True) - - for news in tqdm(news_list, desc="Find author for exist news"): - old_news = PageTexts.objects.filter(id=news.old_id).annotate( - account_id=F('page__account_id'), - ).first() - if old_news: - user = User.objects.filter(old_id=old_news.account_id).first() - if user: - news.created_by = user - news.modified_by = user - news.save() - count += 1 - - self.stdout.write(self.style.WARNING(f'Update {count} objects.')) diff --git a/apps/news/management/commands/add_news_tags.py b/apps/news/management/commands/add_news_tags.py deleted file mode 100644 index b4c5f8eb..00000000 --- a/apps/news/management/commands/add_news_tags.py +++ /dev/null @@ -1,37 +0,0 @@ -from django.core.management.base import BaseCommand - -from news.models import News, NewsType -from tag.models import Tag, TagCategory -from transfer.models import PageMetadata, Pages, PageTexts - - -class Command(BaseCommand): - help = 'Remove old news from new bd'\ - # TODO: изменить перенос тэгов по old_id новостей (они теперь от page) - - def handle(self, *args, **kwargs): - count = 0 - news_type, _ = NewsType.objects.get_or_create(name='News') - tag_cat, _ = TagCategory.objects.get_or_create(index_name='category') - news_type.tag_categories.add(tag_cat) - news_type.save() - - old_news_tag = PageMetadata.objects.filter(key='category', page__pagetexts__isnull=False) - for old_tag in old_news_tag: - old_id_list = old_tag.page.pagetexts_set.all().values_list('id', flat=True) - - # Make Tag - new_tag, created = Tag.objects.get_or_create(category=tag_cat, value=old_tag.value) - if created: - text_value = ' '.join(new_tag.value.split('_')) - new_tag.label = {'en-GB': text_value} - new_tag.save() - for id in old_id_list: - if isinstance(id, int): - news = News.objects.filter(old_id=id).first() - if news: - news.tags.add(new_tag) - news.save() - count += 1 - - self.stdout.write(self.style.WARNING(f'Create or update {count} objects.')) diff --git a/apps/news/migrations/0049_auto_20191223_0619.py b/apps/news/migrations/0049_auto_20191223_0619.py new file mode 100644 index 00000000..4d9cf52c --- /dev/null +++ b/apps/news/migrations/0049_auto_20191223_0619.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.7 on 2019-12-23 06:19 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0048_remove_news_must_of_the_week'), + ] + + operations = [ + migrations.AlterField( + model_name='news', + name='views_count', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='news', to='rating.ViewCount'), + ), + ] diff --git a/apps/news/models.py b/apps/news/models.py index 64b22ffb..76348a3d 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -105,10 +105,10 @@ class NewsQuerySet(TranslationQuerysetMixin): date_now = now.date() time_now = now.time() return self.exclude(models.Q(publication_date__isnull=True) | models.Q(publication_time__isnull=True)). \ - filter(models.Q(models.Q(end__gte=now) | - models.Q(end__isnull=True)), - state__in=self.model.PUBLISHED_STATES, publication_date__lte=date_now, - publication_time__lte=time_now) + filter(models.Q(models.Q(end__gte=now) | + models.Q(end__isnull=True)), + state__in=self.model.PUBLISHED_STATES, publication_date__lte=date_now, + publication_time__lte=time_now) # todo: filter by best score # todo: filter by country? @@ -215,7 +215,8 @@ class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixi tags = models.ManyToManyField('tag.Tag', related_name='news', verbose_name=_('Tags')) gallery = models.ManyToManyField('gallery.Image', through='news.NewsGallery') - views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL) + views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL, + related_name='news') ratings = generic.GenericRelation(Rating) favorites = generic.GenericRelation(to='favorites.Favorites') carousels = generic.GenericRelation(to='main.Carousel') diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 8596bcf7..c2f621a5 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,34 +1,47 @@ from pprint import pprint -from django.db.models import Aggregate, CharField, Value from django.db.models import IntegerField, F +from django.db.models import Value +from tqdm import tqdm -from news.models import NewsType -from tag.models import TagCategory -from transfer.models import PageTexts +from gallery.models import Image +from news.models import NewsType, News +from rating.models import ViewCount +from tag.models import TagCategory, Tag +from transfer.models import PageTexts, PageCounters, PageMetadata from transfer.serializers.news import NewsSerializer -class GroupConcat(Aggregate): - function = 'GROUP_CONCAT' - template = '%(function)s(%(expressions)s)' +def add_locale(locale, data): + if isinstance(data, dict) and locale not in data: + data.update({ + locale: next(iter(data.values())) + }) + return data - def __init__(self, expression, **extra): - output_field = extra.pop('output_field', CharField()) - super().__init__(expression, output_field=output_field, **extra) - def as_postgresql(self, compiler, connection): - self.function = 'STRING_AGG' - return super().as_sql(compiler, connection) +def clear_old_news(): + """ + Clear lod news and news images + """ + images = Image.objects.filter( + news_gallery__isnull=False, + news__gallery__news__old_id__isnull=False + ) + img_num = images.count() + + news = News.objects.filter(old_id__isnull=False) + news_num = news.count() + + images.delete() + news.delete() + + print(f'Deleted {img_num} images') + print(f'Deleted {news_num} news') def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name='News') - tag_cat_tag, _ = TagCategory.objects.get_or_create(index_name='tag') - tag_cat_category, _ = TagCategory.objects.get_or_create(index_name='category') - news_type.tag_categories.add(tag_cat_tag) - news_type.tag_categories.add(tag_cat_category) - news_type.save() queryset = PageTexts.objects.filter( page__type='News', @@ -43,10 +56,6 @@ def transfer_news(): page__root_title=F('page__root_title'), page__attachment_suffix_url=F('page__attachment_suffix_url'), page__published_at=F('page__published_at'), - - tags=GroupConcat('page__tags__id'), - tag_cat_tag_id=Value(tag_cat_tag.id, output_field=IntegerField()), - tag_cat_category_id=Value(tag_cat_category.id, output_field=IntegerField()), ) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) @@ -56,6 +65,102 @@ def transfer_news(): pprint(f'News serializer errors: {serialized_data.errors}') +def update_en_gb_locales(): + """ + Update default locales (en-GB) + """ + news = News.objects.filter(old_id__isnull=False) + + update_news = [] + for news_item in tqdm(news): + news_item.slugs = add_locale('en-GB', news_item.slugs) + news_item.title = add_locale('en-GB', news_item.title) + news_item.locale_to_description_is_active = add_locale('en-GB', news_item.locale_to_description_is_active) + news_item.description = add_locale('en-GB', news_item.description) + news_item.subtitle = add_locale('en-GB', news_item.subtitle) + update_news.append(news_item) + News.objects.bulk_update(update_news, [ + 'slugs', + 'title', + 'locale_to_description_is_active', + 'description', + 'subtitle', + ]) + print(f'Updated {len(update_news)} news locales') + + +def add_views_count(): + """ + Add views count to news from page_counters + """ + + news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + counters = PageCounters.objects.filter(page_id__in=list(news)) + + update_counters = [] + for counter in tqdm(counters): + news_item = News.objects.filter(old_id=counter.page_id).first() + if news_item: + obj, _ = ViewCount.objects.update_or_create( + news=news_item, + defaults={'count': counter.count}, + ) + news_item.views_count = obj + update_counters.append(news_item) + News.objects.bulk_update(update_counters, ['views_count', ]) + print(f'Updated {len(update_counters)} news counters') + + +def add_tags(): + """ + Add news tags + """ + + news_type, _ = NewsType.objects.get_or_create(name='News') + tag_category, _ = TagCategory.objects.get_or_create(index_name='category') + tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag') + news_type.tag_categories.add(tag_category) + news_type.tag_categories.add(tag_tag) + news_type.save() + + tag_cat = { + 'category': tag_category, + 'tag': tag_tag, + } + + news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + old_news_tag = PageMetadata.objects.filter( + key__in=('category', 'tag'), + page_id__in=list(news), + ) + + count = 0 + for old_tag in tqdm(old_news_tag): + old_id = old_tag.page.id + new_tag, created = Tag.objects.get_or_create( + category=tag_cat.get(old_tag.key), + value=old_tag.value, + ) + if created: + text_value = ' '.join(new_tag.value.split('_')) + new_tag.label = {'en-GB': text_value} + new_tag.save() + + news = News.objects.filter(old_id=old_id).first() + if news: + news.tags.add(new_tag) + news.save() + count += 1 + + print(f'Updated {count} tags') + + data_types = { - 'news': [transfer_news] + 'news': [ + clear_old_news, + transfer_news, + update_en_gb_locales, + add_views_count, + add_tags, + ] } diff --git a/apps/rating/migrations/0005_auto_20191223_0850.py b/apps/rating/migrations/0005_auto_20191223_0850.py new file mode 100644 index 00000000..437056c7 --- /dev/null +++ b/apps/rating/migrations/0005_auto_20191223_0850.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-12-23 08:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rating', '0004_auto_20191114_2041'), + ] + + operations = [ + migrations.AlterField( + model_name='viewcount', + name='count', + field=models.PositiveIntegerField(), + ), + ] diff --git a/apps/rating/models.py b/apps/rating/models.py index 5db8332e..a4048128 100644 --- a/apps/rating/models.py +++ b/apps/rating/models.py @@ -23,4 +23,4 @@ class Rating(models.Model): class ViewCount(models.Model): - count = models.IntegerField() + count = models.PositiveIntegerField() diff --git a/apps/recipe/models.py b/apps/recipe/models.py index c419be4c..f8c4aedf 100644 --- a/apps/recipe/models.py +++ b/apps/recipe/models.py @@ -10,6 +10,7 @@ class RecipeQuerySet(models.QuerySet): # todo: what records are considered published? def published(self): + # TODO: проверка по полю published_at return self.filter(state__in=[self.model.PUBLISHED, self.model.PUBLISHED_EXCLUSIVE]) @@ -67,3 +68,6 @@ class Recipe(TranslatedFieldsMixin, ImageMixin, BaseAttributes): verbose_name = _('Recipe') verbose_name_plural = _('Recipes') + + # TODO: в save добавить обновление published_at если state в PUBLISHED или PUBLISHED_EXCLUSIVE + # TODO: в save добавить обновление published_at в None если state в WAITING или HIDDEN diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 15634ae7..ab6a0b4b 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,35 +1,13 @@ from rest_framework import serializers +from account.models import User from gallery.models import Image from location.models import Country from news.models import News, NewsGallery -from tag.models import Tag -from transfer.models import PageMetadata from utils.legacy_parser import parse_legacy_news_content -from utils.slug_generator import generate_unique_slug -from account.models import User class NewsSerializer(serializers.Serializer): - # old_id = page__id id -done - # news_type = 'News' создали или получили в трансфере -done - # title = {"en-GB":"some text"} из locale и title -done - # backoffice_title = page__root_title -done - # subtitle = {"en-GB":"some text"} из locale и summary -done - # description = {"en-GB":"some text"} из locale и body -done - # locale_to_description_is_active = {"en-GB": true, "fr-FR": false} из locale и true -done - # publication_date = DateField из page published_at -done ??? проверить - # publication_time = DateField из page published_at -done ??? проверить - # slugs = {"en-GB":"some slug"} из locale и slug -done - # state = page__state -done - # template = page__template -done - # country = по page__site__country_code_2 -done - # tags = по page__tags__id -progress -!!! - # gallery = в методе make_gallery из page__attachment_suffix_url -done - # created_by = page__account_id -done - # modified_by = page__account_id -done - # created = page created_at -done - locale = serializers.CharField() page__id = serializers.IntegerField() news_type_id = serializers.IntegerField() @@ -46,10 +24,6 @@ class NewsSerializer(serializers.Serializer): page__attachment_suffix_url = serializers.CharField() page__published_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S', allow_null=True) - tags = serializers.CharField(allow_null=True) - tag_cat_tag_id = serializers.IntegerField() - tag_cat_category_id = serializers.IntegerField() - def create(self, data): account = self.get_account(data) payload = { @@ -61,7 +35,7 @@ class NewsSerializer(serializers.Serializer): 'state': self.get_state(data), 'template': self.get_template(data), 'country': self.get_country(data), - 'slug': {data['locale']: data['slug']}, + 'slugs': {data['locale']: data['slug']}, 'description': self.get_description(data), 'title': {data['locale']: data['title']}, 'backoffice_title': data['page__root_title'], @@ -71,15 +45,26 @@ class NewsSerializer(serializers.Serializer): 'publication_time': self.get_publication_time(data), } - obj, _ = News.objects.update_or_create( - old_id=data['old_id'], + obj, created = News.objects.get_or_create( + old_id=payload['old_id'], defaults=payload, ) + if not created: + obj.slugs.update(payload['slugs']) + obj.title.update(payload['title']) + obj.locale_to_description_is_active.update(payload['locale_to_description_is_active']) - tags = self.get_tags(data) - for tag in tags: - obj.tags.add(tag) - obj.save() + if obj.description and payload['description']: + obj.description.update(payload['description']) + else: + obj.description = payload['description'] + + if obj.subtitle and payload['subtitle']: + obj.subtitle.update(payload['subtitle']) + else: + obj.subtitle = payload['subtitle'] + + obj.save() self.make_gallery(data, obj) return obj @@ -139,25 +124,6 @@ class NewsSerializer(serializers.Serializer): return {data['locale']: data['summary']} return None - @staticmethod - def get_tags(data): - results = [] - if not data['tags']: - return results - - meta_ids = (int(_id) for _id in data['tags'].split(',')) - tags = PageMetadata.objects.filter( - id__in=meta_ids, - value__isnull=False, - ) - for old_tag in tags: - tag, _ = Tag.objects.get_or_create( - category_id=data['tag_cat_id'], - label={data['locale']: old_tag.value}, - ) - results.append(tag) - return results - @staticmethod def make_gallery(data, obj): if not data['page__attachment_suffix_url'] or data['page__attachment_suffix_url'] == 'default/missing.png': diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index d62ff677..a8900226 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -13,7 +13,6 @@ services: MYSQL_ROOT_PASSWORD: rootPassword volumes: - gm-mysql_db:/var/lib/mysql - - .:/code # PostgreSQL database @@ -30,7 +29,6 @@ services: - "5436:5432" volumes: - gm-db:/var/lib/postgresql/data/ - - .:/code elasticsearch: