phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

news transfer fin

Browse Source
This commit is contained in:
alex 2019-12-23 11:53:34 +03:00
parent 56b2b243da
commit 17fdc7705e
10 changed files with 195 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
apps/news/management/commands/add_author.py
View File

@ -1,29 +0,0 @@
from django.core.management.base import BaseCommand
from django.db.models import F
from tqdm import tqdm
from account.models import User
from news.models import News
from transfer.models import PageTexts
class Command(BaseCommand):
help = 'Add author of News'
def handle(self, *args, **kwargs):
count = 0
news_list = News.objects.filter(created_by__isnull=True)
for news in tqdm(news_list, desc="Find author for exist news"):
old_news = PageTexts.objects.filter(id=news.old_id).annotate(
account_id=F('page__account_id'),
).first()
if old_news:
user = User.objects.filter(old_id=old_news.account_id).first()
if user:
news.created_by = user
news.modified_by = user
news.save()
count += 1
self.stdout.write(self.style.WARNING(f'Update {count} objects.'))

37
apps/news/management/commands/add_news_tags.py
View File

@ -1,37 +0,0 @@
from django.core.management.base import BaseCommand
from news.models import News, NewsType
from tag.models import Tag, TagCategory
from transfer.models import PageMetadata, Pages, PageTexts
class Command(BaseCommand):
help = 'Remove old news from new bd'\
# TODO: изменить перенос тэгов по old_id новостей (они теперь от page)
def handle(self, *args, **kwargs):
count = 0
news_type, _ = NewsType.objects.get_or_create(name='News')
tag_cat, _ = TagCategory.objects.get_or_create(index_name='category')
news_type.tag_categories.add(tag_cat)
news_type.save()
old_news_tag = PageMetadata.objects.filter(key='category', page__pagetexts__isnull=False)
for old_tag in old_news_tag:
old_id_list = old_tag.page.pagetexts_set.all().values_list('id', flat=True)
# Make Tag
new_tag, created = Tag.objects.get_or_create(category=tag_cat, value=old_tag.value)
if created:
text_value = ' '.join(new_tag.value.split('_'))
new_tag.label = {'en-GB': text_value}
new_tag.save()
for id in old_id_list:
if isinstance(id, int):
news = News.objects.filter(old_id=id).first()
if news:
news.tags.add(new_tag)
news.save()
count += 1
self.stdout.write(self.style.WARNING(f'Create or update {count} objects.'))

19
apps/news/migrations/0049_auto_20191223_0619.py Normal file
View File

@ -0,0 +1,19 @@
# Generated by Django 2.2.7 on 2019-12-23 06:19
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('news', '0048_remove_news_must_of_the_week'),
]
operations = [
migrations.AlterField(
model_name='news',
name='views_count',
field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='news', to='rating.ViewCount'),
),
]

3
apps/news/models.py
View File

@ -215,7 +215,8 @@ class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixi
tags = models.ManyToManyField('tag.Tag', related_name='news',
verbose_name=_('Tags'))
gallery = models.ManyToManyField('gallery.Image', through='news.NewsGallery')
views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL)
views_count = models.OneToOneField('rating.ViewCount', blank=True, null=True, on_delete=models.SET_NULL,
related_name='news')
ratings = generic.GenericRelation(Rating)
favorites = generic.GenericRelation(to='favorites.Favorites')
carousels = generic.GenericRelation(to='main.Carousel')

153
apps/news/transfer_data.py
View File

@ -1,34 +1,47 @@
from pprint import pprint
from django.db.models import Aggregate, CharField, Value
from django.db.models import IntegerField, F
from django.db.models import Value
from tqdm import tqdm
from news.models import NewsType
from tag.models import TagCategory
from transfer.models import PageTexts
from gallery.models import Image
from news.models import NewsType, News
from rating.models import ViewCount
from tag.models import TagCategory, Tag
from transfer.models import PageTexts, PageCounters, PageMetadata
from transfer.serializers.news import NewsSerializer
class GroupConcat(Aggregate):
function = 'GROUP_CONCAT'
template = '%(function)s(%(expressions)s)'
def add_locale(locale, data):
if isinstance(data, dict) and locale not in data:
data.update({
locale: next(iter(data.values()))
})
return data
def __init__(self, expression, **extra):
output_field = extra.pop('output_field', CharField())
super().__init__(expression, output_field=output_field, **extra)
def as_postgresql(self, compiler, connection):
self.function = 'STRING_AGG'
return super().as_sql(compiler, connection)
def clear_old_news():
"""
Clear lod news and news images
"""
images = Image.objects.filter(
news_gallery__isnull=False,
news__gallery__news__old_id__isnull=False
)
img_num = images.count()
news = News.objects.filter(old_id__isnull=False)
news_num = news.count()
images.delete()
news.delete()
print(f'Deleted {img_num} images')
print(f'Deleted {news_num} news')
def transfer_news():
news_type, _ = NewsType.objects.get_or_create(name='News')
tag_cat_tag, _ = TagCategory.objects.get_or_create(index_name='tag')
tag_cat_category, _ = TagCategory.objects.get_or_create(index_name='category')
news_type.tag_categories.add(tag_cat_tag)
news_type.tag_categories.add(tag_cat_category)
news_type.save()
queryset = PageTexts.objects.filter(
page__type='News',
@ -43,10 +56,6 @@ def transfer_news():
page__root_title=F('page__root_title'),
page__attachment_suffix_url=F('page__attachment_suffix_url'),
page__published_at=F('page__published_at'),
tags=GroupConcat('page__tags__id'),
tag_cat_tag_id=Value(tag_cat_tag.id, output_field=IntegerField()),
tag_cat_category_id=Value(tag_cat_category.id, output_field=IntegerField()),
)
serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
@ -56,6 +65,102 @@ def transfer_news():
pprint(f'News serializer errors: {serialized_data.errors}')
data_types = {
'news': [transfer_news]
def update_en_gb_locales():
"""
Update default locales (en-GB)
"""
news = News.objects.filter(old_id__isnull=False)
update_news = []
for news_item in tqdm(news):
news_item.slugs = add_locale('en-GB', news_item.slugs)
news_item.title = add_locale('en-GB', news_item.title)
news_item.locale_to_description_is_active = add_locale('en-GB', news_item.locale_to_description_is_active)
news_item.description = add_locale('en-GB', news_item.description)
news_item.subtitle = add_locale('en-GB', news_item.subtitle)
update_news.append(news_item)
News.objects.bulk_update(update_news, [
'slugs',
'title',
'locale_to_description_is_active',
'description',
'subtitle',
])
print(f'Updated {len(update_news)} news locales')
def add_views_count():
"""
Add views count to news from page_counters
"""
news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
counters = PageCounters.objects.filter(page_id__in=list(news))
update_counters = []
for counter in tqdm(counters):
news_item = News.objects.filter(old_id=counter.page_id).first()
if news_item:
obj, _ = ViewCount.objects.update_or_create(
news=news_item,
defaults={'count': counter.count},
)
news_item.views_count = obj
update_counters.append(news_item)
News.objects.bulk_update(update_counters, ['views_count', ])
print(f'Updated {len(update_counters)} news counters')
def add_tags():
"""
Add news tags
"""
news_type, _ = NewsType.objects.get_or_create(name='News')
tag_category, _ = TagCategory.objects.get_or_create(index_name='category')
tag_tag, _ = TagCategory.objects.get_or_create(index_name='tag')
news_type.tag_categories.add(tag_category)
news_type.tag_categories.add(tag_tag)
news_type.save()
tag_cat = {
'category': tag_category,
'tag': tag_tag,
}
news = News.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
old_news_tag = PageMetadata.objects.filter(
key__in=('category', 'tag'),
page_id__in=list(news),
)
count = 0
for old_tag in tqdm(old_news_tag):
old_id = old_tag.page.id
new_tag, created = Tag.objects.get_or_create(
category=tag_cat.get(old_tag.key),
value=old_tag.value,
)
if created:
text_value = ' '.join(new_tag.value.split('_'))
new_tag.label = {'en-GB': text_value}
new_tag.save()
news = News.objects.filter(old_id=old_id).first()
if news:
news.tags.add(new_tag)
news.save()
count += 1
print(f'Updated {count} tags')
data_types = {
'news': [
clear_old_news,
transfer_news,
update_en_gb_locales,
add_views_count,
add_tags,
]
}

18
apps/rating/migrations/0005_auto_20191223_0850.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.7 on 2019-12-23 08:50
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rating', '0004_auto_20191114_2041'),
]
operations = [
migrations.AlterField(
model_name='viewcount',
name='count',
field=models.PositiveIntegerField(),
),
]

2
apps/rating/models.py
View File

@ -23,4 +23,4 @@ class Rating(models.Model):
class ViewCount(models.Model):
count = models.IntegerField()
count = models.PositiveIntegerField()

4
apps/recipe/models.py
View File

@ -10,6 +10,7 @@ class RecipeQuerySet(models.QuerySet):
# todo: what records are considered published?
def published(self):
# TODO: проверка по полю published_at
return self.filter(state__in=[self.model.PUBLISHED,
self.model.PUBLISHED_EXCLUSIVE])
@ -67,3 +68,6 @@ class Recipe(TranslatedFieldsMixin, ImageMixin, BaseAttributes):
verbose_name = _('Recipe')
verbose_name_plural = _('Recipes')
# TODO: в save добавить обновление published_at если state в PUBLISHED или PUBLISHED_EXCLUSIVE
# TODO: в save добавить обновление published_at в None если state в WAITING или HIDDEN

70
apps/transfer/serializers/news.py
View File

@ -1,35 +1,13 @@
from rest_framework import serializers
from account.models import User
from gallery.models import Image
from location.models import Country
from news.models import News, NewsGallery
from tag.models import Tag
from transfer.models import PageMetadata
from utils.legacy_parser import parse_legacy_news_content
from utils.slug_generator import generate_unique_slug
from account.models import User
class NewsSerializer(serializers.Serializer):
# old_id = page__id id -done
# news_type = 'News' создали или получили в трансфере -done
# title = {"en-GB":"some text"} из locale и title -done
# backoffice_title = page__root_title -done
# subtitle = {"en-GB":"some text"} из locale и summary -done
# description = {"en-GB":"some text"} из locale и body -done
# locale_to_description_is_active = {"en-GB": true, "fr-FR": false} из locale и true -done
# publication_date = DateField из page published_at -done ??? проверить
# publication_time = DateField из page published_at -done ??? проверить
# slugs = {"en-GB":"some slug"} из locale и slug -done
# state = page__state -done
# template = page__template -done
# country = по page__site__country_code_2 -done
# tags = по page__tags__id -progress -!!!
# gallery = в методе make_gallery из page__attachment_suffix_url -done
# created_by = page__account_id -done
# modified_by = page__account_id -done
# created = page created_at -done
locale = serializers.CharField()
page__id = serializers.IntegerField()
news_type_id = serializers.IntegerField()
@ -46,10 +24,6 @@ class NewsSerializer(serializers.Serializer):
page__attachment_suffix_url = serializers.CharField()
page__published_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S', allow_null=True)
tags = serializers.CharField(allow_null=True)
tag_cat_tag_id = serializers.IntegerField()
tag_cat_category_id = serializers.IntegerField()
def create(self, data):
account = self.get_account(data)
payload = {
@ -61,7 +35,7 @@ class NewsSerializer(serializers.Serializer):
'state': self.get_state(data),
'template': self.get_template(data),
'country': self.get_country(data),
'slug': {data['locale']: data['slug']},
'slugs': {data['locale']: data['slug']},
'description': self.get_description(data),
'title': {data['locale']: data['title']},
'backoffice_title': data['page__root_title'],
@ -71,14 +45,25 @@ class NewsSerializer(serializers.Serializer):
'publication_time': self.get_publication_time(data),
}
obj, _ = News.objects.update_or_create(
old_id=data['old_id'],
obj, created = News.objects.get_or_create(
old_id=payload['old_id'],
defaults=payload,
)
if not created:
obj.slugs.update(payload['slugs'])
obj.title.update(payload['title'])
obj.locale_to_description_is_active.update(payload['locale_to_description_is_active'])
if obj.description and payload['description']:
obj.description.update(payload['description'])
else:
obj.description = payload['description']
if obj.subtitle and payload['subtitle']:
obj.subtitle.update(payload['subtitle'])
else:
obj.subtitle = payload['subtitle']
tags = self.get_tags(data)
for tag in tags:
obj.tags.add(tag)
obj.save()
self.make_gallery(data, obj)
@ -139,25 +124,6 @@ class NewsSerializer(serializers.Serializer):
return {data['locale']: data['summary']}
return None
@staticmethod
def get_tags(data):
results = []
if not data['tags']:
return results
meta_ids = (int(_id) for _id in data['tags'].split(','))
tags = PageMetadata.objects.filter(
id__in=meta_ids,
value__isnull=False,
)
for old_tag in tags:
tag, _ = Tag.objects.get_or_create(
category_id=data['tag_cat_id'],
label={data['locale']: old_tag.value},
)
results.append(tag)
return results
@staticmethod
def make_gallery(data, obj):
if not data['page__attachment_suffix_url'] or data['page__attachment_suffix_url'] == 'default/missing.png':

2
docker-compose.mysql.yml
View File

@ -13,7 +13,6 @@ services:
MYSQL_ROOT_PASSWORD: rootPassword
volumes:
- gm-mysql_db:/var/lib/mysql
- .:/code
# PostgreSQL database
@ -30,7 +29,6 @@ services:
- "5436:5432"
volumes:
- gm-db:/var/lib/postgresql/data/
- .:/code
elasticsearch: