From bf8afab0fbb86006e4e70dac25d8e43586d1eb87 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 14 Nov 2019 15:33:46 +0300 Subject: [PATCH] refactor refactor reviews --- apps/review/admin.py | 2 +- .../migrations/0016_remove_review_language.py | 17 +++ apps/review/models.py | 63 +++++---- apps/review/transfer_data.py | 109 ++++++---------- apps/transfer/models.py | 2 +- apps/transfer/serializers/reviews.py | 123 +++++++++--------- 6 files changed, 157 insertions(+), 159 deletions(-) create mode 100644 apps/review/migrations/0016_remove_review_language.py diff --git a/apps/review/admin.py b/apps/review/admin.py index 4d50ceda..b1ac0636 100644 --- a/apps/review/admin.py +++ b/apps/review/admin.py @@ -8,4 +8,4 @@ from utils.admin import BaseModelAdminMixin class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin): """Admin model for model Review.""" - raw_id_fields = ('reviewer', 'language', 'child', 'country') + raw_id_fields = ('reviewer', 'child', 'country') diff --git a/apps/review/migrations/0016_remove_review_language.py b/apps/review/migrations/0016_remove_review_language.py new file mode 100644 index 00000000..dda4ffd2 --- /dev/null +++ b/apps/review/migrations/0016_remove_review_language.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.7 on 2019-11-14 07:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('review', '0015_review_mark'), + ] + + operations = [ + migrations.RemoveField( + model_name='review', + name='language', + ), + ] diff --git a/apps/review/models.py b/apps/review/models.py index 08e0a26c..3f095504 100644 --- a/apps/review/models.py +++ b/apps/review/models.py @@ -39,38 +39,49 @@ class Review(BaseAttributes, TranslatedFieldsMixin): (READY, _('Ready')), ) - reviewer = models.ForeignKey('account.User', - related_name='reviews', - on_delete=models.CASCADE, - verbose_name=_('Reviewer')) + reviewer = models.ForeignKey( + 'account.User', + related_name='reviews', + on_delete=models.CASCADE, + verbose_name=_('Reviewer'), + ) + country = models.ForeignKey( + 'location.Country', + on_delete=models.CASCADE, + related_name='country', + verbose_name=_('Country'), + null=True, + ) + child = models.ForeignKey( + 'self', + blank=True, + default=None, + null=True, + on_delete=models.CASCADE, + verbose_name=_('Child review'), + ) text = TJSONField( - _('text'), null=True, blank=True, - default=None, help_text='{"en-GB":"Text review"}') + _('text'), + null=True, + blank=True, + default=None, + help_text='{"en-GB":"Text review"}', + ) content_type = models.ForeignKey(generic.ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey('content_type', 'object_id') - language = models.ForeignKey('translation.Language', - on_delete=models.CASCADE, - related_name='reviews', - verbose_name=_('Review language')) + status = models.PositiveSmallIntegerField(choices=REVIEW_STATUSES, default=TO_INVESTIGATE) - child = models.ForeignKey('self', - blank=True, default=None, null=True, - on_delete=models.CASCADE, - verbose_name=_('Child review')) - published_at = models.DateTimeField(verbose_name=_('Publish datetime'), - blank=True, default=None, null=True, - help_text=_('Review published datetime')) - vintage = models.IntegerField(verbose_name=_('Year of review'), - validators=[MinValueValidator(1900), - MaxValueValidator(2100)]) - - country = models.ForeignKey('location.Country', on_delete=models.CASCADE, - related_name='country', verbose_name=_('Country'), - null=True) - - old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) + published_at = models.DateTimeField( + _('Publish datetime'), + blank=True, + default=None, + null=True, + help_text=_('Review published datetime'), + ) + vintage = models.IntegerField(_('Year of review'), validators=[MinValueValidator(1900), MaxValueValidator(2100)]) mark = models.FloatField(verbose_name=_('mark'), blank=True, null=True, default=None) + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) objects = ReviewQuerySet.as_manager() diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index cde55efe..8ac1104c 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -1,15 +1,16 @@ -import json from pprint import pprint from django.db.models import Q +from account.models import User from account.transfer_data import STOP_LIST +from establishment.models import Establishment from review.models import Inquiries as NewInquiries, Review from transfer.models import Reviews, ReviewTexts, Inquiries, GridItems, InquiryPhotos from transfer.serializers.grid import GridItemsSerializer from transfer.serializers.inquiries import InquiriesSerializer from transfer.serializers.inquiry_gallery import InquiryGallerySerializer -from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment +from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, ReviewTextSerializer def transfer_languages(): @@ -28,77 +29,42 @@ def transfer_languages(): def transfer_reviews(): - # TODO: убрать LIKE UPPER("%%paris%%"), accounts.email IN - queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, - reviews.reviewer_id, review_texts.text AS text, reviews.mark, reviews.published_at, - review_texts.created_at AS published, review_texts.locale AS locale, - reviews.aasm_state - FROM reviews - LEFT OUTER JOIN review_texts - ON (reviews.id = review_texts.review_id) - WHERE reviews.reviewer_id > 0 - AND reviews.reviewer_id IS NOT NULL - AND review_texts.text IS NOT NULL - AND review_texts.locale IS NOT NULL - AND reviews.mark IS NOT NULL - AND reviews.reviewer_id IN ( - SELECT accounts.id - FROM accounts - WHERE accounts.confirmed_at IS NOT NULL - AND NOT accounts.email IN ( - "cyril@tomatic.net", - "cyril2@tomatic.net", - "cyril2@tomatic.net", - "d.sadykova@id-east.ru", - "d.sadykova@octopod.ru", - "n.yurchenko@id-east.ru" - )) - AND reviews.establishment_id IN ( - SELECT establishments.id - FROM establishments - INNER JOIN locations - ON (establishments.location_id = locations.id) - INNER JOIN cities - ON (locations.city_id = cities.id) - WHERE establishments.type IS NOT NULL AND locations.timezone IS NOT NULL - AND NOT establishments.type = "Wineyard" - ) - ORDER BY review_texts.created_at DESC - """) - - queryset_result = [] - establishments_mark_list = {} - - for query in queryset: - query = vars(query) - if query['establishment_id'] not in establishments_mark_list.keys(): - if "aasm_state" in query and query['aasm_state'] is not None and query['aasm_state'] == "published": - establishments_mark_list[query['establishment_id']] = [ - int(query['mark']), - json.dumps({query['locale']: query['text']}) - ] - else: - establishments_mark_list[query['establishment_id']] = int(query['mark']) - del (query['mark']) - queryset_result.append(query) - - serialized_data = ReviewSerializer(data=queryset_result, many=True) + establishments = Establishment.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + users = User.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + queryset = Reviews.objects.filter( + establishment_id__in=list(establishments), + reviewer_id__in=list(users), + ).values('id', 'reviewer_id', 'aasm_state', 'created_at', 'establishment_id', 'mark', 'vintage') + serialized_data = ReviewSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() - - for establishment_id, mark in establishments_mark_list.items(): - try: - establishment = Establishment.objects.get(old_id=establishment_id) - except Establishment.DoesNotExist: - continue - if isinstance(mark, list): - mark, review_text = mark - - establishment.public_mark = mark - establishment.save() else: - pprint(serialized_data.errors) + pprint(f"ReviewSerializer serializer errors: {serialized_data.errors}") + + +def transfer_text_review(): + reviews = Review.objects.filter(old_id__isnull=False).values_list('old_id', flat=True) + queryset = ReviewTexts.objects.filter( + review_id__in=list(reviews), + ).exclude( + Q(text__isnull=True) | Q(text='') + ).values('review_id', 'locale', 'text') + + serialized_data = ReviewTextSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"ReviewTextSerializer serializer errors: {serialized_data.errors}") + + for review in Review.objects.filter(old_id__isnull=False): + text = review.text + if text and 'en-GB' not in text: + text.update({ + 'en-GB': next(iter(text.values())) + }) + review.text = text + review.save() def transfer_inquiries(): @@ -139,8 +105,9 @@ def transfer_inquiry_photos(): data_types = { "overlook": [ - transfer_languages, - transfer_reviews + # transfer_languages, + transfer_reviews, + transfer_text_review, ], 'inquiries': [ transfer_inquiries, diff --git a/apps/transfer/models.py b/apps/transfer/models.py index d0c44609..96b5aa36 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -708,7 +708,7 @@ class Reviews(MigrateMixin): published_at = models.DateTimeField(blank=True, null=True) updated_at = models.DateTimeField() aasm_state = models.CharField(max_length=255, blank=True, null=True) - reviewer_id = models.IntegerField() + reviewer = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) priority = models.IntegerField(blank=True, null=True) # TODO: модель Products в postgres закомментирована # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) diff --git a/apps/transfer/serializers/reviews.py b/apps/transfer/serializers/reviews.py index 7bfe2004..5cef5803 100644 --- a/apps/transfer/serializers/reviews.py +++ b/apps/transfer/serializers/reviews.py @@ -5,82 +5,85 @@ from translation.models import Language from establishment.models import Establishment -class ReviewSerializer(serializers.ModelSerializer): - id = serializers.IntegerField() - reviewer_id = serializers.IntegerField() +class ReviewSerializer(serializers.Serializer): vintage = serializers.IntegerField() - published = serializers.DateTimeField() - published_at = serializers.DateTimeField(allow_null=True) + mark = serializers.FloatField(allow_null=True) establishment_id = serializers.IntegerField() - text = serializers.CharField(allow_null=True, allow_blank=True) - locale = serializers.CharField() + created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') aasm_state = serializers.CharField(allow_null=True) - - class Meta: - model = Review - fields = ( - "id", "reviewer_id", "published", "vintage", - "establishment_id", "text", "locale", - "published_at", "aasm_state" - ) + reviewer_id = serializers.IntegerField() + id = serializers.IntegerField() def validate(self, data): - data = self.set_old_id(data) - data = self.set_published_at(data) - data = self.set_reviewer(data) - data = self.set_establishment(data) - data = self.set_language(data) - data = self.set_published(data) + data.update({ + 'reviewer': self.get_reviewer(data), + 'status': Review.READY if data['aasm_state'] == 'published' else Review.TO_INVESTIGATE, + 'published_at': data.pop('created_at'), + 'old_id': data.pop('id'), + 'content_object': self.get_establishment(data), + }) + data.pop('reviewer_id') + data.pop('establishment_id') + data.pop('aasm_state') return data def create(self, validated_data): - try: - return Review.objects.create(**validated_data) - except Exception as e: - raise ValueError(f"Error creating review with {validated_data}: {e}") + obj, _ = Review.objects.update_or_create( + old_id=validated_data['old_id'], + defaults=validated_data, + ) + return obj - def set_old_id(self, data): - data['old_id'] = data.pop("id") + @staticmethod + def get_reviewer(data): + user = User.objects.filter(old_id=data['reviewer_id']).first() + if not user: + raise ValueError(f"User account not found with old_id {data['reviewer_id']}") + return user + + @staticmethod + def get_establishment(data): + establishment = Establishment.objects.filter(old_id=data['establishment_id']).first() + if not establishment: + raise ValueError(f"Establishment not found with old_id {data['establishment_id']}: ") + return establishment + + +class ReviewTextSerializer(serializers.Serializer): + review_id = serializers.IntegerField() + locale = serializers.CharField(allow_null=True) + text = serializers.CharField() + + def validate(self, data): + data.update({ + 'new_text': self.get_text(data), + 'review': self.get_review(data), + }) return data - def set_reviewer(self, data): - try: - data['reviewer'] = User.objects.get(old_id=data['reviewer_id']) - except User.DoesNotExist as e: - raise ValueError(f"Cannot find reviewer with {data}: {e}") - del(data['reviewer_id']) - return data + def create(self, validated_data): + review = validated_data['review'] + if review.text: + review.text.update(validated_data['new_text']) + else: + review.text = validated_data['new_text'] + review.save() + return review - def set_establishment(self, data): - try: - data['content_object'] = Establishment.objects.get(old_id=data.pop('establishment_id')) - except Establishment.DoesNotExist as e: - raise ValueError(f"Cannot find review establishment with {data}: {e}") - return data + @staticmethod + def get_text(data): + locale = data['locale'] or 'en-GB' + return {locale: data['text']} - def set_language(self, data): - try: - data['language'] = Language.objects.get(locale=data['locale']) - except Language.DoesNotExist as e: - raise ValueError(f"Cannot find language with {data}: {e}") - - del(data['locale']) - - return data - - def set_published(self, data): - data['published_at'] = data.pop("published") - return data - - def set_published_at(self, data): - if "aasm_state" in data and data['aasm_state'] is not None and data['aasm_state'] == "published": - data['status'] = Review.READY - del(data['aasm_state']) - return data + @staticmethod + def get_review(data): + review = Review.objects.filter(old_id=data['review_id']).first() + if not review: + raise ValueError(f"Review not found with old_id {data['review_id']}: ") + return review class LanguageSerializer(serializers.ModelSerializer): - LANGUAGES = { "be-BE": "Belarusian - Belarusia", "el-GR": "Greek - Greek",