refactor refactor reviews

2019-11-14 15:33:46 +03:00 · 2019-11-14 15:33:46 +03:00 · bf8afab0fb
commit bf8afab0fb
parent 32e29b19e6
6 changed files with 157 additions and 159 deletions
--- a/apps/review/admin.py
+++ b/apps/review/admin.py
@ -8,4 +8,4 @@ from utils.admin import BaseModelAdminMixin
 class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin):
    """Admin model for model Review."""

-    raw_id_fields = ('reviewer', 'language', 'child', 'country')
+    raw_id_fields = ('reviewer', 'child', 'country')
--- a/apps/review/migrations/0016_remove_review_language.py
+++ b/apps/review/migrations/0016_remove_review_language.py
@ -0,0 +1,17 @@
+# Generated by Django 2.2.7 on 2019-11-14 07:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0015_review_mark'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='review',
+            name='language',
+        ),
+    ]
--- a/apps/review/models.py
+++ b/apps/review/models.py
@ -39,38 +39,49 @@ class Review(BaseAttributes, TranslatedFieldsMixin):
        (READY, _('Ready')),
    )

-    reviewer = models.ForeignKey('account.User',
-                                 related_name='reviews',
-                                 on_delete=models.CASCADE,
-                                 verbose_name=_('Reviewer'))
+    reviewer = models.ForeignKey(
+        'account.User',
+        related_name='reviews',
+        on_delete=models.CASCADE,
+        verbose_name=_('Reviewer'),
+    )
+    country = models.ForeignKey(
+        'location.Country',
+        on_delete=models.CASCADE,
+        related_name='country',
+        verbose_name=_('Country'),
+        null=True,
+    )
+    child = models.ForeignKey(
+        'self',
+        blank=True,
+        default=None,
+        null=True,
+        on_delete=models.CASCADE,
+        verbose_name=_('Child review'),
+    )
    text = TJSONField(
-        _('text'), null=True, blank=True,
-        default=None, help_text='{"en-GB":"Text review"}')
+        _('text'),
+        null=True,
+        blank=True,
+        default=None,
+        help_text='{"en-GB":"Text review"}',
+    )
    content_type = models.ForeignKey(generic.ContentType, on_delete=models.CASCADE)
    object_id = models.PositiveIntegerField()
    content_object = generic.GenericForeignKey('content_type', 'object_id')
-    language = models.ForeignKey('translation.Language',
-                                 on_delete=models.CASCADE,
-                                 related_name='reviews',
-                                 verbose_name=_('Review language'))
+
    status = models.PositiveSmallIntegerField(choices=REVIEW_STATUSES, default=TO_INVESTIGATE)
-    child = models.ForeignKey('self',
-                              blank=True, default=None, null=True,
-                              on_delete=models.CASCADE,
-                              verbose_name=_('Child review'))
-    published_at = models.DateTimeField(verbose_name=_('Publish datetime'),
-                                        blank=True, default=None, null=True,
-                                        help_text=_('Review published datetime'))
-    vintage = models.IntegerField(verbose_name=_('Year of review'),
-                                  validators=[MinValueValidator(1900),
-                                              MaxValueValidator(2100)])
-
-    country = models.ForeignKey('location.Country', on_delete=models.CASCADE,
-                                related_name='country', verbose_name=_('Country'),
-                                null=True)
-
-    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
+    published_at = models.DateTimeField(
+        _('Publish datetime'),
+        blank=True,
+        default=None,
+        null=True,
+        help_text=_('Review published datetime'),
+    )
+    vintage = models.IntegerField(_('Year of review'), validators=[MinValueValidator(1900), MaxValueValidator(2100)])
    mark = models.FloatField(verbose_name=_('mark'), blank=True, null=True, default=None)
+    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)

    objects = ReviewQuerySet.as_manager()

--- a/apps/review/transfer_data.py
+++ b/apps/review/transfer_data.py
@ -1,15 +1,16 @@
-import json
 from pprint import pprint

 from django.db.models import Q

+from account.models import User
 from account.transfer_data import STOP_LIST
+from establishment.models import Establishment
 from review.models import Inquiries as NewInquiries, Review
 from transfer.models import Reviews, ReviewTexts, Inquiries, GridItems, InquiryPhotos
 from transfer.serializers.grid import GridItemsSerializer
 from transfer.serializers.inquiries import InquiriesSerializer
 from transfer.serializers.inquiry_gallery import InquiryGallerySerializer
-from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment
+from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, ReviewTextSerializer


 def transfer_languages():
@ -28,77 +29,42 @@ def transfer_languages():


 def transfer_reviews():
-    # TODO: убрать LIKE UPPER("%%paris%%"), accounts.email IN
-    queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, 
-                                    reviews.reviewer_id, review_texts.text AS text, reviews.mark, reviews.published_at,
-                                    review_texts.created_at AS published, review_texts.locale AS locale, 
-                                    reviews.aasm_state
-                                    FROM reviews 
-                                    LEFT OUTER JOIN review_texts 
-                                    ON (reviews.id = review_texts.review_id) 
-                                    WHERE reviews.reviewer_id > 0 
-                                    AND reviews.reviewer_id IS NOT NULL 
-                                    AND review_texts.text IS NOT NULL
-                                    AND review_texts.locale IS NOT NULL
-                                    AND reviews.mark IS NOT NULL
-                                    AND reviews.reviewer_id IN (
-                                      SELECT accounts.id 
-                                      FROM accounts 
-                                      WHERE accounts.confirmed_at IS NOT NULL 
-                                      AND NOT accounts.email IN (
-                                        "cyril@tomatic.net", 
-                                        "cyril2@tomatic.net", 
-                                        "cyril2@tomatic.net", 
-                                        "d.sadykova@id-east.ru", 
-                                        "d.sadykova@octopod.ru", 
-                                        "n.yurchenko@id-east.ru"
-                                      ))
-                                    AND reviews.establishment_id IN (
-                                      SELECT establishments.id 
-                                      FROM establishments 
-                                      INNER JOIN locations 
-                                      ON (establishments.location_id = locations.id) 
-                                      INNER JOIN cities 
-                                      ON (locations.city_id = cities.id) 
-                                      WHERE establishments.type IS NOT NULL AND locations.timezone IS NOT NULL
-                                      AND NOT establishments.type = "Wineyard"
-                                    )
-                                    ORDER BY review_texts.created_at DESC
-                                  """)
-
-    queryset_result = []
-    establishments_mark_list = {}
-
-    for query in queryset:
-        query = vars(query)
-        if query['establishment_id'] not in establishments_mark_list.keys():
-            if "aasm_state" in query and query['aasm_state'] is not None and query['aasm_state'] == "published":
-                establishments_mark_list[query['establishment_id']] = [
-                    int(query['mark']),
-                    json.dumps({query['locale']: query['text']})
-                ]
-            else:
-                establishments_mark_list[query['establishment_id']] = int(query['mark'])
-        del (query['mark'])
-        queryset_result.append(query)
-
-    serialized_data = ReviewSerializer(data=queryset_result, many=True)
+    establishments = Establishment.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
+    users = User.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
+    queryset = Reviews.objects.filter(
+        establishment_id__in=list(establishments),
+        reviewer_id__in=list(users),
+    ).values('id', 'reviewer_id', 'aasm_state', 'created_at', 'establishment_id', 'mark', 'vintage')

+    serialized_data = ReviewSerializer(data=list(queryset.values()), many=True)
    if serialized_data.is_valid():
        serialized_data.save()
-
-        for establishment_id, mark in establishments_mark_list.items():
-            try:
-                establishment = Establishment.objects.get(old_id=establishment_id)
-            except Establishment.DoesNotExist:
-                continue
-            if isinstance(mark, list):
-                mark, review_text = mark
-
-            establishment.public_mark = mark
-            establishment.save()
    else:
-        pprint(serialized_data.errors)
+        pprint(f"ReviewSerializer serializer errors: {serialized_data.errors}")
+
+
+def transfer_text_review():
+    reviews = Review.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
+    queryset = ReviewTexts.objects.filter(
+        review_id__in=list(reviews),
+    ).exclude(
+        Q(text__isnull=True) | Q(text='')
+    ).values('review_id', 'locale', 'text')
+
+    serialized_data = ReviewTextSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"ReviewTextSerializer serializer errors: {serialized_data.errors}")
+
+    for review in Review.objects.filter(old_id__isnull=False):
+        text = review.text
+        if text and 'en-GB' not in text:
+            text.update({
+                'en-GB': next(iter(text.values()))
+            })
+            review.text = text
+            review.save()


 def transfer_inquiries():
@ -139,8 +105,9 @@ def transfer_inquiry_photos():

 data_types = {
    "overlook": [
-        transfer_languages,
-        transfer_reviews
+        # transfer_languages,
+        transfer_reviews,
+        transfer_text_review,
    ],
    'inquiries': [
        transfer_inquiries,
--- a/apps/transfer/models.py
+++ b/apps/transfer/models.py
@ -708,7 +708,7 @@ class Reviews(MigrateMixin):
    published_at = models.DateTimeField(blank=True, null=True)
    updated_at = models.DateTimeField()
    aasm_state = models.CharField(max_length=255, blank=True, null=True)
-    reviewer_id = models.IntegerField()
+    reviewer = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True)
    priority = models.IntegerField(blank=True, null=True)
    # TODO: модель Products в postgres закомментирована
    # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True)
--- a/apps/transfer/serializers/reviews.py
+++ b/apps/transfer/serializers/reviews.py
@ -5,82 +5,85 @@ from translation.models import Language
 from establishment.models import Establishment


-class ReviewSerializer(serializers.ModelSerializer):
-    id = serializers.IntegerField()
-    reviewer_id = serializers.IntegerField()
+class ReviewSerializer(serializers.Serializer):
    vintage = serializers.IntegerField()
-    published = serializers.DateTimeField()
-    published_at = serializers.DateTimeField(allow_null=True)
+    mark = serializers.FloatField(allow_null=True)
    establishment_id = serializers.IntegerField()
-    text = serializers.CharField(allow_null=True, allow_blank=True)
-    locale = serializers.CharField()
+    created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
    aasm_state = serializers.CharField(allow_null=True)
-
-    class Meta:
-        model = Review
-        fields = (
-            "id", "reviewer_id", "published", "vintage",
-            "establishment_id", "text", "locale",
-            "published_at", "aasm_state"
-        )
+    reviewer_id = serializers.IntegerField()
+    id = serializers.IntegerField()

    def validate(self, data):
-        data = self.set_old_id(data)
-        data = self.set_published_at(data)
-        data = self.set_reviewer(data)
-        data = self.set_establishment(data)
-        data = self.set_language(data)
-        data = self.set_published(data)
+        data.update({
+            'reviewer': self.get_reviewer(data),
+            'status': Review.READY if data['aasm_state'] == 'published' else Review.TO_INVESTIGATE,
+            'published_at': data.pop('created_at'),
+            'old_id': data.pop('id'),
+            'content_object': self.get_establishment(data),
+        })
+        data.pop('reviewer_id')
+        data.pop('establishment_id')
+        data.pop('aasm_state')
        return data

    def create(self, validated_data):
-        try:
-            return Review.objects.create(**validated_data)
-        except Exception as e:
-            raise ValueError(f"Error creating review with {validated_data}: {e}")
+        obj, _ = Review.objects.update_or_create(
+            old_id=validated_data['old_id'],
+            defaults=validated_data,
+        )
+        return obj

-    def set_old_id(self, data):
-        data['old_id'] = data.pop("id")
+    @staticmethod
+    def get_reviewer(data):
+        user = User.objects.filter(old_id=data['reviewer_id']).first()
+        if not user:
+            raise ValueError(f"User account not found with old_id {data['reviewer_id']}")
+        return user
+
+    @staticmethod
+    def get_establishment(data):
+        establishment = Establishment.objects.filter(old_id=data['establishment_id']).first()
+        if not establishment:
+            raise ValueError(f"Establishment not found with old_id {data['establishment_id']}: ")
+        return establishment
+
+
+class ReviewTextSerializer(serializers.Serializer):
+    review_id = serializers.IntegerField()
+    locale = serializers.CharField(allow_null=True)
+    text = serializers.CharField()
+
+    def validate(self, data):
+        data.update({
+            'new_text': self.get_text(data),
+            'review': self.get_review(data),
+        })
        return data

-    def set_reviewer(self, data):
-        try:
-            data['reviewer'] = User.objects.get(old_id=data['reviewer_id'])
-        except User.DoesNotExist as e:
-            raise ValueError(f"Cannot find reviewer with {data}: {e}")
-        del(data['reviewer_id'])
-        return data
+    def create(self, validated_data):
+        review = validated_data['review']
+        if review.text:
+            review.text.update(validated_data['new_text'])
+        else:
+            review.text = validated_data['new_text']
+        review.save()
+        return review

-    def set_establishment(self, data):
-        try:
-            data['content_object'] = Establishment.objects.get(old_id=data.pop('establishment_id'))
-        except Establishment.DoesNotExist as e:
-            raise ValueError(f"Cannot find review establishment with {data}: {e}")
-        return data
+    @staticmethod
+    def get_text(data):
+        locale = data['locale'] or 'en-GB'
+        return {locale: data['text']}

-    def set_language(self, data):
-        try:
-            data['language'] = Language.objects.get(locale=data['locale'])
-        except Language.DoesNotExist as e:
-            raise ValueError(f"Cannot find language with {data}: {e}")
-
-        del(data['locale'])
-
-        return data
-
-    def set_published(self, data):
-        data['published_at'] = data.pop("published")
-        return data
-
-    def set_published_at(self, data):
-        if "aasm_state" in data and data['aasm_state'] is not None and data['aasm_state'] == "published":
-            data['status'] = Review.READY
-        del(data['aasm_state'])
-        return data
+    @staticmethod
+    def get_review(data):
+        review = Review.objects.filter(old_id=data['review_id']).first()
+        if not review:
+            raise ValueError(f"Review not found with old_id {data['review_id']}: ")
+        return review


 class LanguageSerializer(serializers.ModelSerializer):
-    
    LANGUAGES = {
        "be-BE": "Belarusian - Belarusia",
        "el-GR": "Greek - Greek",