refactor refactor reviews

apps/review/admin.py

@@ -8,4 +8,4 @@ from utils.admin import BaseModelAdminMixin
 class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin):
     """Admin model for model Review."""
 
-    raw_id_fields = ('reviewer', 'language', 'child', 'country')
+    raw_id_fields = ('reviewer', 'child', 'country')

apps/review/migrations/0016_remove_review_language.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.7 on 2019-11-14 07:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0015_review_mark'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='review',
+            name='language',
+        ),
+    ]

apps/review/models.py

@@ -39,38 +39,49 @@ class Review(BaseAttributes, TranslatedFieldsMixin):
         (READY, _('Ready')),
     )
 
-    reviewer = models.ForeignKey('account.User',
+    reviewer = models.ForeignKey(
+        'account.User',
         related_name='reviews',
         on_delete=models.CASCADE,
-                                 verbose_name=_('Reviewer'))
+        verbose_name=_('Reviewer'),
+    )
+    country = models.ForeignKey(
+        'location.Country',
+        on_delete=models.CASCADE,
+        related_name='country',
+        verbose_name=_('Country'),
+        null=True,
+    )
+    child = models.ForeignKey(
+        'self',
+        blank=True,
+        default=None,
+        null=True,
+        on_delete=models.CASCADE,
+        verbose_name=_('Child review'),
+    )
     text = TJSONField(
-        _('text'), null=True, blank=True,
+        _('text'),
-        default=None, help_text='{"en-GB":"Text review"}')
+        null=True,
+        blank=True,
+        default=None,
+        help_text='{"en-GB":"Text review"}',
+    )
     content_type = models.ForeignKey(generic.ContentType, on_delete=models.CASCADE)
     object_id = models.PositiveIntegerField()
     content_object = generic.GenericForeignKey('content_type', 'object_id')
-    language = models.ForeignKey('translation.Language',
+
-                                 on_delete=models.CASCADE,
-                                 related_name='reviews',
-                                 verbose_name=_('Review language'))
     status = models.PositiveSmallIntegerField(choices=REVIEW_STATUSES, default=TO_INVESTIGATE)
-    child = models.ForeignKey('self',
+    published_at = models.DateTimeField(
-                              blank=True, default=None, null=True,
+        _('Publish datetime'),
-                              on_delete=models.CASCADE,
+        blank=True,
-                              verbose_name=_('Child review'))
+        default=None,
-    published_at = models.DateTimeField(verbose_name=_('Publish datetime'),
+        null=True,
-                                        blank=True, default=None, null=True,
+        help_text=_('Review published datetime'),
-                                        help_text=_('Review published datetime'))
+    )
-    vintage = models.IntegerField(verbose_name=_('Year of review'),
+    vintage = models.IntegerField(_('Year of review'), validators=[MinValueValidator(1900), MaxValueValidator(2100)])
-                                  validators=[MinValueValidator(1900),
-                                              MaxValueValidator(2100)])
-
-    country = models.ForeignKey('location.Country', on_delete=models.CASCADE,
-                                related_name='country', verbose_name=_('Country'),
-                                null=True)
-
-    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
     mark = models.FloatField(verbose_name=_('mark'), blank=True, null=True, default=None)
+    old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
 
     objects = ReviewQuerySet.as_manager()
 

apps/review/transfer_data.py

@@ -1,15 +1,16 @@
-import json
 from pprint import pprint
 
 from django.db.models import Q
 
+from account.models import User
 from account.transfer_data import STOP_LIST
+from establishment.models import Establishment
 from review.models import Inquiries as NewInquiries, Review
 from transfer.models import Reviews, ReviewTexts, Inquiries, GridItems, InquiryPhotos
 from transfer.serializers.grid import GridItemsSerializer
 from transfer.serializers.inquiries import InquiriesSerializer
 from transfer.serializers.inquiry_gallery import InquiryGallerySerializer
-from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment
+from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, ReviewTextSerializer
 
 
 def transfer_languages():
@@ -28,77 +29,42 @@ def transfer_languages():
 
 
 def transfer_reviews():
-    # TODO: убрать LIKE UPPER("%%paris%%"), accounts.email IN
+    establishments = Establishment.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
-    queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, 
+    users = User.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
-                                    reviews.reviewer_id, review_texts.text AS text, reviews.mark, reviews.published_at,
+    queryset = Reviews.objects.filter(
-                                    review_texts.created_at AS published, review_texts.locale AS locale, 
+        establishment_id__in=list(establishments),
-                                    reviews.aasm_state
+        reviewer_id__in=list(users),
-                                    FROM reviews 
+    ).values('id', 'reviewer_id', 'aasm_state', 'created_at', 'establishment_id', 'mark', 'vintage')
-                                    LEFT OUTER JOIN review_texts 
-                                    ON (reviews.id = review_texts.review_id) 
-                                    WHERE reviews.reviewer_id > 0 
-                                    AND reviews.reviewer_id IS NOT NULL 
-                                    AND review_texts.text IS NOT NULL
-                                    AND review_texts.locale IS NOT NULL
-                                    AND reviews.mark IS NOT NULL
-                                    AND reviews.reviewer_id IN (
-                                      SELECT accounts.id 
-                                      FROM accounts 
-                                      WHERE accounts.confirmed_at IS NOT NULL 
-                                      AND NOT accounts.email IN (
-                                        "cyril@tomatic.net", 
-                                        "cyril2@tomatic.net", 
-                                        "cyril2@tomatic.net", 
-                                        "d.sadykova@id-east.ru", 
-                                        "d.sadykova@octopod.ru", 
-                                        "n.yurchenko@id-east.ru"
-                                      ))
-                                    AND reviews.establishment_id IN (
-                                      SELECT establishments.id 
-                                      FROM establishments 
-                                      INNER JOIN locations 
-                                      ON (establishments.location_id = locations.id) 
-                                      INNER JOIN cities 
-                                      ON (locations.city_id = cities.id) 
-                                      WHERE establishments.type IS NOT NULL AND locations.timezone IS NOT NULL
-                                      AND NOT establishments.type = "Wineyard"
-                                    )
-                                    ORDER BY review_texts.created_at DESC
-                                  """)
-
-    queryset_result = []
-    establishments_mark_list = {}
-
-    for query in queryset:
-        query = vars(query)
-        if query['establishment_id'] not in establishments_mark_list.keys():
-            if "aasm_state" in query and query['aasm_state'] is not None and query['aasm_state'] == "published":
-                establishments_mark_list[query['establishment_id']] = [
-                    int(query['mark']),
-                    json.dumps({query['locale']: query['text']})
-                ]
-            else:
-                establishments_mark_list[query['establishment_id']] = int(query['mark'])
-        del (query['mark'])
-        queryset_result.append(query)
-
-    serialized_data = ReviewSerializer(data=queryset_result, many=True)
 
+    serialized_data = ReviewSerializer(data=list(queryset.values()), many=True)
     if serialized_data.is_valid():
         serialized_data.save()
-
-        for establishment_id, mark in establishments_mark_list.items():
-            try:
-                establishment = Establishment.objects.get(old_id=establishment_id)
-            except Establishment.DoesNotExist:
-                continue
-            if isinstance(mark, list):
-                mark, review_text = mark
-
-            establishment.public_mark = mark
-            establishment.save()
     else:
-        pprint(serialized_data.errors)
+        pprint(f"ReviewSerializer serializer errors: {serialized_data.errors}")
+
+
+def transfer_text_review():
+    reviews = Review.objects.filter(old_id__isnull=False).values_list('old_id', flat=True)
+    queryset = ReviewTexts.objects.filter(
+        review_id__in=list(reviews),
+    ).exclude(
+        Q(text__isnull=True) | Q(text='')
+    ).values('review_id', 'locale', 'text')
+
+    serialized_data = ReviewTextSerializer(data=list(queryset.values()), many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"ReviewTextSerializer serializer errors: {serialized_data.errors}")
+
+    for review in Review.objects.filter(old_id__isnull=False):
+        text = review.text
+        if text and 'en-GB' not in text:
+            text.update({
+                'en-GB': next(iter(text.values()))
+            })
+            review.text = text
+            review.save()
 
 
 def transfer_inquiries():
@@ -139,8 +105,9 @@ def transfer_inquiry_photos():
 
 data_types = {
     "overlook": [
-        transfer_languages,
+        # transfer_languages,
-        transfer_reviews
+        transfer_reviews,
+        transfer_text_review,
     ],
     'inquiries': [
         transfer_inquiries,

apps/transfer/models.py

@@ -708,7 +708,7 @@ class Reviews(MigrateMixin):
     published_at = models.DateTimeField(blank=True, null=True)
     updated_at = models.DateTimeField()
     aasm_state = models.CharField(max_length=255, blank=True, null=True)
-    reviewer_id = models.IntegerField()
+    reviewer = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True)
     priority = models.IntegerField(blank=True, null=True)
     # TODO: модель Products в postgres закомментирована
     # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True)

apps/transfer/serializers/reviews.py

@@ -5,82 +5,85 @@ from translation.models import Language
 from establishment.models import Establishment
 
 
-class ReviewSerializer(serializers.ModelSerializer):
+class ReviewSerializer(serializers.Serializer):
-    id = serializers.IntegerField()
-    reviewer_id = serializers.IntegerField()
     vintage = serializers.IntegerField()
-    published = serializers.DateTimeField()
+    mark = serializers.FloatField(allow_null=True)
-    published_at = serializers.DateTimeField(allow_null=True)
     establishment_id = serializers.IntegerField()
-    text = serializers.CharField(allow_null=True, allow_blank=True)
+    created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
-    locale = serializers.CharField()
     aasm_state = serializers.CharField(allow_null=True)
-
+    reviewer_id = serializers.IntegerField()
-    class Meta:
+    id = serializers.IntegerField()
-        model = Review
-        fields = (
-            "id", "reviewer_id", "published", "vintage",
-            "establishment_id", "text", "locale",
-            "published_at", "aasm_state"
-        )
 
     def validate(self, data):
-        data = self.set_old_id(data)
+        data.update({
-        data = self.set_published_at(data)
+            'reviewer': self.get_reviewer(data),
-        data = self.set_reviewer(data)
+            'status': Review.READY if data['aasm_state'] == 'published' else Review.TO_INVESTIGATE,
-        data = self.set_establishment(data)
+            'published_at': data.pop('created_at'),
-        data = self.set_language(data)
+            'old_id': data.pop('id'),
-        data = self.set_published(data)
+            'content_object': self.get_establishment(data),
+        })
+        data.pop('reviewer_id')
+        data.pop('establishment_id')
+        data.pop('aasm_state')
         return data
 
     def create(self, validated_data):
-        try:
+        obj, _ = Review.objects.update_or_create(
-            return Review.objects.create(**validated_data)
+            old_id=validated_data['old_id'],
-        except Exception as e:
+            defaults=validated_data,
-            raise ValueError(f"Error creating review with {validated_data}: {e}")
+        )
+        return obj
 
-    def set_old_id(self, data):
+    @staticmethod
-        data['old_id'] = data.pop("id")
+    def get_reviewer(data):
+        user = User.objects.filter(old_id=data['reviewer_id']).first()
+        if not user:
+            raise ValueError(f"User account not found with old_id {data['reviewer_id']}")
+        return user
+
+    @staticmethod
+    def get_establishment(data):
+        establishment = Establishment.objects.filter(old_id=data['establishment_id']).first()
+        if not establishment:
+            raise ValueError(f"Establishment not found with old_id {data['establishment_id']}: ")
+        return establishment
+
+
+class ReviewTextSerializer(serializers.Serializer):
+    review_id = serializers.IntegerField()
+    locale = serializers.CharField(allow_null=True)
+    text = serializers.CharField()
+
+    def validate(self, data):
+        data.update({
+            'new_text': self.get_text(data),
+            'review': self.get_review(data),
+        })
         return data
 
-    def set_reviewer(self, data):
+    def create(self, validated_data):
-        try:
+        review = validated_data['review']
-            data['reviewer'] = User.objects.get(old_id=data['reviewer_id'])
+        if review.text:
-        except User.DoesNotExist as e:
+            review.text.update(validated_data['new_text'])
-            raise ValueError(f"Cannot find reviewer with {data}: {e}")
+        else:
-        del(data['reviewer_id'])
+            review.text = validated_data['new_text']
-        return data
+        review.save()
+        return review
 
-    def set_establishment(self, data):
+    @staticmethod
-        try:
+    def get_text(data):
-            data['content_object'] = Establishment.objects.get(old_id=data.pop('establishment_id'))
+        locale = data['locale'] or 'en-GB'
-        except Establishment.DoesNotExist as e:
+        return {locale: data['text']}
-            raise ValueError(f"Cannot find review establishment with {data}: {e}")
-        return data
 
-    def set_language(self, data):
+    @staticmethod
-        try:
+    def get_review(data):
-            data['language'] = Language.objects.get(locale=data['locale'])
+        review = Review.objects.filter(old_id=data['review_id']).first()
-        except Language.DoesNotExist as e:
+        if not review:
-            raise ValueError(f"Cannot find language with {data}: {e}")
+            raise ValueError(f"Review not found with old_id {data['review_id']}: ")
-
+        return review
-        del(data['locale'])
-
-        return data
-
-    def set_published(self, data):
-        data['published_at'] = data.pop("published")
-        return data
-
-    def set_published_at(self, data):
-        if "aasm_state" in data and data['aasm_state'] is not None and data['aasm_state'] == "published":
-            data['status'] = Review.READY
-        del(data['aasm_state'])
-        return data
 
 
 class LanguageSerializer(serializers.ModelSerializer):
-    
     LANGUAGES = {
         "be-BE": "Belarusian - Belarusia",
         "el-GR": "Greek - Greek",