Merge remote-tracking branch 'origin/feature/migrate-inquiries' into feature/migrate-inquiries

apps/account/management/commands/add_account.py

@@ -0,0 +1,43 @@
+from django.core.management.base import BaseCommand
+from django.db import connections
+from django.db.models import Q
+from establishment.management.commands.add_position import namedtuplefetchall
+from account.models import User
+
+
+class Command(BaseCommand):
+    help = 'Add account from old db to new db'
+
+    def account_sql(self):
+        with connections['legacy'].cursor() as cursor:
+            cursor.execute('''
+                            select a.email, a.id as account_id, a.encrypted_password
+                            from accounts as a 
+                            where a.email is not null 
+                             and a.email not in ('cyril@tomatic.net', 
+                                                 'cyril2@tomatic.net', 
+                                                 'd.sadykova@id-east.ru', 
+                                                 'd.sadykova@octopod.ru', 
+                                                 'n.yurchenko@id-east.ru'
+                                                 ) 
+                            and a.confirmed_at is not null     
+            ''')
+            return namedtuplefetchall(cursor)
+
+    def handle(self,  *args, **kwargs):
+        objects = []
+        for a in self.account_sql():
+            count = User.objects.filter(Q(email=a.email) | Q(old_id=a.account_id)).count()
+            if count == 0:
+                objects.append(User(email=a.email,
+                                    unconfirmed_email=False,
+                                    email_confirmed=True,
+                                    old_id=a.account_id,
+                                    password='bcrypt$'+a.encrypted_password
+                                    ))
+            else:
+               user = User.objects.filter(Q(email=a.email) | Q(old_id=a.account_id))
+               user.update(password='bcrypt$'+a.encrypted_password)
+
+        User.objects.bulk_create(objects)
+        self.stdout.write(self.style.WARNING(f'Created accounts objects.'))

apps/account/management/commands/add_image.py

@@ -0,0 +1,31 @@
+from django.core.management.base import BaseCommand
+from django.db import connections
+from django.db.models import Q
+from establishment.management.commands.add_position import namedtuplefetchall
+from account.models import User
+
+
+class Command(BaseCommand):
+    help = 'Update accounts image from old db to new db'
+
+    def account_sql(self):
+        with connections['legacy'].cursor() as cursor:
+            cursor.execute('''
+                                select	
+                                       url as image_url,
+                                       t.account_id
+                                from 
+                                (
+                                    select a.account_id,  a.attachment_file_name,  
+                                           trim(CONCAT(u.url, a.attachment_suffix_url))  as url 
+                                    from account_pictures a, 
+                                    (select  'https://s3.eu-central-1.amazonaws.com/gm-test.com/media/' as url) u  
+                                ) t  
+                            ''')
+            return namedtuplefetchall(cursor)
+
+    def handle(self,  *args, **kwargs):
+        for a in self.account_sql():
+            users = User.objects.filter(old_id=a.account_id)
+            users.update(image_url= a.image_url)
+        self.stdout.write(self.style.WARNING(f'Update accounts image url.'))

apps/account/management/commands/add_social.py

@@ -0,0 +1,51 @@
+from django.core.management.base import BaseCommand
+from django.db import connections
+from django.db.models import Q
+from social_django.models import UserSocialAuth
+from establishment.management.commands.add_position import namedtuplefetchall
+from account.models import User
+
+
+class Command(BaseCommand):
+    help = '''Add account social networks from old db to new db. 
+              Run after add_account!!!'''
+
+    def social_sql(self):
+        with connections['legacy'].cursor() as cursor:
+            cursor.execute('''
+                                select 
+                                    DISTINCT
+                                    i.account_id, i.provider, i.uid 
+                                from 
+                                (
+                                    select a.email, a.id as account_id
+                                    from accounts as a 
+                                    where a.email is not null 
+                                     and a.email not in ('cyril@tomatic.net', 
+                                                         'cyril2@tomatic.net', 
+                                                         'd.sadykova@id-east.ru', 
+                                                         'd.sadykova@octopod.ru', 
+                                                         'n.yurchenko@id-east.ru'
+                                                         ) 
+                                    and a.confirmed_at is not null 
+                                ) t
+                                join identities i on i.account_id = t.account_id	
+            ''')
+            return namedtuplefetchall(cursor)
+
+    def handle(self, *args, **kwargs):
+        objects = []
+        for s in self.social_sql():
+            user = User.objects.filter(old_id=s.account_id)
+            if user.count() > 0:
+                social = UserSocialAuth.objects.filter(user=user.first(),
+                                                      provider=s.provider,
+                                                      uid=s.uid)
+                if social.count() == 0:
+                    objects.append(UserSocialAuth(user=user.first(), provider=s.provider,
+                                                  uid=s.uid)
+                                                 )
+                    print('INSERT')
+
+        UserSocialAuth.objects.bulk_create(objects)
+        self.stdout.write(self.style.WARNING(f'Created social objects.'))

apps/account/migrations/0019_auto_20191108_0827.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-11-08 08:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('account', '0018_user_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='image_url',
+            field=models.URLField(blank=True, default=None, max_length=500, null=True, verbose_name='Image URL path'),
+        ),
+    ]

apps/account/models.py

@@ -89,7 +89,8 @@ class UserQuerySet(models.QuerySet):
 class User(AbstractUser):
     """Base user model."""
     image_url = models.URLField(verbose_name=_('Image URL path'),
-                                blank=True, null=True, default=None)
+                                blank=True, null=True, default=None,
+                                max_length=500)
     cropped_image_url = models.URLField(verbose_name=_('Cropped image URL path'),
                                         blank=True, null=True, default=None)
     email = models.EmailField(_('email address'), unique=True,

apps/establishment/management/commands/add_position.py

@@ -24,7 +24,6 @@ class Command(BaseCommand):
             return namedtuplefetchall(cursor)
 
     def handle(self,  *args, **kwargs):
-
         objects = []
         for p in self.position_sql():
             count = Position.objects.filter(name={"en-GB": p.position_name}).count()

apps/review/migrations/0009_auto_20191110_0615.py

@@ -0,0 +1,44 @@
+# Generated by Django 2.2.4 on 2019-11-10 06:15
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gallery', '0006_merge_20191027_1758'),
+        ('review', '0008_auto_20191108_0927'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='inquiries',
+            name='attachment_content_type',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='inquiries',
+            name='bill_content_type',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.CreateModel(
+            name='InquiriesGallery',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_main', models.BooleanField(default=False, verbose_name='Is the main image')),
+                ('image', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='inquiries_gallery', to='gallery.Image', verbose_name='gallery')),
+                ('inquiry', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='inquiries_gallery', to='review.Inquiries', verbose_name='inquiry')),
+            ],
+            options={
+                'verbose_name': 'inquiry gallery',
+                'verbose_name_plural': 'inquiry galleries',
+                'unique_together': {('inquiry', 'is_main'), ('inquiry', 'image')},
+            },
+        ),
+        migrations.AddField(
+            model_name='inquiries',
+            name='gallery',
+            field=models.ManyToManyField(through='review.InquiriesGallery', to='gallery.Image'),
+        ),
+    ]

apps/review/models.py

@@ -94,15 +94,18 @@ class Inquiries(ProjectBaseMixin):
     final_comment = models.TextField(_('final comment'), blank=True, null=True)
     mark = models.FloatField(verbose_name=_('mark'), blank=True, null=True, default=None)
     attachment_file = models.URLField(verbose_name=_('attachment'), max_length=255, blank=True, null=True, default=None)
+    attachment_content_type = models.CharField(max_length=255, blank=True, null=True)
     author = models.ForeignKey('account.User', related_name='incuiries', on_delete=models.CASCADE,
                                verbose_name=_('author'))
     bill_file = models.URLField(verbose_name=_('bill'), max_length=255, blank=True, null=True, default=None)
+    bill_content_type = models.CharField(max_length=255, blank=True, null=True)
     price = models.DecimalField(_('price'), max_digits=7, decimal_places=2, blank=True, null=True)
     moment = models.PositiveSmallIntegerField(choices=MOMENTS, default=NONE)
-
+    gallery = models.ManyToManyField('gallery.Image', through='review.InquiriesGallery')
     decibels = models.CharField(max_length=255, blank=True, null=True)
     nomination = models.CharField(max_length=255, blank=True, null=True)
     nominee = models.CharField(max_length=255, blank=True, null=True)
+    published = models.BooleanField(_('is published'), default=False)
 
     class Meta:
         verbose_name = _('Inquiry')
@@ -126,3 +129,36 @@ class GridItems(ProjectBaseMixin):
 
     def __str__(self):
         return f'inquiry: {self.inquiry.id}, grid id: {self.id}'
+
+
+class InquiriesGalleryQuerySet(models.QuerySet):
+    """QuerySet for model Inquiries"""
+
+    def main_image(self):
+        """Return objects with flag is_main is True"""
+        return self.filter(is_main=True)
+
+
+class InquiriesGallery(models.Model):
+    inquiry = models.ForeignKey(
+        Inquiries,
+        null=True,
+        related_name='inquiries_gallery',
+        on_delete=models.CASCADE,
+        verbose_name=_('inquiry'),
+    )
+    image = models.ForeignKey(
+        'gallery.Image',
+        null=True,
+        related_name='inquiries_gallery',
+        on_delete=models.CASCADE,
+        verbose_name=_('gallery'),
+    )
+    is_main = models.BooleanField(default=False, verbose_name=_('Is the main image'))
+
+    objects = InquiriesGalleryQuerySet.as_manager()
+
+    class Meta:
+        verbose_name = _('inquiry gallery')
+        verbose_name_plural = _('inquiry galleries')
+        unique_together = (('inquiry', 'is_main'), ('inquiry', 'image'))

apps/review/transfer_data.py

@@ -1,3 +1,6 @@
+from django.db.models import Q
+
+from account.transfer_data import STOP_LIST
 from transfer.models import Reviews, ReviewTexts, Inquiries
 from transfer.serializers.inquiries import InquiriesSerializer
 from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment
@@ -95,7 +98,19 @@ def transfer_reviews():
 
 
 def transfer_inquiries():
-    inquiries = Inquiries.objects.all()
+    # отфильтровать по review, account, establishment
+    # TODO: нужно ли переносить данные у которых нет привязки к аккаунту? .filter(account__isnull=False)
+    inquiries = Inquiries.objects.exclude(
+        Q(review__reviewer_id__gt=0) |
+        Q(review__reviewer_id__isnull=True) |
+        Q(review__mark__isnull=True) |
+        Q(review__reviewtexts__text__isnull=True) |
+        Q(review__reviewtexts__locale__isnull=True) |
+        Q(review__establishment__type='Wineyard') |
+        Q(review__establishment__location__timezone__isnull=True) |
+        Q(account__confirmed_at__isnull=True) |
+        Q(account__email__in=STOP_LIST)
+    )
 
     serialized_data = InquiriesSerializer(data=list(inquiries.values()), many=True)
     if serialized_data.is_valid():

apps/transfer/models.py

@@ -1025,7 +1025,7 @@ class Inquiries(MigrateMixin):
     bill_file_size = models.IntegerField(blank=True, null=True)
     bill_updated_at = models.DateTimeField(blank=True, null=True)
     bill_suffix_url = models.TextField(blank=True, null=True)
-    price = models.DecimalField(max_digits=10, decimal_places=2, blank=True, null=True)
+    price = models.DecimalField(max_digits=7, decimal_places=2, blank=True, null=True)
     moment = models.CharField(max_length=255, blank=True, null=True)
     published = models.PositiveSmallIntegerField(blank=True, null=True)
     menu_id = models.IntegerField(blank=True, null=True)

apps/transfer/serializers/inquiries.py

@@ -1,28 +1,48 @@
 from rest_framework import serializers
 
-from review.models import Inquiries
+from account.models import User
+from review.models import Inquiries, Review
 
 
 class InquiriesSerializer(serializers.Serializer):
     id = serializers.IntegerField()
-    comment = serializers.CharField()
+    comment = serializers.CharField(allow_null=True, allow_blank=True)
-    mark = serializers.DecimalField(max_digits=4, decimal_places=2)
+    final_comment = serializers.CharField(allow_null=True, allow_blank=True)
-    locale = serializers.CharField()
+    mark = serializers.FloatField(allow_null=True, allow_blank=True)
-    account_id = serializers.IntegerField()
+    created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S')
-    establishment_id = serializers.CharField()
+    review_id = serializers.IntegerField()
+
+    attachment_content_type = serializers.CharField(allow_null=True, allow_blank=True)
+    attachment_suffix_url = serializers.CharField(allow_null=True, allow_blank=True)
+
+    account_id = serializers.IntegerField(allow_null=True, allow_blank=True)
+
+    bill_content_type = serializers.CharField(allow_null=True, allow_blank=True)
+    bill_suffix_url = serializers.CharField(allow_null=True, allow_blank=True)
+
+    price = serializers.DecimalField(max_digits=7, decimal_places=2, blank=True, null=True)
+    moment = serializers.CharField(allow_null=True, allow_blank=True)
+    published = serializers.IntegerField(allow_null=True, allow_blank=True)
+
+    decibels = serializers.CharField(blank=True, null=True)
+    nomination = serializers.CharField(blank=True, null=True)
+    nominee = serializers.CharField(blank=True, null=True)
 
     def validate(self, data):
         data.update({
             'old_id': data.pop('id'),
-            'text': data.pop('comment'),
+            'review': self.get_review(data),
-            'mark': data['mark'] * -1 if data['mark'] < 0 else data['mark'],
+            'created': data.pop('created_at'),
-            'content_object': self.get_content_object(data),
+            'published': bool(data['published']),
-            'user': self.get_account(data),
+            'moment': self.get_moment(data),
-            'country': self.get_country(data),
+            'author': self.get_author(data),
+            'attachment_file': data['attachment_suffix_url'] if data['attachment_content_type'] else '',
+            'bill_file': data['bill_suffix_url'] if data['bill_content_type'] else '',
         })
-        data.pop('establishment_id')
+        data.pop('review_id')
         data.pop('account_id')
-        data.pop('locale')
+        data.pop('attachment_suffix_url')
+        data.pop('bill_suffix_url')
         return data
 
     def create(self, validated_data):
@@ -31,9 +51,25 @@ class InquiriesSerializer(serializers.Serializer):
         except Exception as e:
             raise ValueError(f"Error creating Inquiries with {validated_data}: {e}")
 
-    # @staticmethod
+    @staticmethod
-    # def get_content_object(data):
+    def get_review(data):
-    #     establishment = Establishment.objects.filter(old_id=data['establishment_id']).first()
+        review = Review.objects.filter(old_id=data['review_id']).first()
-    #     if not establishment:
+        if not review:
-    #         raise ValueError(f"Establishment not found with old_id {data['establishment_id']}: ")
+            raise ValueError(f"Review not found with old_id {data['review_id']}")
-    #     return establishment
+        return review
+
+    @staticmethod
+    def get_moment(data):
+        moments = {
+            'lanch': Inquiries.LUNCH,
+            'diner': Inquiries.DINER,
+        }
+        moment = data['moment']
+        return moments.get(moment, Inquiries.NONE)
+
+    @staticmethod
+    def get_author(data):
+        user = User.objects.filter(old_id=data['account_id']).first()
+        if not user:
+            raise ValueError(f"User account not found with old_id {data['account_id']}")
+        return user

project/settings/base.py

@@ -188,6 +188,14 @@ AUTH_PASSWORD_VALIDATORS = [
     },
 ]
 
+PASSWORD_HASHERS = [
+    'django.contrib.auth.hashers.PBKDF2PasswordHasher',
+    'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher',
+    'django.contrib.auth.hashers.Argon2PasswordHasher',
+    'django.contrib.auth.hashers.BCryptPasswordHasher',
+    'django.contrib.auth.hashers.BCryptSHA256PasswordHasher',
+]
+
 # Account settings
 AUTH_USER_MODEL = 'account.User'
 LOGIN_URL = 'admin:login'

project/settings/local.py

@@ -73,6 +73,28 @@ LOGGING = {
 }
 
 
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.contrib.gis.db.backends.postgis',
+        'NAME': os.environ.get('DB_NAME'),
+        'USER': os.environ.get('DB_USERNAME'),
+        'PASSWORD': os.environ.get('DB_PASSWORD'),
+        'HOST': os.environ.get('DB_HOSTNAME'),
+        'PORT': os.environ.get('DB_PORT'),
+    },
+    'legacy': {
+        'ENGINE': 'django.db.backends.mysql',
+        # 'HOST': '172.17.0.1',
+        # 'HOST': '172.23.0.1',
+        'HOST': 'mysql_db',
+        'PORT': 3306,
+        'NAME': 'dev',
+        'USER': 'dev',
+        'PASSWORD': 'octosecret123'
+    }
+}
+
 # ELASTICSEARCH SETTINGS
 ELASTICSEARCH_DSL = {
     'default': {

requirements/base.txt

@@ -1,4 +1,4 @@
-Django==2.2.4
+Django[bcrypt]==2.2.7
 psycopg2-binary==2.8.3
 pytz==2019.1
 sqlparse==0.3.0

requirements/development.txt

@@ -1,4 +1,4 @@
 -r base.txt
 ipdb
 ipython
-mysqlclient
+mysqlclient==1.4.4