From e8dcaeaa9a56131ccbb3f9f5f024d36274d97637 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 1/5] Fix validators --- apps/authorization/serializers/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/authorization/serializers/common.py b/apps/authorization/serializers/common.py index 13121f78..d99a2dc2 100644 --- a/apps/authorization/serializers/common.py +++ b/apps/authorization/serializers/common.py @@ -38,13 +38,13 @@ class SignupSerializer(serializers.ModelSerializer): valid = utils_methods.username_validator(username=value) if not valid: raise utils_exceptions.NotValidUsernameError() - if account_models.User.objects.filter(username__icontains=value).exists(): + if account_models.User.objects.filter(username__ilike=value).exists(): raise serializers.ValidationError() return value def validate_email(self, value): """Validate email""" - if account_models.User.objects.filter(email__icontains=value).exists(): + if account_models.User.objects.filter(email__ilike=value).exists(): raise serializers.ValidationError() return value From a372d29677d6cae3d07b73a7dcc4cf71c4fdc99a Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 6 Nov 2019 14:24:14 +0300 Subject: [PATCH 2/5] refactor comment migration --- apps/account/transfer_data.py | 16 +++-- apps/comment/admin.py | 1 + apps/comment/models.py | 15 ++--- apps/comment/transfer_data.py | 26 +++++++- apps/establishment/transfer_data.py | 1 - apps/transfer/serializers/comments.py | 91 +++++++++------------------ 6 files changed, 70 insertions(+), 80 deletions(-) diff --git a/apps/account/transfer_data.py b/apps/account/transfer_data.py index c39e0ac1..41d430fa 100644 --- a/apps/account/transfer_data.py +++ b/apps/account/transfer_data.py @@ -3,17 +3,21 @@ from pprint import pprint from transfer.models import Profiles, Accounts from transfer.serializers.account import UserSerializer +STOP_LIST = ( + 'cyril@tomatic.net', + 'cyril2@tomatic.net', + 'd.sadykova@id-east.ru', + 'd.sadykova@octopod.ru', + 'n.yurchenko@id-east.ru', +) + def transfer_user(): # queryset = Profiles.objects.all() # queryset = queryset.annotate(nickname=F("account__nickname")) # queryset = queryset.annotate(email=F("account__email")) - stop_list = ['cyril@tomatic.net', - 'cyril2@tomatic.net', - 'd.sadykova@id-east.ru', - 'd.sadykova@octopod.ru', - 'n.yurchenko@id-east.ru'] - queryset = Accounts.objects.filter(confirmed_at__isnull=False).exclude(email__in=stop_list) + + queryset = Accounts.objects.filter(confirmed_at__isnull=False).exclude(email__in=STOP_LIST) serialized_data = UserSerializer(data=list(queryset.values()), many=True) diff --git a/apps/comment/admin.py b/apps/comment/admin.py index 855f6b3e..061c9c8d 100644 --- a/apps/comment/admin.py +++ b/apps/comment/admin.py @@ -6,3 +6,4 @@ from . import models @admin.register(models.Comment) class CommentModelAdmin(admin.ModelAdmin): """Model admin for model Comment""" + raw_id_fields = ('user', 'country') diff --git a/apps/comment/models.py b/apps/comment/models.py index 29021677..3a8c8c37 100644 --- a/apps/comment/models.py +++ b/apps/comment/models.py @@ -32,21 +32,16 @@ class CommentQuerySet(ContentTypeQuerySetMixin): class Comment(ProjectBaseMixin): """Comment model.""" text = models.TextField(verbose_name=_('Comment text')) - mark = models.PositiveIntegerField(blank=True, null=True, default=None, - verbose_name=_('Mark')) - user = models.ForeignKey('account.User', - related_name='comments', - on_delete=models.CASCADE, - verbose_name=_('User')) + mark = models.PositiveIntegerField(blank=True, null=True, default=None, verbose_name=_('Mark')) + user = models.ForeignKey('account.User', related_name='comments', on_delete=models.CASCADE, verbose_name=_('User')) + country = models.ForeignKey(Country, verbose_name=_('Country'), on_delete=models.SET_NULL, null=True) + old_id = models.IntegerField(null=True, blank=True, default=None) + content_type = models.ForeignKey(generic.ContentType, on_delete=models.CASCADE) object_id = models.PositiveIntegerField() content_object = generic.GenericForeignKey('content_type', 'object_id') objects = CommentQuerySet.as_manager() - country = models.ForeignKey(Country, verbose_name=_('Country'), - on_delete=models.SET_NULL, null=True) - old_id = models.IntegerField(null=True, blank=True, default=None) - class Meta: """Meta class""" diff --git a/apps/comment/transfer_data.py b/apps/comment/transfer_data.py index a9ac4f35..843da78f 100644 --- a/apps/comment/transfer_data.py +++ b/apps/comment/transfer_data.py @@ -1,11 +1,31 @@ from pprint import pprint + +from django.db.models import Q + +from account.transfer_data import STOP_LIST from transfer.models import Comments from transfer.serializers.comments import CommentSerializer def transfer_comments(): - queryset = Comments.objects.filter(account__isnull=False, mark__isnull=False)\ - .only("id", "comment", "mark", "locale", "establishment_id", "account_id") + # В queryset исключены объекты по условию в связанные моделях + # см. transfer_establishment() и transfer_user() + queryset = Comments.objects.exclude( + Q(establishment__type='Wineyard') | + Q(establishment__location__timezone__isnull=True) | + Q(account__confirmed_at__isnull=True) | + Q(account__email__in=STOP_LIST) + ).filter( + account__isnull=False, + mark__isnull=False + ).only( + 'id', + 'comment', + 'mark', + 'locale', + 'establishment_id', + 'account_id', + ) serialized_data = CommentSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): @@ -15,7 +35,7 @@ def transfer_comments(): data_types = { - "comment": [ + 'comment': [ transfer_comments ] } diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index b178b18a..15f9524d 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -1,6 +1,5 @@ from pprint import pprint -import requests from django.db.models import Q, F from establishment.models import Establishment diff --git a/apps/transfer/serializers/comments.py b/apps/transfer/serializers/comments.py index 14ad70e3..cd4233d1 100644 --- a/apps/transfer/serializers/comments.py +++ b/apps/transfer/serializers/comments.py @@ -4,7 +4,7 @@ from establishment.models import Establishment from location.models import Country -class CommentSerializer(serializers.ModelSerializer): +class CommentSerializer(serializers.Serializer): id = serializers.IntegerField() comment = serializers.CharField() mark = serializers.DecimalField(max_digits=4, decimal_places=2) @@ -12,25 +12,18 @@ class CommentSerializer(serializers.ModelSerializer): account_id = serializers.IntegerField() establishment_id = serializers.CharField() - class Meta: - model = Comment - fields = ( - "id", - "comment", - "mark", - "locale", - "account_id", - "establishment_id" - ) - def validate(self, data): - data = self.set_old_id(data) - data = self.set_text(data) - data = self.set_mark(data) - data = self.set_establishment(data) - data = self.set_account(data) - data = self.set_country(data) - + data.update({ + 'old_id': data.pop('id'), + 'text': data.pop('comment'), + 'mark': data['mark'] * -1 if data['mark'] < 0 else data['mark'], + 'content_object': self.get_content_object(data), + 'user': self.get_account(data), + 'country': self.get_country(data), + }) + data.pop('establishment_id') + data.pop('account_id') + data.pop('locale') return data def create(self, validated_data): @@ -39,47 +32,25 @@ class CommentSerializer(serializers.ModelSerializer): except Exception as e: raise ValueError(f"Error creating comment with {validated_data}: {e}") - def set_text(self, data): - data['text'] = data.pop('comment') - return data + @staticmethod + def get_content_object(data): + establishment = Establishment.objects.filter(old_id=data['establishment_id']).first() + if not establishment: + raise ValueError(f"Establishment not found with old_id {data['establishment_id']}: ") + return establishment - def set_mark(self, data): - if data['mark'] < 0: - data['mark'] = data['mark'] * -1 - return data + @staticmethod + def get_account(data): + user = User.objects.filter(old_id=data['account_id']).first() + if not user: + raise ValueError(f"User account not found with old_id {data['account_id']}") + return user - def set_account(self, data): - try: - data['user'] = User.objects.filter(old_id=data['account_id']).first() - except User.DoesNotExist as e: - raise ValueError(f"User account not found with {data}: {e}") - - del(data['account_id']) - - return data - - def set_establishment(self, data): - try: - data['content_object'] = Establishment.objects.filter(old_id=data['establishment_id']).first() - except Establishment.DoesNotExist as e: - raise ValueError(f"Establishment not found with {data}: {e}") - # print(f"Establishment not found with {data}: {e}") - - del(data['establishment_id']) - - return data - - def set_old_id(self, data): - data['old_id'] = data.pop("id") - return data - - def set_country(self, data): - locale = data.pop("locale") + @staticmethod + def get_country(data): + locale = data['locale'] country_code = locale[:locale.index("-")] if len(locale) > 2 else locale - - try: - data['country'] = Country.objects.filter(code=country_code).first() - except Country.DoesNotExist as e: - raise ValueError(f"Country not found with {data}: {e}") - - return data \ No newline at end of file + country = Country.objects.filter(code=country_code).first() + if not country: + raise ValueError(f"Country not found with code {country_code}") + return country From d717b091537e156f883e877acb6ac56d3c877a33 Mon Sep 17 00:00:00 2001 From: Semyon Yekhmenin Date: Wed, 6 Nov 2019 11:35:43 +0000 Subject: [PATCH 3/5] Fixed nearest establishments search --- apps/establishment/views/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 05391827..b01c8c49 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -162,7 +162,7 @@ class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIVi qs = super(EstablishmentNearestRetrieveView, self).get_queryset() if lat and lon and radius and unit: - center = Point(float(lat), float(lon)) + center = Point(float(lon), float(lat)) filter_kwargs = {'center': center, 'radius': float(radius), 'unit': unit} return qs.by_distance_from_point(**{k: v for k, v in filter_kwargs.items() if v is not None}) From 9b16f46215f4a9b6c108cd71ccdc288f2baf2515 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 6 Nov 2019 15:12:33 +0300 Subject: [PATCH 4/5] migrate identities --- apps/account/transfer_data.py | 35 +++++++++++++++---- apps/transfer/management/commands/transfer.py | 1 + apps/transfer/models.py | 14 ++++++++ apps/transfer/serializers/user_social_auth.py | 30 ++++++++++++++++ 4 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 apps/transfer/serializers/user_social_auth.py diff --git a/apps/account/transfer_data.py b/apps/account/transfer_data.py index 41d430fa..75e0d8ee 100644 --- a/apps/account/transfer_data.py +++ b/apps/account/transfer_data.py @@ -1,7 +1,10 @@ -from django.db.models import Value, IntegerField, F from pprint import pprint -from transfer.models import Profiles, Accounts + +from django.db.models import Q + +from transfer.models import Accounts, Identities from transfer.serializers.account import UserSerializer +from transfer.serializers.user_social_auth import UserSocialAuthSerializer STOP_LIST = ( 'cyril@tomatic.net', @@ -14,8 +17,8 @@ STOP_LIST = ( def transfer_user(): # queryset = Profiles.objects.all() - # queryset = queryset.annotate(nickname=F("account__nickname")) - # queryset = queryset.annotate(email=F("account__email")) + # queryset = queryset.annotate(nickname=F('account__nickname')) + # queryset = queryset.annotate(email=F('account__email')) queryset = Accounts.objects.filter(confirmed_at__isnull=False).exclude(email__in=STOP_LIST) @@ -24,9 +27,29 @@ def transfer_user(): if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + pprint(f'News serializer errors: {serialized_data.errors}') + + +def transfer_identities(): + queryset = Identities.objects.exclude( + Q(account_id__isnull=True) | + Q(account__confirmed_at__isnull=True) | + Q(account__email__in=STOP_LIST) + ).values_list( + 'account_id', + 'provider', + 'uid', + ) + + serialized_data = UserSocialAuthSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f'UserSocialAuth serializer errors: {serialized_data.errors}') data_types = { - "account": [transfer_user] + 'account': [transfer_user], + 'identities': [transfer_identities], } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index e75deb32..d2387d0d 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -23,6 +23,7 @@ class Command(BaseCommand): 'menu', 'location_establishment', 'whirligig', + 'identities', ] LONG_DATA_TYPES = [ diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 599445c2..8527eeb6 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -987,3 +987,17 @@ class MetadatumAliases(MigrateMixin): class Meta: managed = False db_table = 'metadatum_aliases' + + +class Identities(MigrateMixin): + using = 'legacy' + + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + provider = models.CharField(max_length=255, blank=True, null=True) + uid = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'identities' diff --git a/apps/transfer/serializers/user_social_auth.py b/apps/transfer/serializers/user_social_auth.py new file mode 100644 index 00000000..dc260fa0 --- /dev/null +++ b/apps/transfer/serializers/user_social_auth.py @@ -0,0 +1,30 @@ +from rest_framework import serializers +from social_django.models import UserSocialAuth + +from account.models import User + + +class UserSocialAuthSerializer(serializers.Serializer): + account_id = serializers.IntegerField() + provider = serializers.CharField() + uid = serializers.CharField() + + def validate(self, data): + data.update({ + 'user': self.get_account(data), + }) + data.pop('account_id') + return data + + def create(self, validated_data): + try: + return UserSocialAuth.objects.create(**validated_data) + except Exception as e: + raise ValueError(f"Error creating UserSocialAuth with {validated_data}: {e}") + + @staticmethod + def get_account(data): + user = User.objects.filter(old_id=data['account_id']).first() + if not user: + raise ValueError(f"User account not found with old_id {data['account_id']}") + return user From 82d238c123bf5b2ea526d732e8ff31d74cbd5220 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Wed, 6 Nov 2019 15:25:59 +0300 Subject: [PATCH 5/5] fiz settings --- project/settings/development.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/project/settings/development.py b/project/settings/development.py index 16297f40..fd832329 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -50,14 +50,14 @@ DATABASES = { 'HOST': os.environ.get('DB_HOSTNAME'), 'PORT': os.environ.get('DB_PORT'), }, - # 'legacy': { - # 'ENGINE': 'django.db.backends.mysql', - # 'HOST': os.environ.get('MYSQL_HOSTNAME'), - # 'PORT': os.environ.get('MYSQL_PORT'), - # 'NAME': os.environ.get('MYSQL_DATABASE'), - # 'USER': os.environ.get('MYSQL_USER'), - # 'PASSWORD': os.environ.get('MYSQL_PASSWORD') - # } + 'legacy': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': os.environ.get('MYSQL_HOSTNAME'), + 'PORT': os.environ.get('MYSQL_PORT'), + 'NAME': os.environ.get('MYSQL_DATABASE'), + 'USER': os.environ.get('MYSQL_USER'), + 'PASSWORD': os.environ.get('MYSQL_PASSWORD') + } } BROKER_URL = 'redis://localhost:6379/1'