refactor account transfer

apps/account/management/commands/add_account.py

@@ -2,8 +2,10 @@ from django.core.management.base import BaseCommand
 from django.db import connections
 from django.db.models import Q, F, Value
 from django.db.models.functions import ConcatPair
-from establishment.management.commands.add_position import namedtuplefetchall
+from tqdm import tqdm
+
 from account.models import User
+from establishment.management.commands.add_position import namedtuplefetchall
 
 
 class Command(BaseCommand):
@@ -12,7 +14,8 @@ class Command(BaseCommand):
     def account_sql(self):
         with connections['legacy'].cursor() as cursor:
             cursor.execute('''
-                            select a.email, a.id as account_id, a.encrypted_password, 
+                            select a.email, a.id as account_id, a.encrypted_password, a.locale, a.city,
+                            a.confirmed_at as cfd,
                             case when a.confirmed_at is not null then true else false end as confirmed_at,
                             case when a.confirmed_at is null then true else false end as unconfirmed_email,                            
                             nickname  
@@ -21,20 +24,24 @@ class Command(BaseCommand):
             ''')
             return namedtuplefetchall(cursor)
 
-    def handle(self,  *args, **kwargs):
+    def handle(self, *args, **kwargs):
         objects = []
-        for a in self.account_sql():
+        for a in tqdm(self.account_sql(), desc='find users'):
             users = User.objects.filter(Q(email=a.email) | Q(old_id=a.account_id))
             if not users.exists():
-                objects.append(User(email=a.email,
+                objects.append(User(
-                                    unconfirmed_email=a.unconfirmed_email,
+                    email=a.email,
-                                    email_confirmed=a.confirmed_at,
+                    unconfirmed_email=a.unconfirmed_email,
-                                    old_id=a.account_id,
+                    email_confirmed=a.confirmed_at,
-                                    password=a.encrypted_password,
+                    old_id=a.account_id,
-                                    username=a.nickname
+                    password=a.encrypted_password,
-                                    ))
+                    username=a.nickname,
+                    locale=a.locale,
+                    city=a.city,
+                    confirmed_at=a.cfd,
+                ))
 
         User.objects.bulk_create(objects)
         user = User.objects.filter(old_id__isnull=False)
         user.update(password=ConcatPair(Value('bcrypt$'), F('password')))
-        self.stdout.write(self.style.WARNING(f'Created accounts objects.'))
+        self.stdout.write(self.style.WARNING(f'Created {len(objects)} accounts objects.'))

apps/account/management/commands/add_affilations.py

@@ -1,18 +1,18 @@
-from account.models import OldRole, Role, User, UserRole
-from main.models import SiteSettings
 from django.core.management.base import BaseCommand
 from django.db import connections, transaction
-from django.db.models import Prefetch
-from establishment.management.commands.add_position import namedtuplefetchall
 from tqdm import tqdm
 
+from account.models import OldRole, Role, User, UserRole
+from establishment.management.commands.add_position import namedtuplefetchall
+from main.models import SiteSettings
+
 
 class Command(BaseCommand):
     help = '''Add site affilations from old db to new db. 
               Run after migrate account models!!!'''
 
     def map_role_sql(self):
-         with connections['legacy'].cursor() as cursor:
+        with connections['legacy'].cursor() as cursor:
             cursor.execute('''
                         select distinct 
                             case when role = 'news_editor' then 'CONTENT_PAGE_MANAGER'
@@ -76,14 +76,14 @@ class Command(BaseCommand):
                 if not role.exists():
                     objects.append(
                         Role(**data)
-                        )
+                    )
 
         Role.objects.bulk_create(objects)
         self.stdout.write(self.style.WARNING(f'Added site roles.'))
 
     def update_site_role(self):
-        roles = Role.objects.filter(country__isnull=True).select_related('site')\
+        roles = Role.objects.filter(country__isnull=True).select_related('site') \
-                            .filter(site__id__isnull=False).select_for_update()
+            .filter(site__id__isnull=False).select_for_update()
         with transaction.atomic():
             for role in tqdm(roles, desc='Update role country'):
                 role.country = role.site.country
@@ -150,4 +150,4 @@ class Command(BaseCommand):
         self.add_site_role()
         self.update_site_role()
         self.add_role_user()
-        self.add_superuser()
+        self.add_superuser()

apps/account/management/commands/add_confirmed_at.py

@@ -1,26 +0,0 @@
-from django.core.management.base import BaseCommand
-from tqdm import tqdm
-
-from account.models import User, UserRole, Role
-from transfer.models import OwnershipAffs, Accounts
-from establishment.models import Establishment
-
-
-class Command(BaseCommand):
-    help = """Add confirmed date to User."""
-
-    def handle(self, *args, **kwarg):
-        update_users = []
-
-        old_users = Accounts.objects.filter(confirmed_at__isnull=False)
-        for old_user in tqdm(old_users, desc='find users for update confirmed_at field'):
-            try:
-                user = User.objects.get(email=old_user.email)
-            except User.DoesNotExist:
-                continue
-            else:
-                user.confirmed_at = old_user.confirmed_at
-                update_users.append(user)
-
-        User.objects.bulk_update(update_users, ['confirmed_at', ])
-        self.stdout.write(self.style.WARNING(f'Updated users: {len(update_users)}'))

apps/account/management/commands/add_image.py

@@ -1,8 +1,8 @@
 from django.core.management.base import BaseCommand
 from django.db import connections
-from django.db.models import Q
+
-from establishment.management.commands.add_position import namedtuplefetchall
 from account.models import User
+from establishment.management.commands.add_position import namedtuplefetchall
 
 
 class Command(BaseCommand):
@@ -24,8 +24,8 @@ class Command(BaseCommand):
                             ''')
             return namedtuplefetchall(cursor)
 
-    def handle(self,  *args, **kwargs):
+    def handle(self, *args, **kwargs):
         for a in self.account_sql():
             users = User.objects.filter(old_id=a.account_id)
-            users.update(image_url= a.image_url)
+            users.update(image_url=a.image_url)
-        self.stdout.write(self.style.WARNING(f'Update accounts image url.'))
+        self.stdout.write(self.style.WARNING(f'Update accounts image url.'))

apps/account/management/commands/add_ownership.py

@@ -2,8 +2,8 @@ from django.core.management.base import BaseCommand
 from tqdm import tqdm
 
 from account.models import User, UserRole, Role
-from transfer.models import OwnershipAffs
 from establishment.models import Establishment
+from transfer.models import OwnershipAffs
 
 
 class Command(BaseCommand):

apps/account/management/commands/add_social.py

@@ -1,8 +1,9 @@
 from django.core.management.base import BaseCommand
 from django.db import connections
 from social_django.models import UserSocialAuth
-from establishment.management.commands.add_position import namedtuplefetchall
+
 from account.models import User
+from establishment.management.commands.add_position import namedtuplefetchall
 
 
 class Command(BaseCommand):

apps/account/transfer.py

@@ -1,53 +0,0 @@
-"""
-Структура fields:
-key - поле в таблице postgres
-value - поле или группа полей в таблице legacy
-
-В случае передачи группы полей каждое поле представляет собой кортеж, где:
-field[0] - название аргумента
-field[1] - название поля в таблице legacy
-Опционально: field[2] - тип данных для преобразования
-
-Структура внешних ключей:
-"legacy_table" - спикок кортежей для сопоставления полей
-"legacy_table": [
-    (("legacy_key", "legacy_field"),
-    ("psql_table", "psql_key", "psql_field", "psql_field_type"))
-], где:
-legacy_table - название модели legacy
-legacy_key - ForeignKey в legacy
-legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
-psql_table - название модели psql
-psql_key - ForeignKey в postgresql
-psql_field - уникальное поле в модели postgresql для сопоставления с legacy
-psql_field_type - тип уникального поля в postgresql
-
-"""
-
-card = {
-    "User": {
-        "data_type": "objects",
-        "dependencies": None,
-        "fields": {
-            "Accounts": {
-                "username": "nickname",
-                "email": "email",
-                "email_confirmed": ("confirmed_at", "django.db.models.BooleanField")
-            },
-            "relations": {
-                "Profiles": {
-                    "key": "account",
-                    "fields": {
-                        "first_name": "firstname",
-                        "last_name": "lastname"
-                    }
-                }
-            }
-        }
-    }
-
-}
-
-used_apps = None
-
-

apps/account/transfer_data.py

@@ -1,55 +0,0 @@
-from pprint import pprint
-
-from django.db.models import Q
-
-from transfer.models import Accounts, Identities
-from transfer.serializers.account import UserSerializer
-from transfer.serializers.user_social_auth import UserSocialAuthSerializer
-
-STOP_LIST = (
-    # 'cyril@tomatic.net',
-    # 'cyril2@tomatic.net',
-    # 'd.sadykova@id-east.ru',
-    # 'd.sadykova@octopod.ru',
-    # 'n.yurchenko@id-east.ru',
-)
-
-
-def transfer_user():
-    # queryset = Profiles.objects.all()
-    # queryset = queryset.annotate(nickname=F('account__nickname'))
-    # queryset = queryset.annotate(email=F('account__email'))
-
-    queryset = Accounts.objects.exclude(email__in=STOP_LIST)
-
-    serialized_data = UserSerializer(data=list(queryset.values()), many=True)
-
-    if serialized_data.is_valid():
-        serialized_data.save()
-    else:
-        pprint(f'News serializer errors: {serialized_data.errors}')
-
-
-def transfer_identities():
-    queryset = Identities.objects.exclude(
-        Q(account_id__isnull=True) |
-        # Q(account__confirmed_at__isnull=True) |
-        Q(account__email__in=STOP_LIST)
-    ).values_list(
-        'account_id',
-        'provider',
-        'uid',
-    )
-
-    serialized_data = UserSocialAuthSerializer(data=list(queryset.values()), many=True)
-
-    if serialized_data.is_valid():
-        serialized_data.save()
-    else:
-        pprint(f'UserSocialAuth serializer errors: {serialized_data.errors}')
-
-
-data_types = {
-    'account': [transfer_user],
-    'identities': [transfer_identities],
-}

apps/transfer/management/commands/transfer.py

@@ -12,7 +12,7 @@ class Command(BaseCommand):
     SHORT_DATA_TYPES = [
         'dictionaries',  # №2 - перенос стран, регионов, городов, адресов
         'news',  # перенос новостей (после №2)
-        'account',  # №1 - перенос пользователей
+        # 'account',  # №1 - перенос пользователей - нет, см make_data_migrations.sh !!!
         'subscriber',
         'recipe',  # №2 - рецепты
         'partner',

apps/transfer/serializers/account.py

@@ -1,40 +0,0 @@
-from rest_framework import serializers
-from account.models import User
-
-
-class UserSerializer(serializers.ModelSerializer):
-    nickname = serializers.CharField()
-    email = serializers.CharField()
-    confirmed_at = serializers.DateTimeField(allow_null=True)
-    id = serializers.CharField()
-
-    class Meta:
-        model = User
-
-        fields = (
-            "id",
-            "nickname",
-            "email",
-            "confirmed_at"
-        )
-
-    def validate(self, data):
-        data["old_id"] = data.pop("id")
-        data["username"] = self.get_username(data)
-        data["email_confirmed"] = self.get_email_confirmed(data)
-        data.pop("nickname")
-        data.pop("confirmed_at")
-        return data
-
-    def create(self, validated_data):
-        # использовать get_or_create
-        User.objects.create(**validated_data)
-
-    def get_email_confirmed(self, data):
-        if data.get("confirmed_at"):
-            return True
-        else:
-            return False
-
-    def get_username(self, obj):
-        return obj["email"]

apps/transfer/serializers/user_social_auth.py

@@ -1,30 +0,0 @@
-from rest_framework import serializers
-from social_django.models import UserSocialAuth
-
-from account.models import User
-
-
-class UserSocialAuthSerializer(serializers.Serializer):
-    account_id = serializers.IntegerField()
-    provider = serializers.CharField()
-    uid = serializers.CharField()
-
-    def validate(self, data):
-        data.update({
-            'user': self.get_account(data),
-        })
-        data.pop('account_id')
-        return data
-
-    def create(self, validated_data):
-        try:
-            return UserSocialAuth.objects.create(**validated_data)
-        except Exception as e:
-            raise ValueError(f"Error creating UserSocialAuth with {validated_data}: {e}")
-
-    @staticmethod
-    def get_account(data):
-        user = User.objects.filter(old_id=data['account_id']).first()
-        if not user:
-            raise ValueError(f"User account not found with old_id {data['account_id']}")
-        return user

make_data_migration.sh

@@ -1,5 +1,47 @@
 #!/usr/bin/env bash
-./manage.py transfer -a
+
+# ПОЛЬЗОВАТЕЛИ
+
+# Перенос пользователей из модели Accounts в User
+# --------------------------
+# id -> old_id
+# email -> email
+# unconfirmed_email -> unconfirmed_email
+# confirmed_at (boolean) -> email_confirmed
+# encrypted_password -> password
+# nickname -> username
+# locale -> locale
+# city -> city
+# confirmed_at -> confirmed_at
+./manage.py add_account
+
+# Добавление к уже перенесенным пользователям image_url по old_id
+# --------------------------
+# image_url -> image_url
+./manage.py add_image
+
+# Заполнение модели из identities в UserSocialAuth
+# --------------------------
+# пользователь -> user
+# provider -> provider
+# uid -> uid
+./manage.py add_social
+
+# Заполнение модели OldRole, UserRole (должны быть заполнены Role и SiteSettings) !!!
+# --------------------------
+# image_url -> image_url
+#./manage.py add_affilations
+
+# Заполнение модели из OwnershipAffs в UserRole (запускать после переноса заведений) !!!
+# --------------------------
+# user -> user,
+# role -> role,
+# establishment -> establishment,
+# owner.state -> state,
+# requester -> requester
+#./manage.py add_ownership
+
+
 ./manage.py transfer --setup_clean_db
 ./manage.py transfer -d
 ./manage.py transfer -e