diff --git a/apps/account/management/commands/add_account.py b/apps/account/management/commands/add_account.py index 2e5842c5..fce76f06 100644 --- a/apps/account/management/commands/add_account.py +++ b/apps/account/management/commands/add_account.py @@ -2,8 +2,10 @@ from django.core.management.base import BaseCommand from django.db import connections from django.db.models import Q, F, Value from django.db.models.functions import ConcatPair -from establishment.management.commands.add_position import namedtuplefetchall +from tqdm import tqdm + from account.models import User +from establishment.management.commands.add_position import namedtuplefetchall class Command(BaseCommand): @@ -12,7 +14,8 @@ class Command(BaseCommand): def account_sql(self): with connections['legacy'].cursor() as cursor: cursor.execute(''' - select a.email, a.id as account_id, a.encrypted_password, + select a.email, a.id as account_id, a.encrypted_password, a.locale, a.city, + a.confirmed_at as cfd, case when a.confirmed_at is not null then true else false end as confirmed_at, case when a.confirmed_at is null then true else false end as unconfirmed_email, nickname @@ -21,20 +24,24 @@ class Command(BaseCommand): ''') return namedtuplefetchall(cursor) - def handle(self, *args, **kwargs): + def handle(self, *args, **kwargs): objects = [] - for a in self.account_sql(): + for a in tqdm(self.account_sql(), desc='find users'): users = User.objects.filter(Q(email=a.email) | Q(old_id=a.account_id)) if not users.exists(): - objects.append(User(email=a.email, - unconfirmed_email=a.unconfirmed_email, - email_confirmed=a.confirmed_at, - old_id=a.account_id, - password=a.encrypted_password, - username=a.nickname - )) + objects.append(User( + email=a.email, + unconfirmed_email=a.unconfirmed_email, + email_confirmed=a.confirmed_at, + old_id=a.account_id, + password=a.encrypted_password, + username=a.nickname, + locale=a.locale, + city=a.city, + confirmed_at=a.cfd, + )) User.objects.bulk_create(objects) user = User.objects.filter(old_id__isnull=False) user.update(password=ConcatPair(Value('bcrypt$'), F('password'))) - self.stdout.write(self.style.WARNING(f'Created accounts objects.')) + self.stdout.write(self.style.WARNING(f'Created {len(objects)} accounts objects.')) diff --git a/apps/account/management/commands/add_affilations.py b/apps/account/management/commands/add_affilations.py index c59aaafe..58e24ff9 100644 --- a/apps/account/management/commands/add_affilations.py +++ b/apps/account/management/commands/add_affilations.py @@ -1,18 +1,18 @@ -from account.models import OldRole, Role, User, UserRole -from main.models import SiteSettings from django.core.management.base import BaseCommand from django.db import connections, transaction -from django.db.models import Prefetch -from establishment.management.commands.add_position import namedtuplefetchall from tqdm import tqdm +from account.models import OldRole, Role, User, UserRole +from establishment.management.commands.add_position import namedtuplefetchall +from main.models import SiteSettings + class Command(BaseCommand): help = '''Add site affilations from old db to new db. Run after migrate account models!!!''' def map_role_sql(self): - with connections['legacy'].cursor() as cursor: + with connections['legacy'].cursor() as cursor: cursor.execute(''' select distinct case when role = 'news_editor' then 'CONTENT_PAGE_MANAGER' @@ -76,14 +76,14 @@ class Command(BaseCommand): if not role.exists(): objects.append( Role(**data) - ) + ) Role.objects.bulk_create(objects) self.stdout.write(self.style.WARNING(f'Added site roles.')) def update_site_role(self): - roles = Role.objects.filter(country__isnull=True).select_related('site')\ - .filter(site__id__isnull=False).select_for_update() + roles = Role.objects.filter(country__isnull=True).select_related('site') \ + .filter(site__id__isnull=False).select_for_update() with transaction.atomic(): for role in tqdm(roles, desc='Update role country'): role.country = role.site.country @@ -150,4 +150,4 @@ class Command(BaseCommand): self.add_site_role() self.update_site_role() self.add_role_user() - self.add_superuser() \ No newline at end of file + self.add_superuser() diff --git a/apps/account/management/commands/add_confirmed_at.py b/apps/account/management/commands/add_confirmed_at.py deleted file mode 100644 index 63c8086c..00000000 --- a/apps/account/management/commands/add_confirmed_at.py +++ /dev/null @@ -1,26 +0,0 @@ -from django.core.management.base import BaseCommand -from tqdm import tqdm - -from account.models import User, UserRole, Role -from transfer.models import OwnershipAffs, Accounts -from establishment.models import Establishment - - -class Command(BaseCommand): - help = """Add confirmed date to User.""" - - def handle(self, *args, **kwarg): - update_users = [] - - old_users = Accounts.objects.filter(confirmed_at__isnull=False) - for old_user in tqdm(old_users, desc='find users for update confirmed_at field'): - try: - user = User.objects.get(email=old_user.email) - except User.DoesNotExist: - continue - else: - user.confirmed_at = old_user.confirmed_at - update_users.append(user) - - User.objects.bulk_update(update_users, ['confirmed_at', ]) - self.stdout.write(self.style.WARNING(f'Updated users: {len(update_users)}')) diff --git a/apps/account/management/commands/add_image.py b/apps/account/management/commands/add_image.py index 07efe53b..2901f686 100644 --- a/apps/account/management/commands/add_image.py +++ b/apps/account/management/commands/add_image.py @@ -1,8 +1,8 @@ from django.core.management.base import BaseCommand from django.db import connections -from django.db.models import Q -from establishment.management.commands.add_position import namedtuplefetchall + from account.models import User +from establishment.management.commands.add_position import namedtuplefetchall class Command(BaseCommand): @@ -24,8 +24,8 @@ class Command(BaseCommand): ''') return namedtuplefetchall(cursor) - def handle(self, *args, **kwargs): + def handle(self, *args, **kwargs): for a in self.account_sql(): users = User.objects.filter(old_id=a.account_id) - users.update(image_url= a.image_url) - self.stdout.write(self.style.WARNING(f'Update accounts image url.')) \ No newline at end of file + users.update(image_url=a.image_url) + self.stdout.write(self.style.WARNING(f'Update accounts image url.')) diff --git a/apps/account/management/commands/add_ownership.py b/apps/account/management/commands/add_ownership.py index 84926d9c..c73513cf 100644 --- a/apps/account/management/commands/add_ownership.py +++ b/apps/account/management/commands/add_ownership.py @@ -2,8 +2,8 @@ from django.core.management.base import BaseCommand from tqdm import tqdm from account.models import User, UserRole, Role -from transfer.models import OwnershipAffs from establishment.models import Establishment +from transfer.models import OwnershipAffs class Command(BaseCommand): diff --git a/apps/account/management/commands/add_social.py b/apps/account/management/commands/add_social.py index c9b056f5..b900deb4 100644 --- a/apps/account/management/commands/add_social.py +++ b/apps/account/management/commands/add_social.py @@ -1,8 +1,9 @@ from django.core.management.base import BaseCommand from django.db import connections from social_django.models import UserSocialAuth -from establishment.management.commands.add_position import namedtuplefetchall + from account.models import User +from establishment.management.commands.add_position import namedtuplefetchall class Command(BaseCommand): diff --git a/apps/account/transfer.py b/apps/account/transfer.py deleted file mode 100644 index 54db107b..00000000 --- a/apps/account/transfer.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -Структура fields: -key - поле в таблице postgres -value - поле или группа полей в таблице legacy - -В случае передачи группы полей каждое поле представляет собой кортеж, где: -field[0] - название аргумента -field[1] - название поля в таблице legacy -Опционально: field[2] - тип данных для преобразования - -Структура внешних ключей: -"legacy_table" - спикок кортежей для сопоставления полей -"legacy_table": [ - (("legacy_key", "legacy_field"), - ("psql_table", "psql_key", "psql_field", "psql_field_type")) -], где: -legacy_table - название модели legacy -legacy_key - ForeignKey в legacy -legacy_field - уникальное поле в модели legacy для сопоставления с postgresql -psql_table - название модели psql -psql_key - ForeignKey в postgresql -psql_field - уникальное поле в модели postgresql для сопоставления с legacy -psql_field_type - тип уникального поля в postgresql - -""" - -card = { - "User": { - "data_type": "objects", - "dependencies": None, - "fields": { - "Accounts": { - "username": "nickname", - "email": "email", - "email_confirmed": ("confirmed_at", "django.db.models.BooleanField") - }, - "relations": { - "Profiles": { - "key": "account", - "fields": { - "first_name": "firstname", - "last_name": "lastname" - } - } - } - } - } - -} - -used_apps = None - - diff --git a/apps/account/transfer_data.py b/apps/account/transfer_data.py deleted file mode 100644 index 0f199173..00000000 --- a/apps/account/transfer_data.py +++ /dev/null @@ -1,55 +0,0 @@ -from pprint import pprint - -from django.db.models import Q - -from transfer.models import Accounts, Identities -from transfer.serializers.account import UserSerializer -from transfer.serializers.user_social_auth import UserSocialAuthSerializer - -STOP_LIST = ( - # 'cyril@tomatic.net', - # 'cyril2@tomatic.net', - # 'd.sadykova@id-east.ru', - # 'd.sadykova@octopod.ru', - # 'n.yurchenko@id-east.ru', -) - - -def transfer_user(): - # queryset = Profiles.objects.all() - # queryset = queryset.annotate(nickname=F('account__nickname')) - # queryset = queryset.annotate(email=F('account__email')) - - queryset = Accounts.objects.exclude(email__in=STOP_LIST) - - serialized_data = UserSerializer(data=list(queryset.values()), many=True) - - if serialized_data.is_valid(): - serialized_data.save() - else: - pprint(f'News serializer errors: {serialized_data.errors}') - - -def transfer_identities(): - queryset = Identities.objects.exclude( - Q(account_id__isnull=True) | - # Q(account__confirmed_at__isnull=True) | - Q(account__email__in=STOP_LIST) - ).values_list( - 'account_id', - 'provider', - 'uid', - ) - - serialized_data = UserSocialAuthSerializer(data=list(queryset.values()), many=True) - - if serialized_data.is_valid(): - serialized_data.save() - else: - pprint(f'UserSocialAuth serializer errors: {serialized_data.errors}') - - -data_types = { - 'account': [transfer_user], - 'identities': [transfer_identities], -} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 938893f1..804b57e6 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -12,7 +12,7 @@ class Command(BaseCommand): SHORT_DATA_TYPES = [ 'dictionaries', # №2 - перенос стран, регионов, городов, адресов 'news', # перенос новостей (после №2) - 'account', # №1 - перенос пользователей + # 'account', # №1 - перенос пользователей - нет, см make_data_migrations.sh !!! 'subscriber', 'recipe', # №2 - рецепты 'partner', diff --git a/apps/transfer/serializers/account.py b/apps/transfer/serializers/account.py deleted file mode 100644 index 86b28e03..00000000 --- a/apps/transfer/serializers/account.py +++ /dev/null @@ -1,40 +0,0 @@ -from rest_framework import serializers -from account.models import User - - -class UserSerializer(serializers.ModelSerializer): - nickname = serializers.CharField() - email = serializers.CharField() - confirmed_at = serializers.DateTimeField(allow_null=True) - id = serializers.CharField() - - class Meta: - model = User - - fields = ( - "id", - "nickname", - "email", - "confirmed_at" - ) - - def validate(self, data): - data["old_id"] = data.pop("id") - data["username"] = self.get_username(data) - data["email_confirmed"] = self.get_email_confirmed(data) - data.pop("nickname") - data.pop("confirmed_at") - return data - - def create(self, validated_data): - # использовать get_or_create - User.objects.create(**validated_data) - - def get_email_confirmed(self, data): - if data.get("confirmed_at"): - return True - else: - return False - - def get_username(self, obj): - return obj["email"] diff --git a/apps/transfer/serializers/user_social_auth.py b/apps/transfer/serializers/user_social_auth.py deleted file mode 100644 index dc260fa0..00000000 --- a/apps/transfer/serializers/user_social_auth.py +++ /dev/null @@ -1,30 +0,0 @@ -from rest_framework import serializers -from social_django.models import UserSocialAuth - -from account.models import User - - -class UserSocialAuthSerializer(serializers.Serializer): - account_id = serializers.IntegerField() - provider = serializers.CharField() - uid = serializers.CharField() - - def validate(self, data): - data.update({ - 'user': self.get_account(data), - }) - data.pop('account_id') - return data - - def create(self, validated_data): - try: - return UserSocialAuth.objects.create(**validated_data) - except Exception as e: - raise ValueError(f"Error creating UserSocialAuth with {validated_data}: {e}") - - @staticmethod - def get_account(data): - user = User.objects.filter(old_id=data['account_id']).first() - if not user: - raise ValueError(f"User account not found with old_id {data['account_id']}") - return user diff --git a/make_data_migration.sh b/make_data_migration.sh index ee968ff5..0012f659 100755 --- a/make_data_migration.sh +++ b/make_data_migration.sh @@ -1,5 +1,47 @@ #!/usr/bin/env bash -./manage.py transfer -a + +# ПОЛЬЗОВАТЕЛИ + +# Перенос пользователей из модели Accounts в User +# -------------------------- +# id -> old_id +# email -> email +# unconfirmed_email -> unconfirmed_email +# confirmed_at (boolean) -> email_confirmed +# encrypted_password -> password +# nickname -> username +# locale -> locale +# city -> city +# confirmed_at -> confirmed_at +./manage.py add_account + +# Добавление к уже перенесенным пользователям image_url по old_id +# -------------------------- +# image_url -> image_url +./manage.py add_image + +# Заполнение модели из identities в UserSocialAuth +# -------------------------- +# пользователь -> user +# provider -> provider +# uid -> uid +./manage.py add_social + +# Заполнение модели OldRole, UserRole (должны быть заполнены Role и SiteSettings) !!! +# -------------------------- +# image_url -> image_url +#./manage.py add_affilations + +# Заполнение модели из OwnershipAffs в UserRole (запускать после переноса заведений) !!! +# -------------------------- +# user -> user, +# role -> role, +# establishment -> establishment, +# owner.state -> state, +# requester -> requester +#./manage.py add_ownership + + ./manage.py transfer --setup_clean_db ./manage.py transfer -d ./manage.py transfer -e