phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor account transfer

Browse Source
This commit is contained in:
alex 2020-02-03 12:32:53 +03:00
parent 33942b70b5
commit 3c56b0c061
12 changed files with 80 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
apps/account/management/commands/add_account.py
View File

@ -2,8 +2,10 @@ from django.core.management.base import BaseCommand
from django.db import connections
from django.db.models import Q, F, Value
from django.db.models.functions import ConcatPair
from establishment.management.commands.add_position import namedtuplefetchall
from tqdm import tqdm
from account.models import User
from establishment.management.commands.add_position import namedtuplefetchall
class Command(BaseCommand):
@ -12,7 +14,8 @@ class Command(BaseCommand):
def account_sql(self):
with connections['legacy'].cursor() as cursor:
cursor.execute('''
select a.email, a.id as account_id, a.encrypted_password,
select a.email, a.id as account_id, a.encrypted_password, a.locale, a.city,
a.confirmed_at as cfd,
case when a.confirmed_at is not null then true else false end as confirmed_at,
case when a.confirmed_at is null then true else false end as unconfirmed_email,
nickname
@ -21,20 +24,24 @@ class Command(BaseCommand):
''')
return namedtuplefetchall(cursor)
def handle(self, *args, **kwargs):
def handle(self, *args, **kwargs):
objects = []
for a in self.account_sql():
for a in tqdm(self.account_sql(), desc='find users'):
users = User.objects.filter(Q(email=a.email) | Q(old_id=a.account_id))
if not users.exists():
objects.append(User(email=a.email,
unconfirmed_email=a.unconfirmed_email,
email_confirmed=a.confirmed_at,
old_id=a.account_id,
password=a.encrypted_password,
username=a.nickname
))
objects.append(User(
email=a.email,
unconfirmed_email=a.unconfirmed_email,
email_confirmed=a.confirmed_at,
old_id=a.account_id,
password=a.encrypted_password,
username=a.nickname,
locale=a.locale,
city=a.city,
confirmed_at=a.cfd,
))
User.objects.bulk_create(objects)
user = User.objects.filter(old_id__isnull=False)
user.update(password=ConcatPair(Value('bcrypt$'), F('password')))
self.stdout.write(self.style.WARNING(f'Created accounts objects.'))
self.stdout.write(self.style.WARNING(f'Created {len(objects)} accounts objects.'))

18
apps/account/management/commands/add_affilations.py
View File

@ -1,18 +1,18 @@
from account.models import OldRole, Role, User, UserRole
from main.models import SiteSettings
from django.core.management.base import BaseCommand
from django.db import connections, transaction
from django.db.models import Prefetch
from establishment.management.commands.add_position import namedtuplefetchall
from tqdm import tqdm
from account.models import OldRole, Role, User, UserRole
from establishment.management.commands.add_position import namedtuplefetchall
from main.models import SiteSettings
class Command(BaseCommand):
help = '''Add site affilations from old db to new db.
Run after migrate account models!!!'''
def map_role_sql(self):
with connections['legacy'].cursor() as cursor:
with connections['legacy'].cursor() as cursor:
cursor.execute('''
select distinct
case when role = 'news_editor' then 'CONTENT_PAGE_MANAGER'
@ -76,14 +76,14 @@ class Command(BaseCommand):
if not role.exists():
objects.append(
Role(**data)
)
)
Role.objects.bulk_create(objects)
self.stdout.write(self.style.WARNING(f'Added site roles.'))
def update_site_role(self):
roles = Role.objects.filter(country__isnull=True).select_related('site')\
.filter(site__id__isnull=False).select_for_update()
roles = Role.objects.filter(country__isnull=True).select_related('site') \
.filter(site__id__isnull=False).select_for_update()
with transaction.atomic():
for role in tqdm(roles, desc='Update role country'):
role.country = role.site.country
@ -150,4 +150,4 @@ class Command(BaseCommand):
self.add_site_role()
self.update_site_role()
self.add_role_user()
self.add_superuser()
self.add_superuser()

26
apps/account/management/commands/add_confirmed_at.py
View File

@ -1,26 +0,0 @@
from django.core.management.base import BaseCommand
from tqdm import tqdm
from account.models import User, UserRole, Role
from transfer.models import OwnershipAffs, Accounts
from establishment.models import Establishment
class Command(BaseCommand):
help = """Add confirmed date to User."""
def handle(self, *args, **kwarg):
update_users = []
old_users = Accounts.objects.filter(confirmed_at__isnull=False)
for old_user in tqdm(old_users, desc='find users for update confirmed_at field'):
try:
user = User.objects.get(email=old_user.email)
except User.DoesNotExist:
continue
else:
user.confirmed_at = old_user.confirmed_at
update_users.append(user)
User.objects.bulk_update(update_users, ['confirmed_at', ])
self.stdout.write(self.style.WARNING(f'Updated users: {len(update_users)}'))

10
apps/account/management/commands/add_image.py
View File

@ -1,8 +1,8 @@
from django.core.management.base import BaseCommand
from django.db import connections
from django.db.models import Q
from establishment.management.commands.add_position import namedtuplefetchall
from account.models import User
from establishment.management.commands.add_position import namedtuplefetchall
class Command(BaseCommand):
@ -24,8 +24,8 @@ class Command(BaseCommand):
''')
return namedtuplefetchall(cursor)
def handle(self, *args, **kwargs):
def handle(self, *args, **kwargs):
for a in self.account_sql():
users = User.objects.filter(old_id=a.account_id)
users.update(image_url= a.image_url)
self.stdout.write(self.style.WARNING(f'Update accounts image url.'))
users.update(image_url=a.image_url)
self.stdout.write(self.style.WARNING(f'Update accounts image url.'))

2
apps/account/management/commands/add_ownership.py
View File

@ -2,8 +2,8 @@ from django.core.management.base import BaseCommand
from tqdm import tqdm
from account.models import User, UserRole, Role
from transfer.models import OwnershipAffs
from establishment.models import Establishment
from transfer.models import OwnershipAffs
class Command(BaseCommand):

3
apps/account/management/commands/add_social.py
View File

@ -1,8 +1,9 @@
from django.core.management.base import BaseCommand
from django.db import connections
from social_django.models import UserSocialAuth
from establishment.management.commands.add_position import namedtuplefetchall
from account.models import User
from establishment.management.commands.add_position import namedtuplefetchall
class Command(BaseCommand):

53
apps/account/transfer.py
View File

@ -1,53 +0,0 @@
"""
Структура fields:
key - поле в таблице postgres
value - поле или группа полей в таблице legacy
В случае передачи группы полей каждое поле представляет собой кортеж, где:
field[0] - название аргумента
field[1] - название поля в таблице legacy
Опционально: field[2] - тип данных для преобразования
Структура внешних ключей:
"legacy_table" - спикок кортежей для сопоставления полей
"legacy_table": [
(("legacy_key", "legacy_field"),
("psql_table", "psql_key", "psql_field", "psql_field_type"))
], где:
legacy_table - название модели legacy
legacy_key - ForeignKey в legacy
legacy_field - уникальное поле в модели legacy для сопоставления с postgresql
psql_table - название модели psql
psql_key - ForeignKey в postgresql
psql_field - уникальное поле в модели postgresql для сопоставления с legacy
psql_field_type - тип уникального поля в postgresql
"""
card = {
"User": {
"data_type": "objects",
"dependencies": None,
"fields": {
"Accounts": {
"username": "nickname",
"email": "email",
"email_confirmed": ("confirmed_at", "django.db.models.BooleanField")
},
"relations": {
"Profiles": {
"key": "account",
"fields": {
"first_name": "firstname",
"last_name": "lastname"
}
}
}
}
}
}
used_apps = None

55
apps/account/transfer_data.py
View File

@ -1,55 +0,0 @@
from pprint import pprint
from django.db.models import Q
from transfer.models import Accounts, Identities
from transfer.serializers.account import UserSerializer
from transfer.serializers.user_social_auth import UserSocialAuthSerializer
STOP_LIST = (
# 'cyril@tomatic.net',
# 'cyril2@tomatic.net',
# 'd.sadykova@id-east.ru',
# 'd.sadykova@octopod.ru',
# 'n.yurchenko@id-east.ru',
)
def transfer_user():
# queryset = Profiles.objects.all()
# queryset = queryset.annotate(nickname=F('account__nickname'))
# queryset = queryset.annotate(email=F('account__email'))
queryset = Accounts.objects.exclude(email__in=STOP_LIST)
serialized_data = UserSerializer(data=list(queryset.values()), many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f'News serializer errors: {serialized_data.errors}')
def transfer_identities():
queryset = Identities.objects.exclude(
Q(account_id__isnull=True) |
# Q(account__confirmed_at__isnull=True) |
Q(account__email__in=STOP_LIST)
).values_list(
'account_id',
'provider',
'uid',
)
serialized_data = UserSocialAuthSerializer(data=list(queryset.values()), many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f'UserSocialAuth serializer errors: {serialized_data.errors}')
data_types = {
'account': [transfer_user],
'identities': [transfer_identities],
}

2
apps/transfer/management/commands/transfer.py
View File

@ -12,7 +12,7 @@ class Command(BaseCommand):
SHORT_DATA_TYPES = [
'dictionaries', # №2 - перенос стран, регионов, городов, адресов
'news', # перенос новостей (после №2)
'account', # №1 - перенос пользователей
# 'account', # №1 - перенос пользователей - нет, см make_data_migrations.sh !!!
'subscriber',
'recipe', # №2 - рецепты
'partner',

40
apps/transfer/serializers/account.py
View File

@ -1,40 +0,0 @@
from rest_framework import serializers
from account.models import User
class UserSerializer(serializers.ModelSerializer):
nickname = serializers.CharField()
email = serializers.CharField()
confirmed_at = serializers.DateTimeField(allow_null=True)
id = serializers.CharField()
class Meta:
model = User
fields = (
"id",
"nickname",
"email",
"confirmed_at"
)
def validate(self, data):
data["old_id"] = data.pop("id")
data["username"] = self.get_username(data)
data["email_confirmed"] = self.get_email_confirmed(data)
data.pop("nickname")
data.pop("confirmed_at")
return data
def create(self, validated_data):
# использовать get_or_create
User.objects.create(**validated_data)
def get_email_confirmed(self, data):
if data.get("confirmed_at"):
return True
else:
return False
def get_username(self, obj):
return obj["email"]

30
apps/transfer/serializers/user_social_auth.py
View File

@ -1,30 +0,0 @@
from rest_framework import serializers
from social_django.models import UserSocialAuth
from account.models import User
class UserSocialAuthSerializer(serializers.Serializer):
account_id = serializers.IntegerField()
provider = serializers.CharField()
uid = serializers.CharField()
def validate(self, data):
data.update({
'user': self.get_account(data),
})
data.pop('account_id')
return data
def create(self, validated_data):
try:
return UserSocialAuth.objects.create(**validated_data)
except Exception as e:
raise ValueError(f"Error creating UserSocialAuth with {validated_data}: {e}")
@staticmethod
def get_account(data):
user = User.objects.filter(old_id=data['account_id']).first()
if not user:
raise ValueError(f"User account not found with old_id {data['account_id']}")
return user

44
make_data_migration.sh
View File

@ -1,5 +1,47 @@
#!/usr/bin/env bash
./manage.py transfer -a
# ПОЛЬЗОВАТЕЛИ
# Перенос пользователей из модели Accounts в User
# --------------------------
# id -> old_id
# email -> email
# unconfirmed_email -> unconfirmed_email
# confirmed_at (boolean) -> email_confirmed
# encrypted_password -> password
# nickname -> username
# locale -> locale
# city -> city
# confirmed_at -> confirmed_at
./manage.py add_account
# Добавление к уже перенесенным пользователям image_url по old_id
# --------------------------
# image_url -> image_url
./manage.py add_image
# Заполнение модели из identities в UserSocialAuth
# --------------------------
# пользователь -> user
# provider -> provider
# uid -> uid
./manage.py add_social
# Заполнение модели OldRole, UserRole (должны быть заполнены Role и SiteSettings) !!!
# --------------------------
# image_url -> image_url
#./manage.py add_affilations
# Заполнение модели из OwnershipAffs в UserRole (запускать после переноса заведений) !!!
# --------------------------
# user -> user,
# role -> role,
# establishment -> establishment,
# owner.state -> state,
# requester -> requester
#./manage.py add_ownership
./manage.py transfer --setup_clean_db
./manage.py transfer -d
./manage.py transfer -e