188 lines
8.2 KiB
Python
188 lines
8.2 KiB
Python
from os.path import exists
|
|
from os import makedirs
|
|
from importlib.machinery import SourceFileLoader
|
|
from django.apps import apps
|
|
from django.conf import settings
|
|
from django.db.models import Count, Q
|
|
from tqdm import tqdm
|
|
import sys
|
|
|
|
import timeit
|
|
|
|
|
|
def transfer_objects(data_type):
|
|
start_t = timeit.default_timer()
|
|
for app in apps.get_app_configs():
|
|
if exists(f"{app.path}/transfer_data.py"):
|
|
card_module = SourceFileLoader("transfer", f"{app.path}/transfer_data.py").load_module()
|
|
if not hasattr(card_module, "data_types") \
|
|
or not isinstance(card_module.data_types, dict) \
|
|
or len(card_module.data_types) < 1:
|
|
continue
|
|
|
|
for module_data_type, transfer_funcs in card_module.data_types.items():
|
|
if data_type == module_data_type:
|
|
for transfer_func in transfer_funcs:
|
|
print(f"========================== FUNCTION {transfer_func.__name__} ================================")
|
|
transfer_func = file_log(transfer_func)
|
|
transfer_func()
|
|
end_t = timeit.default_timer()
|
|
print(f"Transfer time: {end_t - start_t}")
|
|
|
|
|
|
def clean_old_records(model, conditions):
|
|
error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
|
|
non_existed_correct_city_ids = []
|
|
to_delete = []
|
|
|
|
try:
|
|
old_records = model.objects.filter(**conditions)
|
|
except Exception as e:
|
|
error_file.write(f"Cannot find model objects: {e}")
|
|
return
|
|
|
|
for old_record in tqdm(old_records):
|
|
correct_record_qs = model.objects.exclude(id=old_record.id) \
|
|
.exclude(**conditions) \
|
|
.filter(name=old_record.name,
|
|
mysql_id=old_record.old_id)
|
|
if correct_record_qs.exists():
|
|
correct_record = correct_record_qs.first()
|
|
# check object dependencies
|
|
for rel_instance in old_record._related_instances:
|
|
field_name = [related_field.name
|
|
for related_field in rel_instance._meta.fields
|
|
if related_field.related_model == correct_record._meta.model][0]
|
|
# rebinding correct dependency instances
|
|
if getattr(rel_instance, field_name) != correct_record:
|
|
setattr(rel_instance, field_name, correct_record)
|
|
rel_instance.save()
|
|
to_delete.append(old_record.id)
|
|
else:
|
|
non_existed_correct_city_ids.append(old_record.id)
|
|
|
|
if non_existed_correct_city_ids:
|
|
print(f"Non existed correct city ids: {non_existed_correct_city_ids}")
|
|
|
|
# delete old records
|
|
counter = len(to_delete)
|
|
old_records.filter(id__in=to_delete).delete()
|
|
print(f'Deleted {counter} objects.')
|
|
|
|
|
|
def clean_old_country_records(model, conditions):
|
|
error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
|
|
non_existed_correct_country_ids = []
|
|
to_delete = []
|
|
|
|
try:
|
|
unique_codes = model.objects.values_list('code', flat=True) \
|
|
.annotate(counter=Count('code')) \
|
|
.filter(counter=1) \
|
|
.values_list('code', flat=True)
|
|
old_records = model.objects.exclude(code__in=unique_codes) \
|
|
.filter(**conditions)
|
|
except Exception as e:
|
|
error_file.write(f"Cannot find model objects: {e}")
|
|
return
|
|
|
|
for old_record in tqdm(old_records):
|
|
correct_record_qs = model.objects.exclude(id=old_record.id) \
|
|
.exclude(**conditions) \
|
|
.filter(Q(code=old_record.code) |
|
|
Q(mysql_ids__contains=[old_record.old_id, ]))
|
|
if correct_record_qs.exists():
|
|
correct_record = correct_record_qs.first()
|
|
# check object dependencies
|
|
for rel_instance in old_record._related_instances:
|
|
if not hasattr(rel_instance, '_meta'):
|
|
for related in rel_instance.all():
|
|
field_name = [related_field.name
|
|
for related_field in related._meta.fields
|
|
if related_field.related_model == correct_record._meta.model][0]
|
|
# rebinding correct dependency instances
|
|
if getattr(rel_instance, field_name) != correct_record:
|
|
setattr(rel_instance, field_name, correct_record)
|
|
rel_instance.save()
|
|
else:
|
|
field_name = [related_field.name
|
|
for related_field in rel_instance._meta.fields
|
|
if related_field.related_model == correct_record._meta.model][0]
|
|
# rebinding correct dependency instances
|
|
if getattr(rel_instance, field_name) != correct_record:
|
|
setattr(rel_instance, field_name, correct_record)
|
|
rel_instance.save()
|
|
to_delete.append(old_record.id)
|
|
else:
|
|
non_existed_correct_country_ids.append(old_record.id)
|
|
|
|
if non_existed_correct_country_ids:
|
|
print(f"Non existed correct country ids: {non_existed_correct_country_ids}")
|
|
|
|
# delete old records
|
|
counter = len(to_delete)
|
|
old_records.filter(id__in=to_delete).delete()
|
|
print(f'Deleted {counter} objects.')
|
|
|
|
|
|
def clean_old_region_records(model, conditions):
|
|
error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w")
|
|
non_existed_correct_country_ids = []
|
|
to_delete = []
|
|
|
|
try:
|
|
old_records = model.objects.filter(**conditions)
|
|
except Exception as e:
|
|
error_file.write(f"Cannot find model objects: {e}")
|
|
return
|
|
|
|
for old_record in tqdm(old_records):
|
|
correct_record_qs = model.objects.exclude(id=old_record.id) \
|
|
.exclude(**conditions) \
|
|
.filter(code__iexact=old_record.code,
|
|
mysql_ids__contains=[old_record.old_id, ])
|
|
if correct_record_qs.exists():
|
|
correct_record = correct_record_qs.first()
|
|
# check object dependencies
|
|
for rel_instance in old_record._related_instances:
|
|
if not hasattr(rel_instance, '_meta'):
|
|
for related in rel_instance.all():
|
|
field_name = [related_field.name
|
|
for related_field in related._meta.fields
|
|
if related_field.related_model == correct_record._meta.model][0]
|
|
# rebinding correct dependency instances
|
|
if getattr(rel_instance, field_name) != correct_record:
|
|
setattr(rel_instance, field_name, correct_record)
|
|
rel_instance.save()
|
|
else:
|
|
field_name = [related_field.name
|
|
for related_field in rel_instance._meta.fields
|
|
if related_field.related_model == correct_record._meta.model][0]
|
|
# rebinding correct dependency instances
|
|
if getattr(rel_instance, field_name) != correct_record:
|
|
setattr(rel_instance, field_name, correct_record)
|
|
rel_instance.save()
|
|
to_delete.append(old_record.id)
|
|
else:
|
|
non_existed_correct_country_ids.append(old_record.id)
|
|
|
|
if non_existed_correct_country_ids:
|
|
print(f"Non existed correct region ids: {non_existed_correct_country_ids}")
|
|
|
|
# delete old records
|
|
counter = len(to_delete)
|
|
old_records.filter(id__in=to_delete).delete()
|
|
print(f'Deleted {counter} objects.')
|
|
|
|
|
|
def file_log(f):
|
|
directory = f"{settings.PROJECT_ROOT}/apps/transfer/log"
|
|
if not exists(directory):
|
|
makedirs(directory)
|
|
sys.stdout = open(f"{directory}/{f.__name__}.log","w+")
|
|
|
|
def res_func(*args, **kwargs):
|
|
return f(*args, **kwargs)
|
|
|
|
return res_func
|