from os.path import exists from os import makedirs from importlib.machinery import SourceFileLoader from django.apps import apps from django.conf import settings from django.db.models import Count, Q from tqdm import tqdm import sys import timeit def transfer_objects(data_type): start_t = timeit.default_timer() for app in apps.get_app_configs(): if exists(f"{app.path}/transfer_data.py"): card_module = SourceFileLoader("transfer", f"{app.path}/transfer_data.py").load_module() if not hasattr(card_module, "data_types") \ or not isinstance(card_module.data_types, dict) \ or len(card_module.data_types) < 1: continue for module_data_type, transfer_funcs in card_module.data_types.items(): if data_type == module_data_type: for transfer_func in transfer_funcs: print(f"========================== FUNCTION {transfer_func.__name__} ================================") transfer_func = file_log(transfer_func) transfer_func() end_t = timeit.default_timer() print(f"Transfer time: {end_t - start_t}") def clean_old_records(model, conditions): error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w") non_existed_correct_city_ids = [] to_delete = [] try: old_records = model.objects.filter(**conditions) except Exception as e: error_file.write(f"Cannot find model objects: {e}") return for old_record in tqdm(old_records): correct_record_qs = model.objects.exclude(id=old_record.id) \ .exclude(**conditions) \ .filter(name=old_record.name, mysql_id=old_record.old_id) if correct_record_qs.exists(): correct_record = correct_record_qs.first() # check object dependencies for rel_instance in old_record._related_instances: field_name = [related_field.name for related_field in rel_instance._meta.fields if related_field.related_model == correct_record._meta.model][0] # rebinding correct dependency instances if getattr(rel_instance, field_name) != correct_record: setattr(rel_instance, field_name, correct_record) rel_instance.save() to_delete.append(old_record.id) else: non_existed_correct_city_ids.append(old_record.id) if non_existed_correct_city_ids: print(f"Non existed correct city ids: {non_existed_correct_city_ids}") # delete old records counter = len(to_delete) old_records.filter(id__in=to_delete).delete() print(f'Deleted {counter} objects.') def clean_old_country_records(model, conditions): error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w") non_existed_correct_country_ids = [] to_delete = [] try: unique_codes = model.objects.values_list('code', flat=True) \ .annotate(counter=Count('code')) \ .filter(counter=1) \ .values_list('code', flat=True) old_records = model.objects.exclude(code__in=unique_codes) \ .filter(**conditions) except Exception as e: error_file.write(f"Cannot find model objects: {e}") return for old_record in tqdm(old_records): correct_record_qs = model.objects.exclude(id=old_record.id) \ .exclude(**conditions) \ .filter(Q(code=old_record.code) | Q(mysql_ids__contains=[old_record.old_id, ])) if correct_record_qs.exists(): correct_record = correct_record_qs.first() # check object dependencies for rel_instance in old_record._related_instances: if not hasattr(rel_instance, '_meta'): for related in rel_instance.all(): field_name = [related_field.name for related_field in related._meta.fields if related_field.related_model == correct_record._meta.model][0] # rebinding correct dependency instances if getattr(rel_instance, field_name) != correct_record: setattr(rel_instance, field_name, correct_record) rel_instance.save() else: field_name = [related_field.name for related_field in rel_instance._meta.fields if related_field.related_model == correct_record._meta.model][0] # rebinding correct dependency instances if getattr(rel_instance, field_name) != correct_record: setattr(rel_instance, field_name, correct_record) rel_instance.save() to_delete.append(old_record.id) else: non_existed_correct_country_ids.append(old_record.id) if non_existed_correct_country_ids: print(f"Non existed correct country ids: {non_existed_correct_country_ids}") # delete old records counter = len(to_delete) old_records.filter(id__in=to_delete).delete() print(f'Deleted {counter} objects.') def clean_old_region_records(model, conditions): error_file = open(f"{settings.PROJECT_ROOT}/apps/transfer/clear.error.txt", "w") non_existed_correct_country_ids = [] to_delete = [] try: old_records = model.objects.filter(**conditions) except Exception as e: error_file.write(f"Cannot find model objects: {e}") return for old_record in tqdm(old_records): correct_record_qs = model.objects.exclude(id=old_record.id) \ .exclude(**conditions) \ .filter(code__iexact=old_record.code, mysql_ids__contains=[old_record.old_id, ]) if correct_record_qs.exists(): correct_record = correct_record_qs.first() # check object dependencies for rel_instance in old_record._related_instances: if not hasattr(rel_instance, '_meta'): for related in rel_instance.all(): field_name = [related_field.name for related_field in related._meta.fields if related_field.related_model == correct_record._meta.model][0] # rebinding correct dependency instances if getattr(rel_instance, field_name) != correct_record: setattr(rel_instance, field_name, correct_record) rel_instance.save() else: field_name = [related_field.name for related_field in rel_instance._meta.fields if related_field.related_model == correct_record._meta.model][0] # rebinding correct dependency instances if getattr(rel_instance, field_name) != correct_record: setattr(rel_instance, field_name, correct_record) rel_instance.save() to_delete.append(old_record.id) else: non_existed_correct_country_ids.append(old_record.id) if non_existed_correct_country_ids: print(f"Non existed correct region ids: {non_existed_correct_country_ids}") # delete old records counter = len(to_delete) old_records.filter(id__in=to_delete).delete() print(f'Deleted {counter} objects.') def file_log(f): directory = f"{settings.PROJECT_ROOT}/apps/transfer/log" if not exists(directory): makedirs(directory) sys.stdout = open(f"{directory}/{f.__name__}.log","w+") def res_func(*args, **kwargs): return f(*args, **kwargs) return res_func