From 47a4d28cf02d5814a1fde3778c81f4b5f3e429b6 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 15 Oct 2019 10:38:51 +0300 Subject: [PATCH] Add fields map for simple field Add fields map for tuple field Refactoring code --- apps/news/transfer.py | 9 +- apps/transfer/management/commands/transfer.py | 1 + apps/transfer/utils.py | 245 +++++++++--------- 3 files changed, 133 insertions(+), 122 deletions(-) diff --git a/apps/news/transfer.py b/apps/news/transfer.py index 68c22dab..6a904ad9 100644 --- a/apps/news/transfer.py +++ b/apps/news/transfer.py @@ -35,21 +35,22 @@ NOTE: среди legacy таблиц совпадение для таблицы card = { # нету аналога для NewsType "NewsType": { - "data_type": "dictionaries", + "data_type": "news", "dependencies": None, "fields": { "Pages": { # будет только один тип новости "News" # значения для поля "name" берутся из поля "type" legacy модели "Pages", притом type="News" # Mysql - select distinct(type) from pages; - "name": "name" + "name": "type" } } }, "News": { - "data_type": "objects", - "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), + "data_type": "news", + # "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), + "dependencies": ("NewsType", ), "fields": { # нету аналогов для start, end, playlist "Pages": { diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 9b99ea6c..8018e38f 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -10,6 +10,7 @@ class Command(BaseCommand): """ DATA_TYPES = [ 'dictionaries', + 'news' ] def handle(self, *args, **options): diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 32cb4bae..cd5a3f19 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,12 +2,7 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from collections import OrderedDict -from django.db.models import OuterRef, Subquery, FilteredRelation, Q - -import transfer.models as legacy_models - -# from pprint import pprint as print - +from pprint import pprint def transfer_objects(data_type): models_list = {} @@ -29,12 +24,134 @@ def transfer_objects(data_type): models_list = sort_by_dependencies(models_list) - legacy_objects = OrderedDict() - for model, card in models_list.items(): - legacy_objects[model] = get_model_data(model, card) + transfer_data(model, card) +# TRANSFER DATA FUNCTION +def transfer_data(model, card): + legacy_model_queryset = get_legacy_data(card) + fields_map = get_fields_map(card, legacy_model_queryset) + + + + + + + + + + + + + + + + + + + + +# ************************** UTILITIES ****************************** + +# Get map of fields +def get_fields_map(card, queryset): + relations = card['fields']['relations'] if "relations" in card['fields'] else None + if relations: + del (card['fields']['relations']) + + fields_list = list(card['fields'].values())[0] + + app_queryset_list = [] + + for legacy_object in queryset.iterator(): + app_queryset_dict = {} + for app_field, legacy_field in fields_list.items(): + if isinstance(legacy_field, tuple): + if isinstance(legacy_field[0], tuple): + # Группа полей с аргументами + pass + else: + print(app_field) + print(legacy_field) + else: + try: + app_queryset_dict[app_field] = getattr(legacy_object, legacy_field) + except Exception as e: + print(f"Attribute {legacy_field} not found in {legacy_object}: {e}") + return + + app_queryset_list.append(app_queryset_dict) + + # pprint(card) + # print("*************************************") + # pprint(fields_list) + # print("*************************************") + # pprint(relations) + # print("*************************************") + # pprint(len(queryset)) + print("==============================") + + return card + + +# Get legacy model data +def get_legacy_data(card): + if "fields" not in card: + print("You must set fields list") + return + card_fields = card['fields'] + relations = card_fields['relations'] if 'relations' in card_fields else None + + if relations: + del (card_fields['relations']) + + if len(card_fields) > 1: + print("You must set one table to transfer") + return + + try: + legacy_table = list(card_fields.keys())[0] + except Exception as e: + print(f"Cannot get legacy table name from {card_fields.keys()}") + return + try: + fields = list(card_fields.values())[0] + except Exception as e: + print(f"Cannot get legacy fields from {card_fields.values()}") + return + + try: + legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) + except LookupError as e: + print(f"ERROR: legacy model {legacy_table} can not be loaded: {e}") + return + + legacy_fields = [] + + for legacy_field in fields.values(): + if isinstance(legacy_field, str): + legacy_fields.append(legacy_field) + + elif isinstance(legacy_field, tuple): + if isinstance(legacy_field[0], tuple): + """Группа полей""" + for legacy_field_group in legacy_field: + legacy_fields.append(legacy_field_group[1]) + else: + """Одиночное поле""" + legacy_fields.append(legacy_field[0]) + + queryset = legacy_model.objects.only(*legacy_fields) + + # Возвращаем зависимости на место + if relations: + card['fields']['relations'] = relations + + return queryset + + +# Models sort def sort_by_dependencies(data): """Сортировка моделей по зависимостям""" @@ -56,112 +173,4 @@ def sort_by_dependencies(data): [(model, data[model]) for model in reversed(result)] ) - return result - - -def get_model_data(model, card): - # print(f"=================================================\r\n" - # f"MODEL: {model}\r\n" - # f"=================================================") - - card_fields = card['fields'] - relations = card_fields['relations'] if 'relations' in card_fields else None - - if relations: - del(card_fields['relations']) - - if len(card_fields) > 1: - print("You must set one table to transfer") - return - - legacy_table = list(card_fields.keys())[0] - fields = list(card_fields.values())[0] - - try: - legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) - except LookupError as e: - print(f"ERROR: legacy model {model} can not be loaded: {e}") - return - - legacy_fields = [] - - for legacy_field in fields.values(): - if isinstance(legacy_field, str): - legacy_fields.append(legacy_field) - - elif isinstance(legacy_field, tuple): - if isinstance(legacy_field[0], tuple): - """Группа полей""" - for legacy_field_group in legacy_field: - legacy_fields.append(legacy_field_group[1]) - else: - """Одиночное поле""" - legacy_fields.append(legacy_field[1]) - - #TODO: убрать после совместимости - if model != "City": - return - - queryset = legacy_model.objects.only(*legacy_fields) - - """Проверяем наличие внешних связей""" - if relations: - queryset = get_relation_data(relations, queryset, fields) - - return get_transfer_data(model, card, queryset) - - -def get_relation_data(relations, queryset, fields): - - # related_legacy_models = [x.related_model._meta.model.__name__ for x in legacy_model._meta.related_objects] - - """Получаем названия внешних зависимостей""" - for relation_table, relation_data in relations.items(): - """Если объекты связаны через ForeignKey, то ничего не надо делать - все аннотации уже подключены""" - if isinstance(relation_data['key'], tuple): - """Вытаскиваем relation по id""" - try: - child_legacy_model = apps.get_model(app_label="transfer", model_name=relation_table) - except LookupError as e: - raise LookupError(f"ERROR: legacy model {relation_table} can not be loaded: {e}") - - child_table_queryset = child_legacy_model.objects.filter( - city_id=OuterRef('pk') - ).only(*relation_data['fields'].values()) - - queryset = queryset.annotate(**{ - f"{relation_table.lower()}": - Subquery( - child_table_queryset - ) - }) - - return queryset - - -def get_transfer_data(model, card, queryset): - from pprint import pprint as print - - try: - app_model = apps.get_model(app_label=card['app_label'], model_name=model) - except LookupError as e: - print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") - return - - if "fields" not in card or not isinstance(card['fields'], dict) or len(card['fields']) != 1: - raise ValueError("You must set correct fields to transfer") - - # for app_field, legacy_field_data in card['fields'].items(): - # print(app_field) - # print(legacy_field_data) - - # print("*************************************************************************") - # print(app_model) - # print("*************************************************************************") - # print(queryset) - # for modellt in queryset: - # print(dir(modellt)) - # if hasattr(modellt, "cityphotos_set"): - # print(modellt.cityphotos_set.all()) - # break - return queryset + return result \ No newline at end of file