Add fields map for simple field

Add fields map for tuple field
Refactoring code

apps/news/transfer.py

@@ -35,21 +35,22 @@ NOTE: среди legacy таблиц совпадение для таблицы
 card = {
     # нету аналога для NewsType
     "NewsType": {
-        "data_type": "dictionaries",
+        "data_type": "news",
         "dependencies": None,
         "fields": {
             "Pages": {
                 # будет только один тип новости "News"
                 # значения для поля "name" берутся из поля "type"  legacy модели "Pages", притом type="News"
                 # Mysql - select distinct(type) from pages;
-                "name": "name"
+                "name": "type"
             }
         }
     },
 
     "News": {
-        "data_type": "objects",
+        "data_type": "news",
-        "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"),
+        # "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"),
+        "dependencies": ("NewsType", ),
         "fields": {
             # нету аналогов для start, end, playlist
             "Pages": {

apps/transfer/management/commands/transfer.py

@@ -10,6 +10,7 @@ class Command(BaseCommand):
     """
     DATA_TYPES = [
         'dictionaries',
+        'news'
     ]
 
     def handle(self, *args, **options):

apps/transfer/utils.py

@@ -2,12 +2,7 @@ from os.path import exists
 from importlib.machinery import SourceFileLoader
 from django.apps import apps
 from collections import OrderedDict
-from django.db.models import OuterRef, Subquery, FilteredRelation, Q
+from pprint import pprint
-
-import transfer.models as legacy_models
-
-# from pprint import pprint as print
-
 
 def transfer_objects(data_type):
     models_list = {}
@@ -29,12 +24,134 @@ def transfer_objects(data_type):
 
     models_list = sort_by_dependencies(models_list)
 
-    legacy_objects = OrderedDict()
-
     for model, card in models_list.items():
-        legacy_objects[model] = get_model_data(model, card)
+        transfer_data(model, card)
 
 
+# TRANSFER DATA FUNCTION
+def transfer_data(model, card):
+    legacy_model_queryset = get_legacy_data(card)
+    fields_map = get_fields_map(card, legacy_model_queryset)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# ************************** UTILITIES ******************************
+
+# Get map of fields
+def get_fields_map(card, queryset):
+    relations = card['fields']['relations'] if "relations" in card['fields'] else None
+    if relations:
+        del (card['fields']['relations'])
+
+    fields_list = list(card['fields'].values())[0]
+
+    app_queryset_list = []
+
+    for legacy_object in queryset.iterator():
+        app_queryset_dict = {}
+        for app_field, legacy_field in fields_list.items():
+            if isinstance(legacy_field, tuple):
+                if isinstance(legacy_field[0], tuple):
+                    # Группа полей с аргументами
+                    pass
+                else:
+                    print(app_field)
+                    print(legacy_field)
+            else:
+                try:
+                    app_queryset_dict[app_field] = getattr(legacy_object, legacy_field)
+                except Exception as e:
+                    print(f"Attribute {legacy_field} not found in {legacy_object}: {e}")
+                    return
+
+        app_queryset_list.append(app_queryset_dict)
+
+    # pprint(card)
+    # print("*************************************")
+    # pprint(fields_list)
+    # print("*************************************")
+    # pprint(relations)
+    # print("*************************************")
+    # pprint(len(queryset))
+    print("==============================")
+
+    return card
+
+
+# Get legacy model data
+def get_legacy_data(card):
+    if "fields" not in card:
+        print("You must set fields list")
+        return
+    card_fields = card['fields']
+    relations = card_fields['relations'] if 'relations' in card_fields else None
+
+    if relations:
+        del (card_fields['relations'])
+
+    if len(card_fields) > 1:
+        print("You must set one table to transfer")
+        return
+
+    try:
+        legacy_table = list(card_fields.keys())[0]
+    except Exception as e:
+        print(f"Cannot get legacy table name from {card_fields.keys()}")
+        return
+    try:
+        fields = list(card_fields.values())[0]
+    except Exception as e:
+        print(f"Cannot get legacy fields from {card_fields.values()}")
+        return
+
+    try:
+        legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table)
+    except LookupError as e:
+        print(f"ERROR: legacy model {legacy_table} can not be loaded: {e}")
+        return
+
+    legacy_fields = []
+
+    for legacy_field in fields.values():
+        if isinstance(legacy_field, str):
+            legacy_fields.append(legacy_field)
+
+        elif isinstance(legacy_field, tuple):
+            if isinstance(legacy_field[0], tuple):
+                """Группа полей"""
+                for legacy_field_group in legacy_field:
+                    legacy_fields.append(legacy_field_group[1])
+            else:
+                """Одиночное поле"""
+                legacy_fields.append(legacy_field[0])
+
+    queryset = legacy_model.objects.only(*legacy_fields)
+
+    # Возвращаем зависимости на место
+    if relations:
+        card['fields']['relations'] = relations
+
+    return queryset
+
+
+# Models sort
 def sort_by_dependencies(data):
     """Сортировка моделей по зависимостям"""
 
@@ -56,112 +173,4 @@ def sort_by_dependencies(data):
         [(model, data[model]) for model in reversed(result)]
     )
 
-    return result
+    return result
-
-
-def get_model_data(model, card):
-    # print(f"=================================================\r\n"
-    #       f"MODEL: {model}\r\n"
-    #       f"=================================================")
-
-    card_fields = card['fields']
-    relations = card_fields['relations'] if 'relations' in card_fields else None
-
-    if relations:
-        del(card_fields['relations'])
-
-    if len(card_fields) > 1:
-        print("You must set one table to transfer")
-        return
-
-    legacy_table = list(card_fields.keys())[0]
-    fields = list(card_fields.values())[0]
-
-    try:
-        legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table)
-    except LookupError as e:
-        print(f"ERROR: legacy model {model} can not be loaded: {e}")
-        return
-
-    legacy_fields = []
-
-    for legacy_field in fields.values():
-        if isinstance(legacy_field, str):
-            legacy_fields.append(legacy_field)
-
-        elif isinstance(legacy_field, tuple):
-            if isinstance(legacy_field[0], tuple):
-                """Группа полей"""
-                for legacy_field_group in legacy_field:
-                    legacy_fields.append(legacy_field_group[1])
-            else:
-                """Одиночное поле"""
-                legacy_fields.append(legacy_field[1])
-
-    #TODO: убрать после совместимости
-    if model != "City":
-        return
-
-    queryset = legacy_model.objects.only(*legacy_fields)
-
-    """Проверяем наличие внешних связей"""
-    if relations:
-        queryset = get_relation_data(relations, queryset, fields)
-
-    return get_transfer_data(model, card, queryset)
-
-
-def get_relation_data(relations, queryset, fields):
-
-    # related_legacy_models = [x.related_model._meta.model.__name__ for x in legacy_model._meta.related_objects]
-
-    """Получаем названия внешних зависимостей"""
-    for relation_table, relation_data in relations.items():
-        """Если объекты связаны через ForeignKey, то ничего не надо делать - все аннотации уже подключены"""
-        if isinstance(relation_data['key'], tuple):
-            """Вытаскиваем relation по id"""
-            try:
-                child_legacy_model = apps.get_model(app_label="transfer", model_name=relation_table)
-            except LookupError as e:
-                raise LookupError(f"ERROR: legacy model {relation_table} can not be loaded: {e}")
-
-            child_table_queryset = child_legacy_model.objects.filter(
-                city_id=OuterRef('pk')
-            ).only(*relation_data['fields'].values())
-
-            queryset = queryset.annotate(**{
-                f"{relation_table.lower()}":
-                Subquery(
-                    child_table_queryset
-                )
-            })
-
-    return queryset
-
-
-def get_transfer_data(model, card, queryset):
-    from pprint import pprint as print
-
-    try:
-        app_model = apps.get_model(app_label=card['app_label'], model_name=model)
-    except LookupError as e:
-        print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}")
-        return
-
-    if "fields" not in card or not isinstance(card['fields'], dict) or len(card['fields']) != 1:
-        raise ValueError("You must set correct fields to transfer")
-
-    # for app_field, legacy_field_data in card['fields'].items():
-        # print(app_field)
-        # print(legacy_field_data)
-
-        # print("*************************************************************************")
-    # print(app_model)
-    # print("*************************************************************************")
-    # print(queryset)
-    # for modellt in queryset:
-        # print(dir(modellt))
-        # if hasattr(modellt, "cityphotos_set"):
-        #     print(modellt.cityphotos_set.all())
-            # break
-    return queryset