from pprint import pprint from tqdm import tqdm from establishment.models import Establishment from review.models import Review from location.models import WineRegion, City from product.models import Product from transfer.models import Guides, GuideFilters, GuideSections, GuideElements, \ GuideAds from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer from collection.models import GuideElementSection, GuideElementSectionCategory, \ GuideWineColorSection, GuideElementType, GuideElement, \ Guide, Advertorial def transfer_guide(): """Transfer Guide model.""" errors = [] queryset = Guides.objects.exclude(title__icontains='test') serialized_data = GuideSerializer( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: for d in serialized_data.errors: errors.append(d) if d else None pprint(f"ERRORS: {errors}") print(f'COUNT OF SERIALIZED OBJECTS: {queryset.values().count()}') def transfer_guide_filter(): """Transfer GuideFilter model.""" errors = [] queryset = GuideFilters.objects.exclude(guide__title__icontains='test') \ .exclude(guide__isnull=True) serialized_data = GuideFilterSerializer( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: for d in serialized_data.errors: errors.append(d) if d else None pprint(f'ERRORS: {errors}') print(f"COUNT: {len(errors)}") print(f'COUNT OF SERIALIZED OBJECTS: {queryset.values().count()}') def transfer_guide_element_section(): """Transfer GuideSections model.""" created_count = 0 category, _ = GuideElementSectionCategory.objects.get_or_create( name='shop_category') queryset_values = GuideSections.objects.values_list('id', 'value_name') for old_id, section_name in tqdm(queryset_values): obj, created = GuideElementSection.objects.get_or_create( name=section_name, category=category, old_id=old_id, ) if created: created_count += 1 print(f'OBJECTS CREATED: {created_count}') def transfer_guide_wine_color_section(): """Transfer GuideElements model (only wine color sections).""" created_count = 0 queryset_values = GuideElements.objects.raw( """ select distinct(color), 1 as id from guide_elements where color is not null; """ ) for section_name in tqdm([i.color for i in queryset_values]): obj, created = GuideWineColorSection.objects.get_or_create( name=section_name ) if created: created_count += 1 print(f'OBJECTS CREATED: {created_count}') def transfer_guide_element_type(): """Transfer GuideElements model (only element types).""" created_count = 0 queryset_values = GuideElements.objects.raw( """ select distinct(type), 1 as id from guide_elements; """ ) for element_type in tqdm([i.type for i in queryset_values]): obj, created = GuideElementType.objects.get_or_create( name=element_type ) if created: created_count += 1 print(f'OBJECTS CREATED: {created_count}') def transfer_guide_elements_bulk(): """Transfer Guide elements via bulk_create.""" def get_guide_element_type(guide_element_type: str): if guide_element_type: qs = GuideElementType.objects.filter(name__iexact=guide_element_type) if qs.exists(): return qs.first() def get_establishment(old_id: int): if old_id: qs = Establishment.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_review(old_id: int): if old_id: qs = Review.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_wine_region(old_id: int): if old_id: qs = WineRegion.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_wine(old_id: int): if old_id: qs = Product.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_wine_color_section(color_section: str): if color_section: qs = GuideWineColorSection.objects.filter(name__iexact=color_section) if qs.exists(): return qs.first() def get_city(old_id: int): if old_id: qs = City.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_guide_element_section(old_id: int): if old_id: qs = GuideElementSection.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_guide(old_id): if old_id: qs = Guide.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_parent(old_id): if old_id: qs = GuideElement.objects.filter(old_id=old_id) if qs.exists(): return qs.first() objects_to_update = [] base_queryset = GuideElements.objects.all() for old_id, type, establishment_id, review_id, wine_region_id, \ wine_id, color, order_number, city_id, section_id, guide_id \ in tqdm(base_queryset.filter(parent_id__isnull=True) .values_list('id', 'type', 'establishment_id', 'review_id', 'wine_region_id', 'wine_id', 'color', 'order_number', 'city_id', 'section_id', 'guide_id'), desc='Check parent guide elements'): if not GuideElement.objects.filter(old_id=old_id).exists(): guide = GuideElement( old_id=old_id, guide_element_type=get_guide_element_type(type), establishment=get_establishment(establishment_id), review=get_review(review_id), wine_region=get_wine_region(wine_region_id), product=get_wine(wine_id), wine_color_section=get_wine_color_section(color), priority=order_number, city=get_city(city_id), section=get_guide_element_section(section_id), parent=None, lft=1, rght=1, tree_id=1, level=1, ) # check old guide if not guide_id: objects_to_update.append(guide) else: old_guide = Guides.objects.exclude(title__icontains='test') \ .filter(id=guide_id) if old_guide.exists(): guide.guide = get_guide(guide_id) objects_to_update.append(guide) # create parents GuideElement.objects.bulk_create(objects_to_update) pprint(f'CREATED PARENT GUIDE ELEMENTS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') # attach child guide elements queryset_values = base_queryset.filter(parent_id__isnull=False) \ .order_by('-parent_id') \ .values_list('id', 'type', 'establishment_id', 'review_id', 'wine_region_id', 'wine_id', 'color', 'order_number', 'city_id', 'section_id', 'guide_id', 'parent_id') for old_id, type, establishment_id, review_id, wine_region_id, \ wine_id, color, order_number, city_id, section_id, guide_id, parent_id \ in tqdm(sorted(queryset_values, key=lambda value: value[len(value)-1]), desc='Check child guide elements'): if not GuideElement.objects.filter(old_id=old_id).exists(): # check old guide if guide_id: old_guide = Guides.objects.exclude(title__icontains='test') \ .filter(id=guide_id) if old_guide.exists(): GuideElement.objects.create( old_id=old_id, guide_element_type=get_guide_element_type(type), establishment=get_establishment(establishment_id), review=get_review(review_id), wine_region=get_wine_region(wine_region_id), product=get_wine(wine_id), wine_color_section=get_wine_color_section(color), priority=order_number, city=get_city(city_id), section=get_guide_element_section(section_id), parent=get_parent(parent_id), lft=1, rght=1, tree_id=1, level=1, guide=get_guide(guide_id), ) pprint(f'CREATED CHILD GUIDE ELEMENTS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') # rebuild trees GuideElement._tree_manager.rebuild() def transfer_guide_element_advertorials(): """Transfer Guide Advertorials model.""" def get_guide_element(old_id: int): if old_id: qs = GuideElement.objects.filter(old_id=old_id) legacy_qs = GuideElements.objects.exclude(guide__isnull=True) \ .exclude(guide__title__icontains='test') \ .filter(id=guide_ad_node_id) if qs.exists() and legacy_qs.exists(): return qs.first() elif legacy_qs.exists() and not qs.exists(): raise ValueError(f'Guide element was not transfer correctly - {old_id}.') objects_to_update = [] advertorials = GuideAds.objects.exclude(nb_pages__isnull=True) \ .exclude(nb_right_pages__isnull=True) \ .exclude(guide_ad_node_id__isnull=True) \ .values_list('id', 'nb_pages', 'nb_right_pages', 'guide_ad_node_id') for old_id, nb_pages, nb_right_pages, guide_ad_node_id in tqdm(advertorials): # check guide element guide_element = get_guide_element(guide_ad_node_id) if not Advertorial.objects.filter(old_id=old_id).exists() and guide_element: objects_to_update.append( Advertorial( old_id=old_id, number_of_pages=nb_pages, right_pages=nb_right_pages, guide_element=guide_element, ) ) # create related child Advertorial.objects.bulk_create(objects_to_update) pprint(f'CREATED ADVERTORIALS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') data_types = { 'guides': [ transfer_guide, ], 'guide_filters': [ transfer_guide_filter, ], 'guide_element_sections': [ transfer_guide_element_section, ], 'guide_wine_color_sections': [ transfer_guide_wine_color_section, ], 'guide_element_types': [ transfer_guide_element_type, ], 'guide_elements_bulk': [ transfer_guide_elements_bulk, ], 'guide_element_advertorials': [ transfer_guide_element_advertorials ], 'guide_complete': [ transfer_guide, # transfer guides from Guides transfer_guide_filter, # transfer guide filters from GuideFilters transfer_guide_element_section, # partial transfer element section from GuideSections transfer_guide_wine_color_section, # partial transfer wine color section from GuideSections transfer_guide_element_type, # partial transfer section types from GuideElements transfer_guide_elements_bulk, # transfer result of GuideFilters from GuideElements transfer_guide_element_advertorials, # transfer advertorials that linked to GuideElements ] }