import operator from pprint import pprint from django.db.models import Subquery from tqdm import tqdm from collection.models import GuideElementSection, GuideElementSectionCategory, \ GuideWineColorSection, GuideElementType, GuideElement, \ Guide, Advertorial from establishment.models import Establishment from gallery.models import Image from location.models import WineRegion, City from product.models import Product from review.models import Review from transfer.models import Guides, GuideFilters, GuideSections, GuideElements, \ GuideAds, LabelPhotos from transfer.serializers.guide import GuideSerializer, GuideFilterSerializer def transfer_guide(): """Transfer Guide model.""" errors = [] queryset = Guides.objects.exclude(title__icontains='test') serialized_data = GuideSerializer( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: for d in serialized_data.errors: errors.append(d) if d else None pprint(f"ERRORS: {errors}") print(f'COUNT OF SERIALIZED OBJECTS: {queryset.values().count()}') def transfer_guide_filter(): """Transfer GuideFilter model.""" errors = [] queryset = GuideFilters.objects.exclude(guide__title__icontains='test') \ .exclude(guide__isnull=True) serialized_data = GuideFilterSerializer( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: for d in serialized_data.errors: errors.append(d) if d else None pprint(f'ERRORS: {errors}') print(f"COUNT: {len(errors)}") print(f'COUNT OF SERIALIZED OBJECTS: {queryset.values().count()}') def transfer_guide_element_section(): """Transfer GuideSections model.""" created_count = 0 category, _ = GuideElementSectionCategory.objects.get_or_create( name='shop_category') queryset_values = GuideSections.objects.values_list('id', 'value_name') for old_id, section_name in tqdm(queryset_values): obj, created = GuideElementSection.objects.get_or_create( name=section_name, category=category, old_id=old_id, ) if created: created_count += 1 print(f'OBJECTS CREATED: {created_count}') def transfer_guide_wine_color_section(): """Transfer GuideElements model (only wine color sections).""" created_count = 0 queryset_values = GuideElements.objects.raw( """ select distinct(color), 1 as id from guide_elements where color is not null; """ ) for section_name in tqdm([i.color for i in queryset_values]): obj, created = GuideWineColorSection.objects.get_or_create( name=section_name ) if created: created_count += 1 print(f'OBJECTS CREATED: {created_count}') def transfer_guide_element_type(): """Transfer GuideElements model (only element types).""" created_count = 0 queryset_values = GuideElements.objects.raw( """ select distinct(type), 1 as id from guide_elements; """ ) for element_type in tqdm([i.type for i in queryset_values]): obj, created = GuideElementType.objects.get_or_create( name=element_type ) if created: created_count += 1 print(f'OBJECTS CREATED: {created_count}') def transfer_guide_elements_bulk(): """Transfer Guide elements via bulk_create.""" def get_guide_element_type(guide_element_type: str): if guide_element_type: qs = GuideElementType.objects.filter(name__iexact=guide_element_type) if qs.exists(): return qs.first() def get_establishment(old_id: int): if old_id: qs = Establishment.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_review(old_id: int): if old_id: qs = Review.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_wine_region(old_id: int): if old_id: qs = WineRegion.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_wine(old_id: int): if old_id: qs = Product.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_wine_color_section(color_section: str): if color_section: qs = GuideWineColorSection.objects.filter(name__iexact=color_section) if qs.exists(): return qs.first() def get_city(old_id: int): if old_id: qs = City.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_guide_element_section(old_id: int): if old_id: qs = GuideElementSection.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_guide(old_id): if old_id: qs = Guide.objects.filter(old_id=old_id) if qs.exists(): return qs.first() def get_parent(old_id): if old_id: qs = GuideElement.objects.filter(old_id=old_id) if qs.exists(): return qs.first() objects_to_update = [] base_queryset = GuideElements.objects.filter(guide_id=407) for old_id, type, establishment_id, review_id, wine_region_id, \ wine_id, color, order_number, city_id, section_id, guide_id \ in tqdm(base_queryset.filter(parent_id__isnull=True) .values_list('id', 'type', 'establishment_id', 'review_id', 'wine_region_id', 'wine_id', 'color', 'order_number', 'city_id', 'section_id', 'guide_id'), desc='Check parent guide elements'): if not GuideElement.objects.filter(old_id=old_id).exists(): guide = GuideElement( old_id=old_id, guide_element_type=get_guide_element_type(type), establishment=get_establishment(establishment_id), review=get_review(review_id), wine_region=get_wine_region(wine_region_id), product=get_wine(wine_id), wine_color_section=get_wine_color_section(color), priority=order_number, city=get_city(city_id), section=get_guide_element_section(section_id), parent=None, lft=1, rght=1, tree_id=1, level=1, ) # check old guide if not guide_id: objects_to_update.append(guide) else: old_guide = Guides.objects.exclude(title__icontains='test') \ .filter(id=guide_id) if old_guide.exists(): guide.guide = get_guide(guide_id) objects_to_update.append(guide) # create parents GuideElement.objects.bulk_create(objects_to_update) print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') # attach child guide elements created_child = 0 queryset_values = base_queryset.exclude(parent_id__isnull=True) \ .values_list('id', 'type', 'establishment_id', 'review_id', 'wine_region_id', 'wine_id', 'color', 'order_number', 'city_id', 'section_id', 'guide_id', 'rgt', 'lft', 'parent_id') for old_id, type, establishment_id, review_id, wine_region_id, \ wine_id, color, order_number, city_id, section_id, guide_id, \ lft, rgt, parent_id \ in tqdm(sorted(queryset_values, key=lambda value: operator.itemgetter(-2, -1)(value)), desc='Check child guide elements'): if not GuideElement.objects.filter(old_id=old_id).exists(): # check old guide if guide_id: old_guide = Guides.objects.exclude(title__icontains='test') \ .filter(id=guide_id) if old_guide.exists(): guide_element, created = GuideElement.objects.get_or_create( old_id=old_id, guide_element_type=get_guide_element_type(type), establishment=get_establishment(establishment_id), review=get_review(review_id), wine_region=get_wine_region(wine_region_id), product=get_wine(wine_id), wine_color_section=get_wine_color_section(color), priority=order_number, city=get_city(city_id), section=get_guide_element_section(section_id), parent=get_parent(parent_id), lft=1, rght=1, tree_id=1, level=1, guide=get_guide(guide_id), ) if created: created_child += 1 print(f'CREATED {created_child} OBJECTS') # rebuild trees GuideElement._tree_manager.rebuild() # record the total count of descendants objects objects_to_update = [] for guide in tqdm(Guide.objects.all(), desc='update count_of_initial_objects field values'): descendants = GuideElement.objects.get_root_node(guide).get_descendants() if descendants: count_of_initial_objects = descendants.filter( guide_element_type__name__in=['EstablishmentNode', 'WineNode']).count() guide.count_related_objects = count_of_initial_objects objects_to_update.append(guide) # update count_of_initial_objects field values Guide.objects.bulk_update(objects_to_update, ['count_objects_during_init', ]) print(f'COUNT OF UPDATED OBJECTS: {len(objects_to_update)}') def transfer_guide_element_advertorials(): """Transfer Guide Advertorials model.""" def get_guide_element(old_id: int): if old_id: qs = GuideElement.objects.filter(old_id=old_id) legacy_qs = GuideElements.objects.exclude(guide__isnull=True) \ .exclude(guide__title__icontains='test') \ .filter(id=old_id) if qs.exists() and legacy_qs.exists(): return qs.first() elif legacy_qs.exists() and not qs.exists(): raise ValueError(f'Guide element was not transfer correctly - {old_id}.') objects_to_update = [] advertorials = GuideAds.objects.exclude(nb_pages__isnull=True) \ .exclude(nb_right_pages__isnull=True) \ .exclude(guide_ad_node_id__isnull=True) \ .values_list('id', 'nb_pages', 'nb_right_pages', 'guide_ad_node_id') for old_id, nb_pages, nb_right_pages, guide_ad_node_id in tqdm(advertorials): # check guide element guide_element = get_guide_element(guide_ad_node_id) if not Advertorial.objects.filter(old_id=old_id).exists() and guide_element: objects_to_update.append( Advertorial( old_id=old_id, number_of_pages=nb_pages, right_pages=nb_right_pages, guide_element=guide_element, ) ) # create related child Advertorial.objects.bulk_create(objects_to_update) pprint(f'CREATED ADVERTORIALS W/ OLD_ID: {[i.old_id for i in objects_to_update]}') print(f'COUNT OF CREATED OBJECTS: {len(objects_to_update)}') def transfer_guide_element_label_photo(): """Transfer galleries for Guide Advertorial model.""" def get_guide_element(guide_ad): legacy_guide_element_id = guide_ad.guide_ad_node.id legacy_guide_element_qs = GuideElements.objects.filter(id=legacy_guide_element_id) guide_element_qs = GuideElement.objects.filter(old_id=legacy_guide_element_id) if guide_element_qs.exists() and legacy_guide_element_qs.exists(): return guide_element_qs.first() else: raise ValueError(f'Guide element was not transfer correctly - ' f'{legacy_guide_element_id}.') to_update = [] not_updated = 0 guide_element_label_photos = LabelPhotos.objects.exclude(guide_ad__isnull=True) \ .filter(guide_ad__type='GuideAdLabel') \ .distinct() \ .values_list('guide_ad', 'attachment_suffix_url') for guide_ad_id, attachment_suffix_url in tqdm(guide_element_label_photos): legacy_guide_element_ids = Subquery( GuideElements.objects.exclude(guide__isnull=True) .exclude(guide__title__icontains='test') .values_list('id', flat=True) ) legacy_guide_ad_qs = GuideAds.objects.filter(id=guide_ad_id, guide_ad_node_id__in=legacy_guide_element_ids) if legacy_guide_ad_qs.exists(): guide_element = get_guide_element(legacy_guide_ad_qs.first()) if guide_element: image, _ = Image.objects.get_or_create(image=attachment_suffix_url, defaults={ 'image': attachment_suffix_url, 'orientation': Image.HORIZONTAL, 'title': f'{guide_element.__str__()} ' f'{guide_element.id} - ' f'{attachment_suffix_url}'}) if not guide_element.label_photo: guide_element.label_photo = image to_update.append(guide_element) else: not_updated += 1 GuideElement.objects.bulk_update(to_update, ['label_photo', ]) print(f'Added label photo to {len(to_update)} objects\n' f'Objects {not_updated} not updated') data_types = { 'guides': [ transfer_guide, ], 'guide_filters': [ transfer_guide_filter, ], 'guide_element_sections': [ transfer_guide_element_section, ], 'guide_wine_color_sections': [ transfer_guide_wine_color_section, ], 'guide_element_types': [ transfer_guide_element_type, ], 'guide_elements_bulk': [ transfer_guide_elements_bulk, ], 'guide_element_advertorials': [ transfer_guide_element_advertorials, ], 'guide_element_label_photo': [ transfer_guide_element_label_photo, ], 'guide_complete': [ transfer_guide, # transfer guides from Guides transfer_guide_filter, # transfer guide filters from GuideFilters transfer_guide_element_section, # partial transfer element section from GuideSections transfer_guide_wine_color_section, # partial transfer wine color section from GuideSections transfer_guide_element_type, # partial transfer section types from GuideElements transfer_guide_elements_bulk, # transfer result of GuideFilters from GuideElements transfer_guide_element_advertorials, # transfer advertorials that linked to GuideElements transfer_guide_element_label_photo, # transfer guide element label photos ] }