from transfer.serializers import location as location_serializers from transfer import models as transfer_models from location.models import Country, Region from pprint import pprint import json from django.conf import settings from django.core.exceptions import MultipleObjectsReturned from requests import get def transfer_countries(): queryset = transfer_models.Cities.objects.raw(""" SELECT cities.id, cities.country_code_2 FROM cities WHERE country_code_2 IS NOT NULL AND country_code_2 != "" GROUP BY cities.id, cities.country_code_2 """) queryset = [vars(query) for query in queryset] serialized_data = location_serializers.CountrySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"Country serializer errors: {serialized_data.errors}") def transfer_regions(): regions_without_subregion_queryset = transfer_models.Cities.objects.raw(""" SELECT cities.id, cities.region_code, cities.country_code_2, cities.subregion_code FROM cities WHERE (subregion_code IS NULL OR subregion_code = "" ) AND region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" GROUP BY cities.id, cities.region_code, cities.country_code_2, cities.subregion_code """) regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] serialized_without_subregion = location_serializers.RegionSerializer(data=regions_without_subregion_queryset, many=True) if serialized_without_subregion.is_valid(): serialized_without_subregion.save() else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") regions_with_subregion_queryset = transfer_models.Cities.objects.raw(""" SELECT cities.id, cities.region_code, cities.country_code_2, cities.subregion_code FROM cities WHERE subregion_code IS NOT NULL AND subregion_code != "" AND region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" AND cities.subregion_code in ( SELECT region_code FROM cities WHERE (subregion_code IS NULL OR subregion_code = "") AND region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" ) GROUP BY cities.id, cities.region_code, cities.country_code_2, cities.subregion_code """) regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] serialized_with_subregion = location_serializers.RegionSerializer(data=regions_with_subregion_queryset, many=True) if serialized_with_subregion.is_valid(): serialized_with_subregion.save() else: pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") def transfer_cities(): queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, cities.is_island, cities.region_code, cities.subregion_code FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" """) queryset = [vars(query) for query in queryset] serialized_data = location_serializers.CitySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"City serializer errors: {serialized_data.errors}") def transfer_addresses(): queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, locations.latitude, locations.address, locations.city_id FROM locations WHERE locations.address != "" AND locations.address IS NOT NULL AND locations.city_id IS NOT NULL AND locations.city_id IN (SELECT cities.id FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "")""") queryset = [vars(query) for query in queryset] serialized_data = location_serializers.AddressSerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"Address serializer errors: {serialized_data.errors}") def transfer_wine_region(): queryset = transfer_models.WineLocations.objects.filter(type='WineRegion') serialized_data = location_serializers.WineRegion( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}") def transfer_wine_sub_region(): queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion') serialized_data = location_serializers.WineSubRegion( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}") def transfer_wine_village(): queryset = transfer_models.WineLocations.objects.filter(type='Village') serialized_data = location_serializers.WineVillage( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}") def update_flags(): queryset = Country.objects.only("id", "code", "svg_image").filter(old_id__isnull=False) link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media" for query in queryset: svg_link = f"/svg/country/10-31-2019/{query.code}.svg" resp = get(f"{link_to_request}{svg_link}") if resp.status_code == 200: query.svg_image = svg_link query.save() def migrate_city_map_situation(): queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, cities.is_island, cities.region_code, cities.subregion_code FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" """) serialized_data = location_serializers.CityMapSerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"City info serializer errors: {serialized_data.errors}") def migrate_city_photos(): queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name FROM city_photos WHERE city_photos.attachment_file_name IS NOT NULL AND city_id IN( SELECT cities.id FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" ) """) queryset = [vars(query) for query in queryset] serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"Address serializer errors: {serialized_data.errors}") """ Update location models with ruby library Utils functions defined before transfer functions """ def get_ruby_socket(type, params): url = 'http://172.21.0.1:5678' # docker host params["type"] = type response = get(url, params=params) data = json.loads(response.text) return data def update_child_regions(): child_regions_list = Region.objects.filter(old_id__isnull=False, parent_region__isnull=False) ruby_data_to_write = {} try: with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file: ruby_regions_list = json.loads(file.read()) except: ruby_regions_list = {} for child_region in child_regions_list: mysql_region = transfer_models.Cities.objects.get(id=child_region.old_id) if child_region.id in ruby_regions_list: ruby_data = ruby_regions_list[child_region.id] else: ruby_params = { "country_code": mysql_region.country_code, "country_code_2": mysql_region.country_code_2, "region_code": mysql_region.region_code, "region_name": mysql_region.name, "subregion_code": mysql_region.subregion_code } ruby_data = get_ruby_socket("subregion", ruby_params) if "error" in ruby_data: continue ruby_data_to_write[child_region.id] = ruby_data try: parent_region = Region.objects.get( code=ruby_data['region'], country__code=ruby_data['country'] ) country = parent_region.country except MultipleObjectsReturned: parent_region = Region.objects.filter( code=ruby_data['region'], country__code=ruby_data['country'] ).first() country = parent_region.country except Region.DoesNotExist: try: parent_region = Region.objects.get(code=ruby_data['region']) country = Country.objects.get(code=ruby_data['country']) except MultipleObjectsReturned: parent_region = Region.objects.filter(code=ruby_data['region']).first() country = parent_region.country child_region.parent_region = parent_region child_region.country = country child_region.save() if len(ruby_data_to_write) > 0: ruby_data_to_write = {**ruby_data_to_write, **ruby_regions_list} with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file: file.write(json.dumps(ruby_data_to_write)) # def update_relations(old_instance, new_instance, relations_dict): # for relation_name, relation_field in relations_dict.items(): # relations_to_update = getattr(old_instance, relation_name, None) # print(relations_to_update) # if relations_to_update is None: # continue # # elif isinstance(relations_to_update) # # # relations_to_update = relations_to_update.all() # # # if len(relations_to_update) > 0: # # # def update_country_from_ruby(): # countries = Country.objects.filter(old_id__isnull=False).distinct('code') # # json_countries_list = [] # for country in countries: # json_country_dict = { # "old_id": [country.old_id] # } # mysql_country = transfer_models.Cities.objects.get(id=country.old_id) # # ruby_params = { # "country_code": mysql_country.country_code, # "country_code_2": mysql_country.country_code_2, # "region_code": mysql_country.region_code # } # # ruby_data = get_ruby_socket("country", ruby_params) # # if "error" in ruby_data: # print("ERROR: ") # print(f"{ruby_data['error']}: {ruby_params}") # continue # # json_country_dict["data"] = ruby_data # # duplicates = Country.objects.\ # filter( # old_id__isnull=False, # code=country.code # ).exclude(old_id=country.old_id) # # for duplicate in duplicates: # json_country_dict["old_id"].append(duplicate.old_id) # # json_countries_list.append(json_country_dict) # # with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/countries.py", "w") as file: # file.write(json.dumps(json_countries_list)) # # data_types = { "dictionaries": [ transfer_countries, transfer_regions, transfer_cities, transfer_addresses, transfer_wine_region, transfer_wine_sub_region, transfer_wine_village, ], "update_country_flag": [ update_flags ], "update_city_info": [ migrate_city_map_situation ], "migrate_city_gallery": [ migrate_city_photos ], "tmp": [ update_child_regions ] }