from transfer.serializers import location as location_serializers from transfer import models as transfer_models from location.models import Country, Region, City from pprint import pprint import json from django.conf import settings from django.core.exceptions import MultipleObjectsReturned from requests import get def transfer_countries(): queryset = transfer_models.Cities.objects.raw(""" SELECT cities.id, cities.country_code_2 FROM cities WHERE country_code_2 IS NOT NULL AND country_code_2 != "" GROUP BY cities.id, cities.country_code_2 """) queryset = [vars(query) for query in queryset] serialized_data = location_serializers.CountrySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"Country serializer errors: {serialized_data.errors}") def transfer_regions(): regions_without_subregion_queryset = transfer_models.Cities.objects.raw(""" SELECT cities.id, cities.region_code, cities.country_code_2, cities.subregion_code FROM cities WHERE (subregion_code IS NULL OR subregion_code = "" ) AND region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" GROUP BY cities.id, cities.region_code, cities.country_code_2, cities.subregion_code """) regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] serialized_without_subregion = location_serializers.RegionSerializer(data=regions_without_subregion_queryset, many=True) if serialized_without_subregion.is_valid(): serialized_without_subregion.save() else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") regions_with_subregion_queryset = transfer_models.Cities.objects.raw(""" SELECT cities.id, cities.region_code, cities.country_code_2, cities.subregion_code FROM cities WHERE subregion_code IS NOT NULL AND subregion_code != "" AND region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" AND cities.subregion_code in ( SELECT region_code FROM cities WHERE (subregion_code IS NULL OR subregion_code = "") AND region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" ) GROUP BY cities.id, cities.region_code, cities.country_code_2, cities.subregion_code """) regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] serialized_with_subregion = location_serializers.RegionSerializer(data=regions_with_subregion_queryset, many=True) if serialized_with_subregion.is_valid(): serialized_with_subregion.save() else: pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") def transfer_cities(): queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, cities.is_island, cities.region_code, cities.subregion_code FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" """) queryset = [vars(query) for query in queryset] serialized_data = location_serializers.CitySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"City serializer errors: {serialized_data.errors}") def transfer_addresses(): queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, locations.latitude, locations.address, locations.city_id FROM locations WHERE locations.address != "" AND locations.address IS NOT NULL AND locations.city_id IS NOT NULL AND locations.city_id IN (SELECT cities.id FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "")""") queryset = [vars(query) for query in queryset] serialized_data = location_serializers.AddressSerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"Address serializer errors: {serialized_data.errors}") def transfer_wine_region(): queryset = transfer_models.WineLocations.objects.filter(type='WineRegion') serialized_data = location_serializers.WineRegion( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}") def transfer_wine_sub_region(): queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion') serialized_data = location_serializers.WineSubRegion( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}") def transfer_wine_village(): queryset = transfer_models.WineLocations.objects.filter(type='Village') serialized_data = location_serializers.WineVillage( data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}") def update_flags(): queryset = Country.objects.only("id", "code", "svg_image").filter(old_id__isnull=False) link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media" for query in queryset: svg_link = f"/svg/country/10-31-2019/{query.code}.svg" resp = get(f"{link_to_request}{svg_link}") if resp.status_code == 200: query.svg_image = svg_link query.save() def migrate_city_map_situation(): queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, cities.is_island, cities.region_code, cities.subregion_code FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" """) serialized_data = location_serializers.CityMapSerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"City info serializer errors: {serialized_data.errors}") def migrate_city_photos(): queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name FROM city_photos WHERE city_photos.attachment_file_name IS NOT NULL AND city_id IN( SELECT cities.id FROM cities WHERE region_code IS NOT NULL AND region_code != "" AND country_code_2 IS NOT NULL AND country_code_2 != "" ) """) queryset = [vars(query) for query in queryset] serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: pprint(f"Address serializer errors: {serialized_data.errors}") """ Update location models with ruby library Utils functions defined before transfer functions """ def get_ruby_socket(params): url = 'http://172.21.0.1:5678' # docker host response = get(url, params=params) try: data = json.loads(response.text) except Exception as e: print(f"{response.text} error: {e}") return None return data def get_ruby_data(): cities = City.objects.filter(old_id__isnull=False) ruby_data = {} error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w") for city in cities: try: mysql_city = transfer_models.Cities.objects.get(id=city.old_id) except transfer_models.Cities.DoesNotExist: print(f"City with id {city.old_id} not found") continue ruby_params = {} if mysql_city.country_code is not None: ruby_params['country_code'] = mysql_city.country_code if mysql_city.country_code_2 is not None: ruby_params['country_code_2'] = mysql_city.country_code_2 if mysql_city.region_code is not None: ruby_params['region_code'] = mysql_city.region_code if mysql_city.subregion_code is not None: ruby_params['subregion_code'] = mysql_city.subregion_code ruby_response = get_ruby_socket(ruby_params) if ruby_response is None: continue if "error" in ruby_response or "country" not in ruby_response: error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n") continue ruby_data[mysql_city.id] = { "mysql_id": mysql_city.id, "name": mysql_city.name, "name_translated": '{"en-GB":"' + mysql_city.name + '"}', "map1": mysql_city.map1, "map2": mysql_city.map2, "map_ref": mysql_city.map_ref, "situation": mysql_city.situation, "is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False, "postal_code": mysql_city.zip_code if mysql_city.zip_code is not None else '', "code": mysql_city.country_code_2.lower() } for ruby_data_key in ruby_response.keys(): if "error" in ruby_response[ruby_data_key]: error_file.write(json.dumps(ruby_response) + '\n') else: ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key] error_file.close() with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file: ruby_data_file.write(json.dumps(ruby_data)) return ruby_data def fix_location_models(): try: ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r") ruby_data = json.loads(ruby_data_file.read()) except FileNotFoundError: ruby_data = get_ruby_data() for mysql_id, city_object in ruby_data.items(): country_data = city_object["country"] try: country = Country.objects.get(code=country_data['code'], mysql_ids__isnull=False) country.mysql_ids.append(mysql_id) country.save() except Country.DoesNotExist: country_data['mysql_ids'] = [mysql_id] country = Country.objects.create(**country_data) city_object['country'] = country if "region" in city_object: region_data = city_object['region'] region_data['country'] = country try: region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False) region.mysql_ids.append(mysql_id) region.save() except Region.DoesNotExist: region_data['mysql_ids'] = [mysql_id] region = Region.objects.create(**region_data) if "subregion" in city_object: subregion_data = city_object.pop('subregion') subregion_data['country'] = country try: subregion = Region.objects.get(code=subregion_data['code'], mysql_ids__isnull=False) subregion.mysql_ids.append(mysql_id) subregion.save() except Region.DoesNotExist: subregion_data['parent_region'] = region subregion_data['mysql_ids'] = [mysql_id] subregion = Region.objects.create(**subregion_data) city_object['region'] = subregion else: city_object['region'] = region if "subregion" in city_object: del(city_object['subregion']) try: City.objects.create(**city_object) except Exception as e: print(city_object) print(e) break data_types = { "dictionaries": [ transfer_countries, transfer_regions, transfer_cities, transfer_addresses, transfer_wine_region, transfer_wine_sub_region, transfer_wine_village, ], "update_country_flag": [ update_flags ], "update_city_info": [ migrate_city_map_situation ], "migrate_city_gallery": [ migrate_city_photos ], "tmp": [ fix_location_models ] }