cf7cc9bd0c
# Conflicts: # apps/transfer/management/commands/transfer.py # apps/transfer/serializers/location.py
382 lines
16 KiB
Python
382 lines
16 KiB
Python
from transfer.serializers import location as location_serializers
|
|
from transfer import models as transfer_models
|
|
from location.models import Country, Region
|
|
from pprint import pprint
|
|
import json
|
|
from django.conf import settings
|
|
from django.core.exceptions import MultipleObjectsReturned
|
|
|
|
from requests import get
|
|
|
|
|
|
def transfer_countries():
|
|
queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT cities.id, cities.country_code_2
|
|
FROM cities
|
|
WHERE country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
GROUP BY cities.id, cities.country_code_2
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CountrySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Country serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_regions():
|
|
regions_without_subregion_queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
FROM cities
|
|
WHERE (subregion_code IS NULL
|
|
OR subregion_code = ""
|
|
)
|
|
AND region_code IS NOT NULL
|
|
AND region_code != ""
|
|
AND country_code_2 IS NOT NULL
|
|
AND country_code_2 != ""
|
|
GROUP BY cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
""")
|
|
|
|
regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]
|
|
|
|
serialized_without_subregion = location_serializers.RegionSerializer(data=regions_without_subregion_queryset,
|
|
many=True)
|
|
if serialized_without_subregion.is_valid():
|
|
serialized_without_subregion.save()
|
|
else:
|
|
pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")
|
|
|
|
regions_with_subregion_queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT
|
|
cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
FROM cities
|
|
WHERE subregion_code IS NOT NULL AND
|
|
subregion_code != "" AND
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
AND cities.subregion_code in
|
|
(
|
|
SELECT region_code FROM cities WHERE
|
|
(subregion_code IS NULL OR
|
|
subregion_code = "") AND
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
)
|
|
GROUP BY cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
""")
|
|
|
|
regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]
|
|
|
|
serialized_with_subregion = location_serializers.RegionSerializer(data=regions_with_subregion_queryset,
|
|
many=True)
|
|
if serialized_with_subregion.is_valid():
|
|
serialized_with_subregion.save()
|
|
else:
|
|
pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")
|
|
|
|
|
|
def transfer_cities():
|
|
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
|
|
cities.is_island, cities.region_code, cities.subregion_code
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CitySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"City serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_addresses():
|
|
queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude,
|
|
locations.latitude, locations.address, locations.city_id
|
|
FROM locations WHERE
|
|
locations.address != "" AND
|
|
locations.address IS NOT NULL AND
|
|
locations.city_id IS NOT NULL AND
|
|
locations.city_id IN (SELECT cities.id
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != "")""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.AddressSerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Address serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_region():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='WineRegion')
|
|
serialized_data = location_serializers.WineRegion(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_sub_region():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion')
|
|
serialized_data = location_serializers.WineSubRegion(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_village():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='Village')
|
|
serialized_data = location_serializers.WineVillage(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def update_flags():
|
|
queryset = Country.objects.only("id", "code", "svg_image").filter(old_id__isnull=False)
|
|
link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media"
|
|
|
|
for query in queryset:
|
|
svg_link = f"/svg/country/10-31-2019/{query.code}.svg"
|
|
resp = get(f"{link_to_request}{svg_link}")
|
|
if resp.status_code == 200:
|
|
query.svg_image = svg_link
|
|
query.save()
|
|
|
|
|
|
def migrate_city_map_situation():
|
|
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
|
|
cities.is_island, cities.region_code, cities.subregion_code
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
""")
|
|
|
|
serialized_data = location_serializers.CityMapSerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"City info serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def migrate_city_photos():
|
|
queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name
|
|
FROM city_photos WHERE
|
|
city_photos.attachment_file_name IS NOT NULL AND
|
|
city_id IN(
|
|
SELECT cities.id
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
)
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Address serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
"""
|
|
Update location models with ruby library
|
|
Utils functions defined before transfer functions
|
|
"""
|
|
|
|
def get_ruby_socket(type, params):
|
|
url = 'http://172.21.0.1:5678' # docker host
|
|
params["type"] = type
|
|
response = get(url, params=params)
|
|
|
|
data = json.loads(response.text)
|
|
return data
|
|
|
|
|
|
def update_child_regions():
|
|
child_regions_list = Region.objects.filter(old_id__isnull=False, parent_region__isnull=False)
|
|
ruby_data_to_write = {}
|
|
|
|
try:
|
|
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file:
|
|
ruby_regions_list = json.loads(file.read())
|
|
except:
|
|
ruby_regions_list = {}
|
|
|
|
for child_region in child_regions_list:
|
|
mysql_region = transfer_models.Cities.objects.get(id=child_region.old_id)
|
|
|
|
|
|
if child_region.id in ruby_regions_list:
|
|
ruby_data = ruby_regions_list[child_region.id]
|
|
else:
|
|
ruby_params = {
|
|
"country_code": mysql_region.country_code,
|
|
"country_code_2": mysql_region.country_code_2,
|
|
"region_code": mysql_region.region_code,
|
|
"region_name": mysql_region.name,
|
|
"subregion_code": mysql_region.subregion_code
|
|
}
|
|
|
|
ruby_data = get_ruby_socket("subregion", ruby_params)
|
|
if "error" in ruby_data:
|
|
continue
|
|
|
|
ruby_data_to_write[child_region.id] = ruby_data
|
|
|
|
try:
|
|
parent_region = Region.objects.get(
|
|
code=ruby_data['region'],
|
|
country__code=ruby_data['country']
|
|
)
|
|
country = parent_region.country
|
|
except MultipleObjectsReturned:
|
|
parent_region = Region.objects.filter(
|
|
code=ruby_data['region'],
|
|
country__code=ruby_data['country']
|
|
).first()
|
|
country = parent_region.country
|
|
except Region.DoesNotExist:
|
|
try:
|
|
parent_region = Region.objects.get(code=ruby_data['region'])
|
|
country = Country.objects.get(code=ruby_data['country'])
|
|
except MultipleObjectsReturned:
|
|
parent_region = Region.objects.filter(code=ruby_data['region']).first()
|
|
country = parent_region.country
|
|
|
|
child_region.parent_region = parent_region
|
|
child_region.country = country
|
|
child_region.save()
|
|
|
|
if len(ruby_data_to_write) > 0:
|
|
ruby_data_to_write = {**ruby_data_to_write, **ruby_regions_list}
|
|
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file:
|
|
file.write(json.dumps(ruby_data_to_write))
|
|
|
|
|
|
# def update_relations(old_instance, new_instance, relations_dict):
|
|
# for relation_name, relation_field in relations_dict.items():
|
|
# relations_to_update = getattr(old_instance, relation_name, None)
|
|
# print(relations_to_update)
|
|
# if relations_to_update is None:
|
|
# continue
|
|
# # elif isinstance(relations_to_update)
|
|
#
|
|
# # relations_to_update = relations_to_update.all()
|
|
#
|
|
# # if len(relations_to_update) > 0:
|
|
#
|
|
#
|
|
# def update_country_from_ruby():
|
|
# countries = Country.objects.filter(old_id__isnull=False).distinct('code')
|
|
#
|
|
# json_countries_list = []
|
|
# for country in countries:
|
|
# json_country_dict = {
|
|
# "old_id": [country.old_id]
|
|
# }
|
|
# mysql_country = transfer_models.Cities.objects.get(id=country.old_id)
|
|
#
|
|
# ruby_params = {
|
|
# "country_code": mysql_country.country_code,
|
|
# "country_code_2": mysql_country.country_code_2,
|
|
# "region_code": mysql_country.region_code
|
|
# }
|
|
#
|
|
# ruby_data = get_ruby_socket("country", ruby_params)
|
|
#
|
|
# if "error" in ruby_data:
|
|
# print("ERROR: ")
|
|
# print(f"{ruby_data['error']}: {ruby_params}")
|
|
# continue
|
|
#
|
|
# json_country_dict["data"] = ruby_data
|
|
#
|
|
# duplicates = Country.objects.\
|
|
# filter(
|
|
# old_id__isnull=False,
|
|
# code=country.code
|
|
# ).exclude(old_id=country.old_id)
|
|
#
|
|
# for duplicate in duplicates:
|
|
# json_country_dict["old_id"].append(duplicate.old_id)
|
|
#
|
|
# json_countries_list.append(json_country_dict)
|
|
#
|
|
# with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/countries.py", "w") as file:
|
|
# file.write(json.dumps(json_countries_list))
|
|
#
|
|
#
|
|
|
|
|
|
|
|
|
|
|
|
data_types = {
|
|
"dictionaries": [
|
|
transfer_countries,
|
|
transfer_regions,
|
|
transfer_cities,
|
|
transfer_addresses,
|
|
transfer_wine_region,
|
|
transfer_wine_sub_region,
|
|
transfer_wine_village,
|
|
],
|
|
"update_country_flag": [
|
|
update_flags
|
|
],
|
|
"update_city_info": [
|
|
migrate_city_map_situation
|
|
],
|
|
"migrate_city_gallery": [
|
|
migrate_city_photos
|
|
],
|
|
"tmp": [
|
|
update_child_regions
|
|
]
|
|
|
|
}
|