432 lines
18 KiB
Python
432 lines
18 KiB
Python
from transfer.serializers import location as location_serializers
|
|
from transfer import models as transfer_models
|
|
from location.models import Country, Region, City, Address, WineRegion
|
|
from pprint import pprint
|
|
import json
|
|
|
|
from django.conf import settings
|
|
from django.core.exceptions import MultipleObjectsReturned
|
|
from collection.models import Collection
|
|
from requests import get
|
|
|
|
|
|
def transfer_countries():
|
|
queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT cities.id, cities.country_code_2
|
|
FROM cities
|
|
WHERE country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
GROUP BY cities.id, cities.country_code_2
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CountrySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Country serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_regions():
|
|
regions_without_subregion_queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
FROM cities
|
|
WHERE (subregion_code IS NULL
|
|
OR subregion_code = ""
|
|
)
|
|
AND region_code IS NOT NULL
|
|
AND region_code != ""
|
|
AND country_code_2 IS NOT NULL
|
|
AND country_code_2 != ""
|
|
GROUP BY cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
""")
|
|
|
|
regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]
|
|
|
|
serialized_without_subregion = location_serializers.RegionSerializer(data=regions_without_subregion_queryset,
|
|
many=True)
|
|
if serialized_without_subregion.is_valid():
|
|
serialized_without_subregion.save()
|
|
else:
|
|
pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")
|
|
|
|
regions_with_subregion_queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT
|
|
cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
FROM cities
|
|
WHERE subregion_code IS NOT NULL AND
|
|
subregion_code != "" AND
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
AND cities.subregion_code in
|
|
(
|
|
SELECT region_code FROM cities WHERE
|
|
(subregion_code IS NULL OR
|
|
subregion_code = "") AND
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
)
|
|
GROUP BY cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
""")
|
|
|
|
regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]
|
|
|
|
serialized_with_subregion = location_serializers.RegionSerializer(data=regions_with_subregion_queryset,
|
|
many=True)
|
|
if serialized_with_subregion.is_valid():
|
|
serialized_with_subregion.save()
|
|
else:
|
|
pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")
|
|
|
|
|
|
def transfer_cities():
|
|
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
|
|
cities.is_island, cities.region_code, cities.subregion_code
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CitySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"City serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_addresses():
|
|
queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude,
|
|
locations.latitude, locations.address, locations.city_id
|
|
FROM locations WHERE
|
|
locations.address != "" AND
|
|
locations.address IS NOT NULL AND
|
|
locations.city_id IS NOT NULL AND
|
|
locations.city_id IN (SELECT cities.id
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != "")""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.AddressSerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Address serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_region():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='WineRegion')
|
|
serialized_data = location_serializers.WineRegion(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_sub_region():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion')
|
|
serialized_data = location_serializers.WineSubRegion(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_village():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='Village')
|
|
serialized_data = location_serializers.WineVillage(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def update_flags():
|
|
queryset = Country.objects.only("id", "code", "svg_image").filter(old_id__isnull=False)
|
|
link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media"
|
|
|
|
for query in queryset:
|
|
svg_link = f"/svg/country/10-31-2019/{query.code}.svg"
|
|
resp = get(f"{link_to_request}{svg_link}")
|
|
if resp.status_code == 200:
|
|
query.svg_image = svg_link
|
|
query.save()
|
|
|
|
|
|
def migrate_city_map_situation():
|
|
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
|
|
cities.is_island, cities.region_code, cities.subregion_code
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
""")
|
|
|
|
serialized_data = location_serializers.CityMapSerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"City info serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def migrate_city_photos():
|
|
queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name
|
|
FROM city_photos WHERE
|
|
city_photos.attachment_file_name IS NOT NULL AND
|
|
city_id IN(
|
|
SELECT cities.id
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
)
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Address serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
# Update location models with ruby library
|
|
# Utils functions defined before transfer functions
|
|
def get_ruby_socket(params):
|
|
url = 'http://172.21.0.1:5678' # docker host
|
|
response = get(url, params=params)
|
|
try:
|
|
data = json.loads(response.text)
|
|
except Exception as e:
|
|
print(f"{response.text} error: {e}")
|
|
return None
|
|
return data
|
|
|
|
|
|
# Get data from ruby and save it to file
|
|
# Save errors from ruby to another file
|
|
def get_ruby_data():
|
|
cities = City.objects.filter(old_id__isnull=False)
|
|
ruby_data = {}
|
|
error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w")
|
|
|
|
for city in cities:
|
|
try:
|
|
mysql_city = transfer_models.Cities.objects.get(id=city.old_id)
|
|
except transfer_models.Cities.DoesNotExist:
|
|
print(f"City with id {city.old_id} not found")
|
|
continue
|
|
|
|
ruby_params = {}
|
|
|
|
if mysql_city.country_code is not None:
|
|
ruby_params['country_code'] = mysql_city.country_code
|
|
|
|
if mysql_city.country_code_2 is not None:
|
|
ruby_params['country_code_2'] = mysql_city.country_code_2
|
|
|
|
if mysql_city.region_code is not None:
|
|
ruby_params['region_code'] = mysql_city.region_code
|
|
|
|
if mysql_city.subregion_code is not None:
|
|
ruby_params['subregion_code'] = mysql_city.subregion_code
|
|
|
|
ruby_response = get_ruby_socket(ruby_params)
|
|
|
|
if ruby_response is None:
|
|
continue
|
|
|
|
if "error" in ruby_response or "country" not in ruby_response:
|
|
error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
|
|
continue
|
|
|
|
ruby_data[mysql_city.id] = {
|
|
"mysql_id": mysql_city.id,
|
|
"name": mysql_city.name,
|
|
"name_translated": '{"en-GB":"' + mysql_city.name + '"}',
|
|
|
|
"map1": mysql_city.map1,
|
|
"map2": mysql_city.map2,
|
|
"map_ref": mysql_city.map_ref,
|
|
"situation": mysql_city.situation,
|
|
|
|
"is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False,
|
|
|
|
"postal_code": mysql_city.zip_code if mysql_city.zip_code is not None else '',
|
|
"code": mysql_city.country_code_2.lower()
|
|
|
|
}
|
|
|
|
for ruby_data_key in ruby_response.keys():
|
|
if "error" in ruby_response[ruby_data_key]:
|
|
error_file.write(json.dumps(ruby_response) + '\n')
|
|
else:
|
|
ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key]
|
|
|
|
error_file.close()
|
|
|
|
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file:
|
|
ruby_data_file.write(json.dumps(ruby_data))
|
|
|
|
return ruby_data
|
|
|
|
|
|
# Add correct objects of Country, Region and City with mysql_ids array (Country, Region) and mysql_id (City)
|
|
def add_correct_location_models(ruby_data):
|
|
for mysql_id, city_object in ruby_data.items():
|
|
country_data = city_object["country"]
|
|
try:
|
|
country = Country.objects.get(code=country_data['code'], mysql_ids__isnull=False)
|
|
country.mysql_ids.append(mysql_id)
|
|
country.save()
|
|
except Country.DoesNotExist:
|
|
country_data['mysql_ids'] = [mysql_id]
|
|
country = Country.objects.create(**country_data)
|
|
|
|
city_object['country'] = country
|
|
|
|
if "region" in city_object:
|
|
region_data = city_object['region']
|
|
region_data['country'] = country
|
|
try:
|
|
region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False)
|
|
region.mysql_ids.append(mysql_id)
|
|
region.save()
|
|
except Region.DoesNotExist:
|
|
region_data['mysql_ids'] = [mysql_id]
|
|
region = Region.objects.create(**region_data)
|
|
|
|
if "subregion" in city_object:
|
|
subregion_data = city_object.pop('subregion')
|
|
subregion_data['country'] = country
|
|
try:
|
|
subregion = Region.objects.get(code=subregion_data['code'], mysql_ids__isnull=False)
|
|
subregion.mysql_ids.append(mysql_id)
|
|
subregion.save()
|
|
except Region.DoesNotExist:
|
|
subregion_data['parent_region'] = region
|
|
subregion_data['mysql_ids'] = [mysql_id]
|
|
subregion = Region.objects.create(**subregion_data)
|
|
|
|
city_object['region'] = subregion
|
|
else:
|
|
city_object['region'] = region
|
|
|
|
if "subregion" in city_object:
|
|
del(city_object['subregion'])
|
|
|
|
try:
|
|
City.objects.create(**city_object)
|
|
except Exception as e:
|
|
print(city_object)
|
|
print(e)
|
|
break
|
|
|
|
|
|
def fix_location_address():
|
|
addresses = Address.objects.filter(old_id__isnull=False)
|
|
for address in addresses:
|
|
mysql_location = transfer_models.Locations.objects.get(id=address.old_id)
|
|
try:
|
|
correct_city = City.objects.get(mysql_id=mysql_location.city_id)
|
|
except City.DoesNotExist:
|
|
continue
|
|
|
|
address.city = correct_city
|
|
address.save()
|
|
|
|
|
|
def fix_location_collection():
|
|
collections = Collection.objects.filter(old_id__isnull=False)
|
|
for collection in collections:
|
|
try:
|
|
mysql_collection = transfer_models.Collections.objects.\
|
|
raw(f"""select
|
|
s.country_code_2,
|
|
c.id
|
|
from collections as c
|
|
join sites s on s.id = c.site_id where c.id={collection.old_id}""")[0]
|
|
except:
|
|
continue
|
|
|
|
try:
|
|
correct_country = Country.objects.get(code=mysql_collection.country_code_2, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
collection.country = correct_country
|
|
collection.save()
|
|
|
|
|
|
def fix_location_models():
|
|
|
|
# try:
|
|
# ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
|
|
# ruby_data = json.loads(ruby_data_file.read())
|
|
# except FileNotFoundError:
|
|
# ruby_data = get_ruby_data()
|
|
#
|
|
# add_correct_location_models(ruby_data)
|
|
#
|
|
# fix_location_address()
|
|
fix_location_collection()
|
|
|
|
|
|
data_types = {
|
|
"dictionaries": [
|
|
transfer_countries,
|
|
transfer_regions,
|
|
transfer_cities,
|
|
transfer_addresses,
|
|
transfer_wine_region,
|
|
transfer_wine_sub_region,
|
|
transfer_wine_village,
|
|
],
|
|
"update_country_flag": [
|
|
update_flags
|
|
],
|
|
"update_city_info": [
|
|
migrate_city_map_situation
|
|
],
|
|
"migrate_city_gallery": [
|
|
migrate_city_photos
|
|
],
|
|
"tmp": [
|
|
fix_location_models
|
|
]
|
|
|
|
}
|