812 lines
30 KiB
Python
812 lines
30 KiB
Python
import csv
|
|
import json
|
|
from pprint import pprint
|
|
|
|
from django.conf import settings
|
|
from django.db.transaction import atomic
|
|
from requests import get
|
|
from tqdm import tqdm
|
|
|
|
from account.models import Role
|
|
from collection.models import Collection
|
|
from gallery.models import Image
|
|
from location.models import Country, Region, City, Address
|
|
from main.models import AwardType
|
|
from news.models import News
|
|
from review.models import Review
|
|
from tag.models import TagCategory, ChosenTagSettings
|
|
from transfer import models as transfer_models
|
|
from transfer.serializers import location as location_serializers
|
|
from transfer.utils import clean_old_records, clean_old_country_records, clean_old_region_records
|
|
|
|
|
|
def transfer_countries():
|
|
queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT cities.id, cities.country_code_2
|
|
FROM cities
|
|
WHERE country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
GROUP BY cities.id, cities.country_code_2
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CountrySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Country serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_regions():
|
|
regions_without_subregion_queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
FROM cities
|
|
WHERE (subregion_code IS NULL
|
|
OR subregion_code = ""
|
|
)
|
|
AND region_code IS NOT NULL
|
|
AND region_code != ""
|
|
AND country_code_2 IS NOT NULL
|
|
AND country_code_2 != ""
|
|
GROUP BY cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
""")
|
|
|
|
regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]
|
|
|
|
serialized_without_subregion = location_serializers.RegionSerializer(data=regions_without_subregion_queryset,
|
|
many=True)
|
|
if serialized_without_subregion.is_valid():
|
|
serialized_without_subregion.save()
|
|
else:
|
|
pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")
|
|
|
|
regions_with_subregion_queryset = transfer_models.Cities.objects.raw("""
|
|
SELECT
|
|
cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
FROM cities
|
|
WHERE subregion_code IS NOT NULL AND
|
|
subregion_code != "" AND
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
AND cities.subregion_code in
|
|
(
|
|
SELECT region_code FROM cities WHERE
|
|
(subregion_code IS NULL OR
|
|
subregion_code = "") AND
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
)
|
|
GROUP BY cities.id,
|
|
cities.region_code,
|
|
cities.country_code_2,
|
|
cities.subregion_code
|
|
""")
|
|
|
|
regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]
|
|
|
|
serialized_with_subregion = location_serializers.RegionSerializer(data=regions_with_subregion_queryset,
|
|
many=True)
|
|
if serialized_with_subregion.is_valid():
|
|
serialized_with_subregion.save()
|
|
else:
|
|
pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")
|
|
|
|
|
|
def transfer_cities():
|
|
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
|
|
cities.is_island, cities.region_code, cities.subregion_code
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CitySerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"City serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
@atomic
|
|
def transfer_addresses():
|
|
queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude,
|
|
locations.latitude, locations.address, locations.city_id
|
|
FROM locations WHERE
|
|
locations.city_id IS NOT NULL""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.AddressSerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"Address serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_region():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='WineRegion')
|
|
serialized_data = location_serializers.WineRegionSerializer(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineRegionSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_sub_region():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion')
|
|
serialized_data = location_serializers.WineSubRegionSerializer(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def transfer_wine_village():
|
|
queryset = transfer_models.WineLocations.objects.filter(type='Village')
|
|
serialized_data = location_serializers.WineVillageSerializer(
|
|
data=list(queryset.values()),
|
|
many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")
|
|
|
|
|
|
def update_flags():
|
|
queryset = Country.objects.only("id", "code", "svg_image").filter(old_id__isnull=False)
|
|
link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media"
|
|
|
|
for query in queryset:
|
|
svg_link = f"/svg/country/10-31-2019/{query.code}.svg"
|
|
resp = get(f"{link_to_request}{svg_link}")
|
|
if resp.status_code == 200:
|
|
query.svg_image = svg_link
|
|
query.save()
|
|
|
|
|
|
def update_fake_country_flag():
|
|
link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media/svg/country/10-31-2019/aa.svg"
|
|
resp = get(link_to_request)
|
|
if resp.status_code == 200:
|
|
country = Country.objects.get(code="aa")
|
|
country.svg_image = "/svg/country/10-31-2019/aa.svg"
|
|
country.save()
|
|
|
|
|
|
def migrate_city_map_situation(get_exists_cities=False):
|
|
if get_exists_cities:
|
|
ids = City.objects.values_list('mysql_id', flat=True)
|
|
queryset = transfer_models.Cities.objects.filter(id__in=list(ids))
|
|
queryset = list(queryset.values())
|
|
else:
|
|
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.map1, cities.map2, cities.map_ref, cities.situation
|
|
FROM cities WHERE
|
|
region_code IS NOT NULL AND
|
|
region_code != "" AND
|
|
country_code_2 IS NOT NULL AND
|
|
country_code_2 != ""
|
|
""")
|
|
|
|
queryset = [vars(query) for query in queryset]
|
|
|
|
serialized_data = location_serializers.CityMapCorrectSerializer(data=queryset, many=True)
|
|
if serialized_data.is_valid():
|
|
serialized_data.save()
|
|
else:
|
|
pprint(f"City info serializer errors: {serialized_data.errors}")
|
|
|
|
|
|
# Update location models with ruby library
|
|
# Utils functions defined before transfer functions
|
|
def get_ruby_socket(params):
|
|
url = 'http://172.21.0.1:5678' # docker host
|
|
response = get(url, params=params)
|
|
try:
|
|
data = json.loads(response.text)
|
|
except Exception as e:
|
|
print(f"{response.text} error: {e}")
|
|
return None
|
|
return data
|
|
|
|
|
|
# Get data from ruby and save it to file
|
|
# Save errors from ruby to another file
|
|
def get_ruby_data():
|
|
cities = transfer_models.Cities.objects.all()
|
|
ruby_data = {}
|
|
error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w")
|
|
|
|
for mysql_city in cities:
|
|
# try:
|
|
# mysql_city = transfer_models.Cities.objects.get(id=city.old_id)
|
|
# except transfer_models.Cities.DoesNotExist:
|
|
# print(f"City with id {city.old_id} not found")
|
|
# continue
|
|
|
|
ruby_params = {}
|
|
|
|
if mysql_city.country_code:
|
|
ruby_params['country_code'] = mysql_city.country_code
|
|
|
|
if mysql_city.country_code_2:
|
|
ruby_params['country_code_2'] = mysql_city.country_code_2
|
|
|
|
if mysql_city.region_code:
|
|
ruby_params['region_code'] = mysql_city.region_code
|
|
|
|
if mysql_city.subregion_code:
|
|
ruby_params['subregion_code'] = mysql_city.subregion_code
|
|
|
|
ruby_response = get_ruby_socket(ruby_params) if ruby_params else None
|
|
|
|
if ruby_response is None:
|
|
continue
|
|
|
|
if "country" not in ruby_response.keys():
|
|
error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
|
|
continue
|
|
|
|
if mysql_city.country_code_2 is None and mysql_city.country_code is None:
|
|
error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
|
|
continue
|
|
|
|
city_name = mysql_city.name.strip()
|
|
ruby_data[mysql_city.id] = {
|
|
"mysql_id": mysql_city.id,
|
|
"name": {'en-GB': city_name.strip()},
|
|
|
|
"map1": mysql_city.map1,
|
|
"map2": mysql_city.map2,
|
|
"map_ref": mysql_city.map_ref,
|
|
"situation": mysql_city.situation,
|
|
|
|
"is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False,
|
|
|
|
"postal_code": mysql_city.zip_code.strip() if mysql_city.zip_code is not None else '',
|
|
"code": ruby_response['country']['code'].lower().strip()
|
|
|
|
}
|
|
|
|
for ruby_data_key in ruby_response.keys():
|
|
if "error" in ruby_response[ruby_data_key]:
|
|
error_file.write(json.dumps(ruby_response) + '\n')
|
|
else:
|
|
ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key]
|
|
|
|
error_file.close()
|
|
|
|
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file:
|
|
ruby_data_file.write(json.dumps(ruby_data))
|
|
|
|
return ruby_data
|
|
|
|
|
|
def get_unused_data():
|
|
ruby_data = {}
|
|
error_file = open(f"{settings.PROJECT_ROOT}/ruby_unused_error.txt", "w")
|
|
|
|
countries = Country.objects.all()
|
|
|
|
for country in countries:
|
|
|
|
ruby_params = {}
|
|
|
|
ruby_response = get_ruby_socket({"country_regions": country.code})
|
|
|
|
if ruby_response is None:
|
|
continue
|
|
|
|
if "error" in ruby_response.keys():
|
|
error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
|
|
continue
|
|
|
|
ruby_data[country.code] = ruby_response
|
|
error_file.close()
|
|
|
|
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_unused_data.py", "w") as ruby_data_file:
|
|
ruby_data_file.write(json.dumps(ruby_data))
|
|
|
|
|
|
# Add correct objects of Country, Region and City with mysql_ids array (Country, Region) and mysql_id (City)
|
|
def add_correct_location_models(ruby_data):
|
|
for mysql_id, city_object in tqdm(ruby_data.items()):
|
|
country_data = city_object['country']
|
|
country_code = country_data['code'] # i.e. fr
|
|
try:
|
|
country = Country.objects.get(
|
|
code=country_code,
|
|
mysql_ids__isnull=False,
|
|
)
|
|
country.mysql_ids.append(mysql_id)
|
|
country.save()
|
|
except Country.DoesNotExist:
|
|
country_data['mysql_ids'] = [mysql_id]
|
|
country_data['calling_code'] = settings.COUNTRY_CALLING_CODES.get(country_code)
|
|
country = Country.objects.create(**country_data)
|
|
|
|
city_object['country'] = country
|
|
|
|
if "region" in city_object:
|
|
region_data = city_object['region']
|
|
region_data['country'] = country
|
|
try:
|
|
region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False)
|
|
region.mysql_ids.append(mysql_id)
|
|
region.save()
|
|
except Region.DoesNotExist:
|
|
region_data['mysql_ids'] = [mysql_id]
|
|
region = Region.objects.create(**region_data)
|
|
|
|
if "subregion" in city_object:
|
|
subregion_data = city_object.pop('subregion')
|
|
subregion_data['country'] = country
|
|
try:
|
|
subregion = Region.objects.get(code=subregion_data['code'],
|
|
mysql_ids__isnull=False)
|
|
subregion.mysql_ids.append(mysql_id)
|
|
subregion.save()
|
|
except Region.DoesNotExist:
|
|
subregion_data['parent_region'] = region
|
|
subregion_data['mysql_ids'] = [mysql_id]
|
|
subregion = Region.objects.create(**subregion_data)
|
|
|
|
city_object['region'] = subregion
|
|
else:
|
|
city_object['region'] = region
|
|
|
|
if "subregion" in city_object:
|
|
del (city_object['subregion'])
|
|
|
|
try:
|
|
City.objects.create(**city_object)
|
|
except Exception as e:
|
|
print(city_object)
|
|
print(e)
|
|
break
|
|
|
|
|
|
def fix_location_address():
|
|
addresses = Address.objects.filter(old_id__isnull=False)
|
|
for address in addresses:
|
|
mysql_location = transfer_models.Locations.objects.get(id=address.old_id)
|
|
try:
|
|
correct_city = City.objects.get(mysql_id=mysql_location.city_id)
|
|
except City.DoesNotExist:
|
|
continue
|
|
|
|
address.city = correct_city
|
|
address.save()
|
|
|
|
|
|
def fix_location_collection():
|
|
collections = Collection.objects.filter(old_id__isnull=False)
|
|
for collection in collections:
|
|
try:
|
|
mysql_collection = transfer_models.Collections.objects. \
|
|
raw(f"""select
|
|
s.country_code_2,
|
|
c.id
|
|
from collections as c
|
|
join sites s on s.id = c.site_id where c.id={collection.old_id}""")[0]
|
|
except:
|
|
continue
|
|
|
|
try:
|
|
correct_country = Country.objects.get(code=mysql_collection.country_code_2, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
collection.country = correct_country
|
|
collection.save()
|
|
|
|
|
|
def fix_award_type():
|
|
award_types = AwardType.objects.filter(old_id__isnull=False)
|
|
for award_type in award_types:
|
|
mysql_award_type = transfer_models.AwardTypes.objects. \
|
|
raw(f"""SELECT at.id, s.country_code_2 AS country_code
|
|
FROM award_types as at
|
|
JOIN sites s on s.id = at.site_id
|
|
WHERE at.id={award_type.old_id}""")[0]
|
|
|
|
try:
|
|
correct_country = Country.objects.get(code=mysql_award_type.country_code, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
award_type.country = correct_country
|
|
award_type.save()
|
|
|
|
|
|
def fix_role():
|
|
roles_list = Role.objects.filter(country__isnull=False)
|
|
for role in roles_list:
|
|
try:
|
|
correct_country = Country.objects.get(code=role.country.code, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
role.country = correct_country
|
|
role.save()
|
|
|
|
|
|
def fix_news():
|
|
news = News.objects.filter(country__isnull=False)
|
|
for news_item in news:
|
|
try:
|
|
correct_country = Country.objects.get(code=news_item.country.code, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
news.country = correct_country
|
|
news_item.save()
|
|
|
|
|
|
def fix_reviews():
|
|
reviews = Review.objects.filter(country__isnull=False)
|
|
for review in reviews:
|
|
try:
|
|
correct_country = Country.objects.get(code=review.country.code, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
review.country = correct_country
|
|
review.save()
|
|
|
|
|
|
def fix_tag_category():
|
|
tags_categories = TagCategory.objects.filter(country__isnull=False)
|
|
for tag_category in tags_categories:
|
|
try:
|
|
correct_country = Country.objects.get(code=tag_category.country.code, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
tag_category.country = correct_country
|
|
tag_category.save()
|
|
|
|
|
|
def fix_chosen_tag():
|
|
chosen_tags = ChosenTagSettings.objects.filter(country__isnull=False)
|
|
for chosen_tag in chosen_tags:
|
|
try:
|
|
correct_country = Country.objects.get(code=chosen_tag.country.code, mysql_ids__isnull=False)
|
|
except Country.DoesNotExist:
|
|
continue
|
|
|
|
chosen_tag.country = correct_country
|
|
chosen_tag.save()
|
|
|
|
|
|
def fix_location_models():
|
|
try:
|
|
ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
|
|
ruby_data = json.loads(ruby_data_file.read())
|
|
except FileNotFoundError:
|
|
print('create ruby data')
|
|
ruby_data = get_ruby_data()
|
|
|
|
print('add_correct_location_models')
|
|
add_correct_location_models(ruby_data)
|
|
# print('fix_location_address')
|
|
# fix_location_address()
|
|
# print('fix_location_collection')
|
|
# fix_location_collection()
|
|
# print('fix_award_type')
|
|
# fix_award_type()
|
|
# print('fix_role')
|
|
# fix_role()
|
|
# print('fix_news')
|
|
# fix_news()
|
|
# print('fix_reviews')
|
|
# fix_reviews()
|
|
# print('fix_tag_category')
|
|
# fix_tag_category()
|
|
# print('fix_chosen_tag')
|
|
# fix_chosen_tag()
|
|
|
|
|
|
def remove_old_records():
|
|
clean_old_records(City, {"mysql_id__isnull": True})
|
|
clean_old_country_records(Country, {"mysql_ids__isnull": True})
|
|
clean_old_region_records(Region, {"mysql_ids__isnull": True})
|
|
|
|
|
|
def transfer_city_photos():
|
|
created_counter = 0
|
|
cities_not_exists = {}
|
|
cities_has_same_image = 0
|
|
|
|
city_gallery = transfer_models.CityPhotos.objects.exclude(city__isnull=True) \
|
|
.exclude(city__country_code_2__isnull=True) \
|
|
.exclude(city__country_code_2__iexact='') \
|
|
.exclude(city__region_code__isnull=True) \
|
|
.exclude(city__region_code__iexact='') \
|
|
.values_list('city_id', 'attachment_suffix_url')
|
|
for old_city_id, image_suffix_url in tqdm(city_gallery):
|
|
city = City.objects.filter(old_id=old_city_id)
|
|
if city.exists():
|
|
city = city.first()
|
|
image, _ = Image.objects.get_or_create(image=image_suffix_url,
|
|
defaults={
|
|
'image': image_suffix_url,
|
|
'orientation': Image.HORIZONTAL,
|
|
'title': f'{city.name} - {image_suffix_url}',
|
|
})
|
|
if city.image != image:
|
|
city.image = image
|
|
city.save()
|
|
created_counter += 1
|
|
else:
|
|
cities_has_same_image += 1
|
|
else:
|
|
cities_not_exists.update({'city_old_id': old_city_id})
|
|
|
|
print(f'Created: {created_counter}\n'
|
|
f'City not exists: {cities_not_exists}\n'
|
|
f'City has same image: {cities_has_same_image}')
|
|
|
|
|
|
@atomic
|
|
def add_fake_country():
|
|
# add country
|
|
country_data = {
|
|
"name": '{"en-GB": "Antilles Guyane West Indies"}',
|
|
"code": "aa",
|
|
"calling_code": "590",
|
|
"svg_image": "svg/country/11-02-2019/658714.svg",
|
|
"mysql_ids": []
|
|
}
|
|
|
|
country, _ = Country.objects.get_or_create(**country_data)
|
|
|
|
# add regions and subregions
|
|
regions_data = [
|
|
{
|
|
"name": "Guadeloupe",
|
|
"code": "gp",
|
|
"subregions": [
|
|
{
|
|
"name": "Basse Terre",
|
|
"code": "bat",
|
|
},
|
|
{
|
|
"name": "Grande Terre",
|
|
"code": "grt",
|
|
},
|
|
{
|
|
"name": "Autres les Îles de Guadeloupe",
|
|
"code": "aug",
|
|
},
|
|
]
|
|
},
|
|
{
|
|
"name": "Martinique",
|
|
"code": "mq",
|
|
"subregions": [
|
|
{
|
|
"name": "Nord",
|
|
"code": "nor",
|
|
},
|
|
{
|
|
"name": "Sud",
|
|
"code": "sud",
|
|
},
|
|
]
|
|
},
|
|
{
|
|
"name": "Guyane",
|
|
"code": "gy",
|
|
},
|
|
{
|
|
"name": "St Barthelemy",
|
|
"code": "bl",
|
|
},
|
|
{
|
|
"name": "St Martin",
|
|
"code": "mf",
|
|
},
|
|
{
|
|
"name": "Sainte-Lucie",
|
|
"code": "lc",
|
|
},
|
|
]
|
|
regions = {}
|
|
|
|
for region_data in regions_data:
|
|
if "subregions" in region_data:
|
|
subregions = region_data['subregions']
|
|
del (region_data['subregions'])
|
|
else:
|
|
subregions = False
|
|
|
|
region_name = region_data['name']
|
|
region_data['name'] = '{"en-GB": "' + region_name + '"}'
|
|
region_data['country'] = country
|
|
|
|
region_data['mysql_ids'] = []
|
|
|
|
regions[region_name] = Region.objects.create(**region_data)
|
|
|
|
if subregions:
|
|
for subregion_data in subregions:
|
|
subregion_name = subregion_data['name']
|
|
subregion_data['name'] = '{"en-GB": "' + subregion_name + '"}'
|
|
subregion_data['country'] = country
|
|
subregion_data['parent_region'] = regions[region_name]
|
|
subregion_data['mysql_ids'] = []
|
|
|
|
regions[subregion_name] = Region.objects.create(**subregion_data)
|
|
|
|
# add cities
|
|
file = open(f"{settings.PROJECT_ROOT}/apps/location/csv/aa_cities.csv")
|
|
reader = csv.DictReader(file, delimiter=',')
|
|
for city_data in reader:
|
|
del (city_data[''])
|
|
|
|
city_data['mysql_id'] = city_data['old_id']
|
|
del (city_data['old_id'])
|
|
|
|
city_data['postal_code'] = city_data['zip_code']
|
|
del (city_data['zip_code'])
|
|
|
|
if city_data['postal_code'] == 'null' or city_data['postal_code'] == '':
|
|
del (city_data['postal_code'])
|
|
|
|
city_data["country"] = country
|
|
|
|
if city_data['subregion'] != 'null':
|
|
region = regions[city_data['subregion']]
|
|
elif city_data['region'] != 'null':
|
|
region = regions[city_data['region']]
|
|
else:
|
|
del (city_data['region'])
|
|
region = None
|
|
|
|
del (city_data['subregion'])
|
|
|
|
city_data['region'] = region
|
|
|
|
city_data['name'] = '{"en-GB": "' + city_data['name'] + '"}'
|
|
|
|
city_data['is_island'] = True if int(city_data['is_island']) > 0 else False
|
|
|
|
if region is not None:
|
|
region.mysql_ids.append(city_data['mysql_id'])
|
|
|
|
country.mysql_ids.append(city_data['mysql_id'])
|
|
|
|
try:
|
|
mysql_data = transfer_models.Cities.objects. \
|
|
only("map1", "map2", "map_ref", "situation").get(id=city_data['mysql_id'])
|
|
city_data['map1'] = mysql_data.map1
|
|
city_data['map2'] = mysql_data.map2
|
|
city_data['map_ref'] = mysql_data.map_ref
|
|
city_data['situation'] = mysql_data.situation
|
|
|
|
except transfer_models.Cities.DoesNotExist:
|
|
pass
|
|
|
|
City.objects.create(**city_data)
|
|
|
|
country.save()
|
|
|
|
for region_name, region in regions.items():
|
|
if len(region.mysql_ids) > 0:
|
|
region.save()
|
|
|
|
|
|
@atomic
|
|
def setup_clean_db():
|
|
try:
|
|
ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
|
|
ruby_data = json.loads(ruby_data_file.read())
|
|
except FileNotFoundError:
|
|
ruby_data = get_ruby_data()
|
|
|
|
print('add_correct_location_models')
|
|
add_correct_location_models(ruby_data)
|
|
|
|
print('add_fake_country')
|
|
add_fake_country()
|
|
|
|
print('migrate_city_map_situation')
|
|
migrate_city_map_situation(True)
|
|
|
|
print('update_flags')
|
|
update_flags()
|
|
|
|
print('transfer_city_photos')
|
|
transfer_city_photos()
|
|
|
|
|
|
def set_unused_regions():
|
|
ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_unused_data.py", "r")
|
|
ruby_data = json.loads(ruby_data_file.read())
|
|
|
|
for country_code, regions in ruby_data.items():
|
|
try:
|
|
country = Country.objects.get(code=country_code)
|
|
except Country.DoesNotExist:
|
|
print(f"Country with code {country_code} does not exists")
|
|
continue
|
|
|
|
for region_code, region_obj in regions.items():
|
|
try:
|
|
region = Region.objects.get(code=region_code, country=country)
|
|
except Region.DoesNotExist:
|
|
region = Region.objects.create(
|
|
name=region_obj['name'],
|
|
code=region_code,
|
|
country=country
|
|
)
|
|
|
|
if "subregions" in region_obj:
|
|
for subregion_code, subregion in region_obj['subregions'].items():
|
|
try:
|
|
subregion = Region.objects.get(code=subregion, country=country)
|
|
except Region.DoesNotExist:
|
|
subregion = Region.objects.create(
|
|
name=subregion,
|
|
code=subregion_code,
|
|
country=country,
|
|
parent_region=region
|
|
)
|
|
|
|
|
|
data_types = {
|
|
"dictionaries": [
|
|
# transfer_countries, # !!! см setup_clean_db
|
|
# transfer_regions, # !!! см setup_clean_db
|
|
# transfer_cities, # !!! см setup_clean_db
|
|
transfer_addresses,
|
|
transfer_wine_region,
|
|
transfer_wine_sub_region,
|
|
transfer_wine_village,
|
|
],
|
|
"update_country_flag": [
|
|
update_flags
|
|
],
|
|
"update_city_info": [
|
|
migrate_city_map_situation
|
|
],
|
|
"fix_location": [
|
|
add_fake_country,
|
|
fix_location_models,
|
|
],
|
|
"remove_old_locations": [
|
|
remove_old_records
|
|
],
|
|
"migrate_city_photos": [
|
|
transfer_city_photos,
|
|
],
|
|
"add_fake_country": [
|
|
add_fake_country,
|
|
],
|
|
"setup_clean_db": [setup_clean_db],
|
|
"set_unused_regions": [set_unused_regions],
|
|
"update_fake_country_flag": [update_fake_country_flag]
|
|
}
|