Add ruby data getter
Add ruby files write Add models migrations REMOVE UNIQUE FROM location.country.code Add function to save correct original objects from old database
This commit is contained in:
littlewolf
parent
6bac3fc9c2
commit
180daeca0c
19
apps/location/migrations/0031_auto_20191117_1750.py
Normal file
19
apps/location/migrations/0031_auto_20191117_1750.py
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
# Generated by Django 2.2.7 on 2019-11-17 17:50
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('location', '0030_auto_20191116_1151'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='city',
|
||||
name='region',
|
||||
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='location.Region', verbose_name='parent region'),
|
||||
),
|
||||
]
|
||||
22
apps/location/migrations/0032_auto_20191117_2300.py
Normal file
22
apps/location/migrations/0032_auto_20191117_2300.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# Generated by Django 2.2.7 on 2019-11-17 23:00
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('location', '0031_auto_20191117_1750'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.RemoveField(
|
||||
model_name='city',
|
||||
name='mysql_ids',
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='city',
|
||||
name='mysql_id',
|
||||
field=models.IntegerField(blank=True, default=None, null=True),
|
||||
),
|
||||
]
|
||||
18
apps/location/migrations/0033_auto_20191118_0326.py
Normal file
18
apps/location/migrations/0033_auto_20191118_0326.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Generated by Django 2.2.7 on 2019-11-18 03:26
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('location', '0032_auto_20191117_2300'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='country',
|
||||
name='code',
|
||||
field=models.CharField(max_length=255, verbose_name='Code'),
|
||||
),
|
||||
]
|
||||
|
|
@ -34,7 +34,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
|
|||
|
||||
name = TJSONField(null=True, blank=True, default=None,
|
||||
verbose_name=_('Name'), help_text='{"en-GB":"some text"}')
|
||||
code = models.CharField(max_length=255, unique=True, verbose_name=_('Code'))
|
||||
code = models.CharField(max_length=255, unique=False, verbose_name=_('Code'))
|
||||
low_price = models.IntegerField(default=25, verbose_name=_('Low price'))
|
||||
high_price = models.IntegerField(default=50, verbose_name=_('High price'))
|
||||
languages = models.ManyToManyField(Language, verbose_name=_('Languages'))
|
||||
|
|
@ -109,7 +109,7 @@ class City(models.Model):
|
|||
verbose_name=_('Translated name'), help_text='{"en-GB":"some text"}')
|
||||
code = models.CharField(_('code'), max_length=250)
|
||||
region = models.ForeignKey(
|
||||
Region, verbose_name=_('parent region'), on_delete=models.CASCADE)
|
||||
Region, verbose_name=_('parent region'), on_delete=models.CASCADE, blank=True, null=True)
|
||||
country = models.ForeignKey(
|
||||
Country, verbose_name=_('country'), on_delete=models.CASCADE)
|
||||
|
||||
|
|
@ -126,7 +126,7 @@ class City(models.Model):
|
|||
|
||||
gallery = models.ManyToManyField('gallery.Image', through='location.CityGallery', blank=True)
|
||||
|
||||
mysql_ids = ArrayField(models.IntegerField(), blank=True, null=True)
|
||||
mysql_id = models.IntegerField(blank=True, null=True, default=None)
|
||||
|
||||
objects = CityQuerySet.as_manager()
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
{"5174": {"code": "15", "name": "Korinthia", "region": "J", "country": "GR"}, "5175": {"code": "17", "name": "Messinia", "region": "J", "country": "GR"}, "5176": {"code": "16", "name": "Lakonia", "region": "J", "country": "GR"}, "5177": {"code": "11", "name": "Argolida", "region": "J", "country": "GR"}, "5178": {"code": "11", "name": "Argolida", "region": "J", "country": "GR"}, "5179": {"code": "03", "name": "Voiotia", "region": "H", "country": "GR"}, "5180": {"code": "04", "name": "Evvoias", "region": "H", "country": "GR"}}
|
||||
|
|
@ -1 +0,0 @@
|
|||
[{"old_id": [9568], "data": {"code": "GP", "name": "Guadeloupe"}}, {"old_id": [1], "data": {"code": "AT", "name": "Austria"}}, {"old_id": [7307], "data": {"code": "AU", "name": "Australia"}}, {"old_id": [2298], "data": {"code": "BE", "name": "Belgium"}}, {"old_id": [11277], "data": {"code": "BR", "name": "Brazil"}}, {"old_id": [6226], "data": {"code": "CA", "name": "Canada"}}, {"old_id": [2955], "data": {"code": "CH", "name": "Switzerland"}}, {"old_id": [202], "data": {"code": "DE", "name": "Germany"}}, {"old_id": [3133], "data": {"code": "ES", "name": "Spain"}}, {"old_id": [514], "data": {"code": "FR", "name": "France"}}, {"old_id": [3758], "data": {"code": "GB", "name": "United Kingdom"}}, {"old_id": [8539], "data": {"code": "GE", "name": "Georgia"}}, {"old_id": [10164], "data": {"code": "GR", "name": "Greece"}}, {"old_id": [2283], "data": {"code": "HR", "name": "Croatia"}}, {"old_id": [8406], "data": {"code": "HU", "name": "Hungary"}}, {"old_id": [8527], "data": {"code": "IL", "name": "Israel"}}, {"old_id": [2274], "data": {"code": "IT", "name": "Italy"}}, {"old_id": [7370], "data": {"code": "JP", "name": "Japan"}}, {"old_id": [141], "data": {"code": "LU", "name": "Luxembourg"}}, {"old_id": [7709], "data": {"code": "MA", "name": "Morocco"}}, {"old_id": [9852], "data": {"code": "MV", "name": "Maldives"}}, {"old_id": [9030], "data": {"code": "NL", "name": "Netherlands"}}, {"old_id": [10390], "data": {"code": "PL", "name": "Poland"}}, {"old_id": [8915], "data": {"code": "RO", "name": "Romania"}}, {"old_id": [8238], "data": {"code": "RU", "name": "Russia"}}, {"old_id": [7230], "data": {"code": "SI", "name": "Slovenia"}}, {"old_id": [7846], "data": {"code": "US", "name": "United States"}}]
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
from transfer.serializers import location as location_serializers
|
||||
from transfer import models as transfer_models
|
||||
from location.models import Country, Region
|
||||
from location.models import Country, Region, City
|
||||
from pprint import pprint
|
||||
import json
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import MultipleObjectsReturned
|
||||
|
||||
|
|
@ -227,132 +228,139 @@ Update location models with ruby library
|
|||
Utils functions defined before transfer functions
|
||||
"""
|
||||
|
||||
def get_ruby_socket(type, params):
|
||||
def get_ruby_socket(params):
|
||||
url = 'http://172.21.0.1:5678' # docker host
|
||||
params["type"] = type
|
||||
response = get(url, params=params)
|
||||
|
||||
try:
|
||||
data = json.loads(response.text)
|
||||
except Exception as e:
|
||||
print(f"{response.text} error: {e}")
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def update_child_regions():
|
||||
child_regions_list = Region.objects.filter(old_id__isnull=False, parent_region__isnull=False)
|
||||
ruby_data_to_write = {}
|
||||
def get_ruby_data():
|
||||
cities = City.objects.filter(old_id__isnull=False)
|
||||
ruby_data = {}
|
||||
error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w")
|
||||
|
||||
for city in cities:
|
||||
try:
|
||||
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file:
|
||||
ruby_regions_list = json.loads(file.read())
|
||||
except:
|
||||
ruby_regions_list = {}
|
||||
|
||||
for child_region in child_regions_list:
|
||||
mysql_region = transfer_models.Cities.objects.get(id=child_region.old_id)
|
||||
|
||||
|
||||
if child_region.id in ruby_regions_list:
|
||||
ruby_data = ruby_regions_list[child_region.id]
|
||||
else:
|
||||
ruby_params = {
|
||||
"country_code": mysql_region.country_code,
|
||||
"country_code_2": mysql_region.country_code_2,
|
||||
"region_code": mysql_region.region_code,
|
||||
"region_name": mysql_region.name,
|
||||
"subregion_code": mysql_region.subregion_code
|
||||
}
|
||||
|
||||
ruby_data = get_ruby_socket("subregion", ruby_params)
|
||||
if "error" in ruby_data:
|
||||
mysql_city = transfer_models.Cities.objects.get(id=city.old_id)
|
||||
except transfer_models.Cities.DoesNotExist:
|
||||
print(f"City with id {city.old_id} not found")
|
||||
continue
|
||||
|
||||
ruby_data_to_write[child_region.id] = ruby_data
|
||||
ruby_params = {}
|
||||
|
||||
if mysql_city.country_code is not None:
|
||||
ruby_params['country_code'] = mysql_city.country_code
|
||||
|
||||
if mysql_city.country_code_2 is not None:
|
||||
ruby_params['country_code_2'] = mysql_city.country_code_2
|
||||
|
||||
if mysql_city.region_code is not None:
|
||||
ruby_params['region_code'] = mysql_city.region_code
|
||||
|
||||
if mysql_city.subregion_code is not None:
|
||||
ruby_params['subregion_code'] = mysql_city.subregion_code
|
||||
|
||||
ruby_response = get_ruby_socket(ruby_params)
|
||||
|
||||
if ruby_response is None:
|
||||
continue
|
||||
|
||||
if "error" in ruby_response or "country" not in ruby_response:
|
||||
error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
|
||||
continue
|
||||
|
||||
ruby_data[mysql_city.id] = {
|
||||
"mysql_id": mysql_city.id,
|
||||
"name": mysql_city.name,
|
||||
"name_translated": '{"en-GB":"' + mysql_city.name + '"}',
|
||||
|
||||
"map1": mysql_city.map1,
|
||||
"map2": mysql_city.map2,
|
||||
"map_ref": mysql_city.map_ref,
|
||||
"situation": mysql_city.situation,
|
||||
|
||||
"is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False,
|
||||
|
||||
"postal_code": mysql_city.zip_code if mysql_city.zip_code is not None else '',
|
||||
"code": mysql_city.country_code_2.lower()
|
||||
|
||||
}
|
||||
|
||||
for ruby_data_key in ruby_response.keys():
|
||||
if "error" in ruby_response[ruby_data_key]:
|
||||
error_file.write(json.dumps(ruby_response) + '\n')
|
||||
else:
|
||||
ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key]
|
||||
|
||||
error_file.close()
|
||||
|
||||
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file:
|
||||
ruby_data_file.write(json.dumps(ruby_data))
|
||||
|
||||
return ruby_data
|
||||
|
||||
|
||||
def fix_location_models():
|
||||
|
||||
try:
|
||||
parent_region = Region.objects.get(
|
||||
code=ruby_data['region'],
|
||||
country__code=ruby_data['country']
|
||||
)
|
||||
country = parent_region.country
|
||||
except MultipleObjectsReturned:
|
||||
parent_region = Region.objects.filter(
|
||||
code=ruby_data['region'],
|
||||
country__code=ruby_data['country']
|
||||
).first()
|
||||
country = parent_region.country
|
||||
ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
|
||||
ruby_data = json.loads(ruby_data_file.read())
|
||||
except FileNotFoundError:
|
||||
ruby_data = get_ruby_data()
|
||||
|
||||
for mysql_id, city_object in ruby_data.items():
|
||||
country_data = city_object["country"]
|
||||
try:
|
||||
country = Country.objects.get(code=country_data['code'], mysql_ids__isnull=False)
|
||||
country.mysql_ids.append(mysql_id)
|
||||
country.save()
|
||||
except Country.DoesNotExist:
|
||||
country_data['mysql_ids'] = [mysql_id]
|
||||
country = Country.objects.create(**country_data)
|
||||
|
||||
city_object['country'] = country
|
||||
|
||||
if "region" in city_object:
|
||||
region_data = city_object['region']
|
||||
region_data['country'] = country
|
||||
try:
|
||||
region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False)
|
||||
region.mysql_ids.append(mysql_id)
|
||||
region.save()
|
||||
except Region.DoesNotExist:
|
||||
region_data['mysql_ids'] = [mysql_id]
|
||||
region = Region.objects.create(**region_data)
|
||||
|
||||
if "subregion" in city_object:
|
||||
subregion_data = city_object.pop('subregion')
|
||||
subregion_data['country'] = country
|
||||
try:
|
||||
parent_region = Region.objects.get(code=ruby_data['region'])
|
||||
country = Country.objects.get(code=ruby_data['country'])
|
||||
except MultipleObjectsReturned:
|
||||
parent_region = Region.objects.filter(code=ruby_data['region']).first()
|
||||
country = parent_region.country
|
||||
|
||||
child_region.parent_region = parent_region
|
||||
child_region.country = country
|
||||
child_region.save()
|
||||
|
||||
if len(ruby_data_to_write) > 0:
|
||||
ruby_data_to_write = {**ruby_data_to_write, **ruby_regions_list}
|
||||
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file:
|
||||
file.write(json.dumps(ruby_data_to_write))
|
||||
|
||||
|
||||
# def update_relations(old_instance, new_instance, relations_dict):
|
||||
# for relation_name, relation_field in relations_dict.items():
|
||||
# relations_to_update = getattr(old_instance, relation_name, None)
|
||||
# print(relations_to_update)
|
||||
# if relations_to_update is None:
|
||||
# continue
|
||||
# # elif isinstance(relations_to_update)
|
||||
#
|
||||
# # relations_to_update = relations_to_update.all()
|
||||
#
|
||||
# # if len(relations_to_update) > 0:
|
||||
#
|
||||
#
|
||||
# def update_country_from_ruby():
|
||||
# countries = Country.objects.filter(old_id__isnull=False).distinct('code')
|
||||
#
|
||||
# json_countries_list = []
|
||||
# for country in countries:
|
||||
# json_country_dict = {
|
||||
# "old_id": [country.old_id]
|
||||
# }
|
||||
# mysql_country = transfer_models.Cities.objects.get(id=country.old_id)
|
||||
#
|
||||
# ruby_params = {
|
||||
# "country_code": mysql_country.country_code,
|
||||
# "country_code_2": mysql_country.country_code_2,
|
||||
# "region_code": mysql_country.region_code
|
||||
# }
|
||||
#
|
||||
# ruby_data = get_ruby_socket("country", ruby_params)
|
||||
#
|
||||
# if "error" in ruby_data:
|
||||
# print("ERROR: ")
|
||||
# print(f"{ruby_data['error']}: {ruby_params}")
|
||||
# continue
|
||||
#
|
||||
# json_country_dict["data"] = ruby_data
|
||||
#
|
||||
# duplicates = Country.objects.\
|
||||
# filter(
|
||||
# old_id__isnull=False,
|
||||
# code=country.code
|
||||
# ).exclude(old_id=country.old_id)
|
||||
#
|
||||
# for duplicate in duplicates:
|
||||
# json_country_dict["old_id"].append(duplicate.old_id)
|
||||
#
|
||||
# json_countries_list.append(json_country_dict)
|
||||
#
|
||||
# with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/countries.py", "w") as file:
|
||||
# file.write(json.dumps(json_countries_list))
|
||||
#
|
||||
#
|
||||
subregion = Region.objects.get(code=subregion_data['code'], mysql_ids__isnull=False)
|
||||
subregion.mysql_ids.append(mysql_id)
|
||||
subregion.save()
|
||||
except Region.DoesNotExist:
|
||||
subregion_data['parent_region'] = region
|
||||
subregion_data['mysql_ids'] = [mysql_id]
|
||||
subregion = Region.objects.create(**subregion_data)
|
||||
|
||||
city_object['region'] = subregion
|
||||
else:
|
||||
city_object['region'] = region
|
||||
|
||||
if "subregion" in city_object:
|
||||
del(city_object['subregion'])
|
||||
|
||||
try:
|
||||
City.objects.create(**city_object)
|
||||
except Exception as e:
|
||||
print(city_object)
|
||||
print(e)
|
||||
break
|
||||
|
||||
|
||||
data_types = {
|
||||
|
|
@ -375,7 +383,7 @@ data_types = {
|
|||
migrate_city_photos
|
||||
],
|
||||
"tmp": [
|
||||
update_child_regions
|
||||
fix_location_models
|
||||
]
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,8 +43,7 @@ INSTALLED_APPS.append('transfer.apps.TransferConfig')
|
|||
DATABASES.update({
|
||||
'legacy': {
|
||||
'ENGINE': 'django.db.backends.mysql',
|
||||
'HOST': '172.22.0.1',
|
||||
# 'HOST': 'mysql_db',
|
||||
'HOST': 'mysql_db',
|
||||
'PORT': 3306,
|
||||
'NAME': 'dev',
|
||||
'USER': 'dev',
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user