phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add child_region updater

Browse Source
This commit is contained in:
littlewolf 2019-11-14 15:21:53 +03:00
parent cf226974c8
commit c872cdd4bc
6 changed files with 151 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/location/ruby_data/child_regions.py Normal file
View File

@ -0,0 +1 @@
{"240": {"code": "15", "name": "Korinthia", "region": "J", "country": "GR"}, "5180": {"code": "17", "name": "Messinia", "region": "J", "country": "GR"}, "5181": {"code": "16", "name": "Lakonia", "region": "J", "country": "GR"}, "5182": {"code": "11", "name": "Argolida", "region": "J", "country": "GR"}, "5183": {"code": "11", "name": "Argolida", "region": "J", "country": "GR"}, "5184": {"code": "03", "name": "Voiotia", "region": "H", "country": "GR"}, "5185": {"code": "04", "name": "Evvoias", "region": "H", "country": "GR"}}

1
apps/location/ruby_data/countries.py Normal file
View File

@ -0,0 +1 @@
[{"old_id": [9568], "data": {"code": "GP", "name": "Guadeloupe"}}, {"old_id": [1], "data": {"code": "AT", "name": "Austria"}}, {"old_id": [7307], "data": {"code": "AU", "name": "Australia"}}, {"old_id": [2298], "data": {"code": "BE", "name": "Belgium"}}, {"old_id": [11277], "data": {"code": "BR", "name": "Brazil"}}, {"old_id": [6226], "data": {"code": "CA", "name": "Canada"}}, {"old_id": [2955], "data": {"code": "CH", "name": "Switzerland"}}, {"old_id": [202], "data": {"code": "DE", "name": "Germany"}}, {"old_id": [3133], "data": {"code": "ES", "name": "Spain"}}, {"old_id": [514], "data": {"code": "FR", "name": "France"}}, {"old_id": [3758], "data": {"code": "GB", "name": "United Kingdom"}}, {"old_id": [8539], "data": {"code": "GE", "name": "Georgia"}}, {"old_id": [10164], "data": {"code": "GR", "name": "Greece"}}, {"old_id": [2283], "data": {"code": "HR", "name": "Croatia"}}, {"old_id": [8406], "data": {"code": "HU", "name": "Hungary"}}, {"old_id": [8527], "data": {"code": "IL", "name": "Israel"}}, {"old_id": [2274], "data": {"code": "IT", "name": "Italy"}}, {"old_id": [7370], "data": {"code": "JP", "name": "Japan"}}, {"old_id": [141], "data": {"code": "LU", "name": "Luxembourg"}}, {"old_id": [7709], "data": {"code": "MA", "name": "Morocco"}}, {"old_id": [9852], "data": {"code": "MV", "name": "Maldives"}}, {"old_id": [9030], "data": {"code": "NL", "name": "Netherlands"}}, {"old_id": [10390], "data": {"code": "PL", "name": "Poland"}}, {"old_id": [8915], "data": {"code": "RO", "name": "Romania"}}, {"old_id": [8238], "data": {"code": "RU", "name": "Russia"}}, {"old_id": [7230], "data": {"code": "SI", "name": "Slovenia"}}, {"old_id": [7846], "data": {"code": "US", "name": "United States"}}]

154
apps/location/transfer_data.py
View File

@ -1,10 +1,12 @@
from transfer.serializers import location as location_serializers
from transfer import models as transfer_models
from location.models import Country
from location.models import Country, Region
from pprint import pprint
import json
from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned
from requests import get
import json
def transfer_countries():
@ -181,7 +183,7 @@ def update_flags():
def migrate_city_map_situation():
queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
cities.is_island, cities.region_code, cities.subregion_code
FROM cities WHERE
region_code IS NOT NULL AND
@ -190,7 +192,7 @@ def migrate_city_map_situation():
country_code_2 != ""
""")
serialized_data = CityMapSerializer(data=queryset, many=True)
serialized_data = location_serializers.CityMapSerializer(data=queryset, many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
@ -198,7 +200,7 @@ def migrate_city_map_situation():
def migrate_city_photos():
queryset = CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name
queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name
FROM city_photos WHERE
city_photos.attachment_file_name IS NOT NULL AND
city_id IN(
@ -213,28 +215,146 @@ def migrate_city_photos():
queryset = [vars(query) for query in queryset]
serialized_data = CityGallerySerializer(data=queryset, many=True)
serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"Address serializer errors: {serialized_data.errors}")
def update_subregions():
# regions = CitySerializer
pass
"""
Update location models with ruby library
Utils functions defined before transfer functions
"""
# TODO: удалить после переноса!
def get_ruby_socket(region_params):
url = 'http://172.22.0.1:5678'
response = get(url, params=region_params)
def get_ruby_socket(type, params):
url = 'http://172.22.0.1:5678' # docker host
params["type"] = type
response = get(url, params=params)
data = json.loads(response.text)
print(data)
return data
def update_child_regions():
child_regions_list = Region.objects.filter(old_id__isnull=False, parent_region__isnull=False)
ruby_data_to_write = {}
try:
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file:
ruby_regions_list = json.loads(file.read())
except:
ruby_regions_list = {}
for child_region in child_regions_list:
mysql_region = transfer_models.Cities.objects.get(id=child_region.old_id)
if child_region.id in ruby_regions_list:
ruby_data = ruby_regions_list[child_region.id]
else:
ruby_params = {
"country_code": mysql_region.country_code,
"country_code_2": mysql_region.country_code_2,
"region_code": mysql_region.region_code,
"region_name": mysql_region.name,
"subregion_code": mysql_region.subregion_code
}
ruby_data = get_ruby_socket("subregion", ruby_params)
if "error" in ruby_data:
continue
ruby_data_to_write[child_region.id] = ruby_data
try:
parent_region = Region.objects.get(
code=ruby_data['region'],
country__code=ruby_data['country']
)
country = parent_region.country
except MultipleObjectsReturned:
parent_region = Region.objects.filter(
code=ruby_data['region'],
country__code=ruby_data['country']
).first()
country = parent_region.country
except Region.DoesNotExist:
try:
parent_region = Region.objects.get(code=ruby_data['region'])
country = Country.objects.get(code=ruby_data['country'])
except MultipleObjectsReturned:
parent_region = Region.objects.filter(code=ruby_data['region']).first()
country = parent_region.country
child_region.parent_region = parent_region
child_region.country = country
child_region.save()
if len(ruby_data_to_write) > 0:
ruby_data_to_write = {**ruby_data_to_write, **ruby_regions_list}
with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file:
file.write(json.dumps(ruby_data_to_write))
# def update_relations(old_instance, new_instance, relations_dict):
# for relation_name, relation_field in relations_dict.items():
# relations_to_update = getattr(old_instance, relation_name, None)
# print(relations_to_update)
# if relations_to_update is None:
# continue
# # elif isinstance(relations_to_update)
#
# # relations_to_update = relations_to_update.all()
#
# # if len(relations_to_update) > 0:
#
#
# def update_country_from_ruby():
# countries = Country.objects.filter(old_id__isnull=False).distinct('code')
#
# json_countries_list = []
# for country in countries:
# json_country_dict = {
# "old_id": [country.old_id]
# }
# mysql_country = transfer_models.Cities.objects.get(id=country.old_id)
#
# ruby_params = {
# "country_code": mysql_country.country_code,
# "country_code_2": mysql_country.country_code_2,
# "region_code": mysql_country.region_code
# }
#
# ruby_data = get_ruby_socket("country", ruby_params)
#
# if "error" in ruby_data:
# print("ERROR: ")
# print(f"{ruby_data['error']}: {ruby_params}")
# continue
#
# json_country_dict["data"] = ruby_data
#
# duplicates = Country.objects.\
# filter(
# old_id__isnull=False,
# code=country.code
# ).exclude(old_id=country.old_id)
#
# for duplicate in duplicates:
# json_country_dict["old_id"].append(duplicate.old_id)
#
# json_countries_list.append(json_country_dict)
#
# with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/countries.py", "w") as file:
# file.write(json.dumps(json_countries_list))
#
#
data_types = {
"dictionaries": [
# transfer_countries,
@ -254,6 +374,8 @@ data_types = {
"migrate_city_gallery": [
migrate_city_photos
],
"tmp": [update_subregions]
"tmp": [
update_child_regions
]
}

14
apps/transfer/serializers/location.py
View File

@ -363,7 +363,7 @@ class CityMapSerializer(serializers.ModelSerializer):
situation = serializers.CharField(allow_blank=True, allow_null=True)
class Meta:
model = City
model = models.City
fields = (
"id",
"map1",
@ -393,8 +393,8 @@ class CityMapSerializer(serializers.ModelSerializer):
def set_city(self, data):
try:
city = City.objects.get(old_id=data['old_id'])
except City.DoesNotExist as e:
city = models.City.objects.get(old_id=data['old_id'])
except models.City.DoesNotExist as e:
raise ValueError(f"Cannot find city with id = {data['old_id']}: {e}")
except MultipleObjectsReturned as e:
raise ValueError(f"Find multiple cities with id = {data['old_id']}: {e}")
@ -415,7 +415,7 @@ class CityGallerySerializer(serializers.ModelSerializer):
attachment_file_name = serializers.CharField()
class Meta:
model = CityGallery
model = models.CityGallery
fields = ("id", "city_id", "attachment_file_name")
def validate(self, data):
@ -425,7 +425,7 @@ class CityGallerySerializer(serializers.ModelSerializer):
return data
def create(self, validated_data):
return CityGallery.objects.create(**validated_data)
return models.CityGallery.objects.create(**validated_data)
def set_old_id(self, data):
data['old_id'] = data.pop('id')
@ -444,8 +444,8 @@ class CityGallerySerializer(serializers.ModelSerializer):
def set_city(self, data):
try:
data['city'] = City.objects.get(old_id=data.pop('city_id'))
except City.DoesNotExist as e:
data['city'] = models.City.objects.get(old_id=data.pop('city_id'))
except models.City.DoesNotExist as e:
raise ValueError(f"Cannot get city with {data}: {e}")
except MultipleObjectsReturned as e:
raise ValueError(f"Multiple cities find with {data}: {e}")

11
project/settings/base.py
View File

@ -75,6 +75,7 @@ PROJECT_APPS = [
'favorites.apps.FavoritesConfig',
'rating.apps.RatingConfig',
'tag.apps.TagConfig',
'transfer.apps.TransferConfig',
]
EXTERNAL_APPS = [
@ -155,16 +156,6 @@ DATABASES = {
'HOST': os.environ.get('DB_HOSTNAME'),
'PORT': os.environ.get('DB_PORT'),
},
'legacy': {
'ENGINE': 'django.db.backends.mysql',
'HOST': '172.17.0.1',
# 'HOST': '172.23.0.1',
# 'HOST': 'mysql_db',
'PORT': 3306,
'NAME': 'dev',
'USER': 'dev',
'PASSWORD': 'octosecret123'
}
}

4
project/settings/local.py
View File

@ -43,7 +43,9 @@ THUMBNAIL_DEBUG = True
DATABASES.update({
'legacy': {
'ENGINE': 'django.db.backends.mysql',
'HOST': 'mysql_db',
'HOST': '172.17.0.1',
# 'HOST': '172.23.0.1',
# 'HOST': 'mysql_db',
'PORT': 3306,
'NAME': 'dev',
'USER': 'dev',