diff --git a/apps/location/migrations/0031_auto_20191117_1750.py b/apps/location/migrations/0031_auto_20191117_1750.py new file mode 100644 index 00000000..44a9479e --- /dev/null +++ b/apps/location/migrations/0031_auto_20191117_1750.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.7 on 2019-11-17 17:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0030_auto_20191116_1151'), + ] + + operations = [ + migrations.AlterField( + model_name='city', + name='region', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='location.Region', verbose_name='parent region'), + ), + ] diff --git a/apps/location/migrations/0032_auto_20191117_2300.py b/apps/location/migrations/0032_auto_20191117_2300.py new file mode 100644 index 00000000..acad9bf2 --- /dev/null +++ b/apps/location/migrations/0032_auto_20191117_2300.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.7 on 2019-11-17 23:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0031_auto_20191117_1750'), + ] + + operations = [ + migrations.RemoveField( + model_name='city', + name='mysql_ids', + ), + migrations.AddField( + model_name='city', + name='mysql_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0033_auto_20191118_0326.py b/apps/location/migrations/0033_auto_20191118_0326.py new file mode 100644 index 00000000..3fdc9310 --- /dev/null +++ b/apps/location/migrations/0033_auto_20191118_0326.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2019-11-18 03:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0032_auto_20191117_2300'), + ] + + operations = [ + migrations.AlterField( + model_name='country', + name='code', + field=models.CharField(max_length=255, verbose_name='Code'), + ), + ] diff --git a/apps/location/models.py b/apps/location/models.py index 795b2670..c979b2d9 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -34,7 +34,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): name = TJSONField(null=True, blank=True, default=None, verbose_name=_('Name'), help_text='{"en-GB":"some text"}') - code = models.CharField(max_length=255, unique=True, verbose_name=_('Code')) + code = models.CharField(max_length=255, unique=False, verbose_name=_('Code')) low_price = models.IntegerField(default=25, verbose_name=_('Low price')) high_price = models.IntegerField(default=50, verbose_name=_('High price')) languages = models.ManyToManyField(Language, verbose_name=_('Languages')) @@ -109,7 +109,7 @@ class City(models.Model): verbose_name=_('Translated name'), help_text='{"en-GB":"some text"}') code = models.CharField(_('code'), max_length=250) region = models.ForeignKey( - Region, verbose_name=_('parent region'), on_delete=models.CASCADE) + Region, verbose_name=_('parent region'), on_delete=models.CASCADE, blank=True, null=True) country = models.ForeignKey( Country, verbose_name=_('country'), on_delete=models.CASCADE) @@ -126,7 +126,7 @@ class City(models.Model): gallery = models.ManyToManyField('gallery.Image', through='location.CityGallery', blank=True) - mysql_ids = ArrayField(models.IntegerField(), blank=True, null=True) + mysql_id = models.IntegerField(blank=True, null=True, default=None) objects = CityQuerySet.as_manager() diff --git a/apps/location/ruby_data/child_regions.py b/apps/location/ruby_data/child_regions.py deleted file mode 100644 index 11ceed49..00000000 --- a/apps/location/ruby_data/child_regions.py +++ /dev/null @@ -1 +0,0 @@ -{"5174": {"code": "15", "name": "Korinthia", "region": "J", "country": "GR"}, "5175": {"code": "17", "name": "Messinia", "region": "J", "country": "GR"}, "5176": {"code": "16", "name": "Lakonia", "region": "J", "country": "GR"}, "5177": {"code": "11", "name": "Argolida", "region": "J", "country": "GR"}, "5178": {"code": "11", "name": "Argolida", "region": "J", "country": "GR"}, "5179": {"code": "03", "name": "Voiotia", "region": "H", "country": "GR"}, "5180": {"code": "04", "name": "Evvoias", "region": "H", "country": "GR"}} \ No newline at end of file diff --git a/apps/location/ruby_data/countries.py b/apps/location/ruby_data/countries.py deleted file mode 100644 index e958cf8e..00000000 --- a/apps/location/ruby_data/countries.py +++ /dev/null @@ -1 +0,0 @@ -[{"old_id": [9568], "data": {"code": "GP", "name": "Guadeloupe"}}, {"old_id": [1], "data": {"code": "AT", "name": "Austria"}}, {"old_id": [7307], "data": {"code": "AU", "name": "Australia"}}, {"old_id": [2298], "data": {"code": "BE", "name": "Belgium"}}, {"old_id": [11277], "data": {"code": "BR", "name": "Brazil"}}, {"old_id": [6226], "data": {"code": "CA", "name": "Canada"}}, {"old_id": [2955], "data": {"code": "CH", "name": "Switzerland"}}, {"old_id": [202], "data": {"code": "DE", "name": "Germany"}}, {"old_id": [3133], "data": {"code": "ES", "name": "Spain"}}, {"old_id": [514], "data": {"code": "FR", "name": "France"}}, {"old_id": [3758], "data": {"code": "GB", "name": "United Kingdom"}}, {"old_id": [8539], "data": {"code": "GE", "name": "Georgia"}}, {"old_id": [10164], "data": {"code": "GR", "name": "Greece"}}, {"old_id": [2283], "data": {"code": "HR", "name": "Croatia"}}, {"old_id": [8406], "data": {"code": "HU", "name": "Hungary"}}, {"old_id": [8527], "data": {"code": "IL", "name": "Israel"}}, {"old_id": [2274], "data": {"code": "IT", "name": "Italy"}}, {"old_id": [7370], "data": {"code": "JP", "name": "Japan"}}, {"old_id": [141], "data": {"code": "LU", "name": "Luxembourg"}}, {"old_id": [7709], "data": {"code": "MA", "name": "Morocco"}}, {"old_id": [9852], "data": {"code": "MV", "name": "Maldives"}}, {"old_id": [9030], "data": {"code": "NL", "name": "Netherlands"}}, {"old_id": [10390], "data": {"code": "PL", "name": "Poland"}}, {"old_id": [8915], "data": {"code": "RO", "name": "Romania"}}, {"old_id": [8238], "data": {"code": "RU", "name": "Russia"}}, {"old_id": [7230], "data": {"code": "SI", "name": "Slovenia"}}, {"old_id": [7846], "data": {"code": "US", "name": "United States"}}] \ No newline at end of file diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index d7ebfd29..0b6ddcdc 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,8 +1,9 @@ from transfer.serializers import location as location_serializers from transfer import models as transfer_models -from location.models import Country, Region +from location.models import Country, Region, City from pprint import pprint import json + from django.conf import settings from django.core.exceptions import MultipleObjectsReturned @@ -227,132 +228,139 @@ Update location models with ruby library Utils functions defined before transfer functions """ -def get_ruby_socket(type, params): +def get_ruby_socket(params): url = 'http://172.21.0.1:5678' # docker host - params["type"] = type response = get(url, params=params) - - data = json.loads(response.text) + try: + data = json.loads(response.text) + except Exception as e: + print(f"{response.text} error: {e}") + return None return data -def update_child_regions(): - child_regions_list = Region.objects.filter(old_id__isnull=False, parent_region__isnull=False) - ruby_data_to_write = {} +def get_ruby_data(): + cities = City.objects.filter(old_id__isnull=False) + ruby_data = {} + error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w") + + for city in cities: + try: + mysql_city = transfer_models.Cities.objects.get(id=city.old_id) + except transfer_models.Cities.DoesNotExist: + print(f"City with id {city.old_id} not found") + continue + + ruby_params = {} + + if mysql_city.country_code is not None: + ruby_params['country_code'] = mysql_city.country_code + + if mysql_city.country_code_2 is not None: + ruby_params['country_code_2'] = mysql_city.country_code_2 + + if mysql_city.region_code is not None: + ruby_params['region_code'] = mysql_city.region_code + + if mysql_city.subregion_code is not None: + ruby_params['subregion_code'] = mysql_city.subregion_code + + ruby_response = get_ruby_socket(ruby_params) + + if ruby_response is None: + continue + + if "error" in ruby_response or "country" not in ruby_response: + error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n") + continue + + ruby_data[mysql_city.id] = { + "mysql_id": mysql_city.id, + "name": mysql_city.name, + "name_translated": '{"en-GB":"' + mysql_city.name + '"}', + + "map1": mysql_city.map1, + "map2": mysql_city.map2, + "map_ref": mysql_city.map_ref, + "situation": mysql_city.situation, + + "is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False, + + "postal_code": mysql_city.zip_code if mysql_city.zip_code is not None else '', + "code": mysql_city.country_code_2.lower() + + } + + for ruby_data_key in ruby_response.keys(): + if "error" in ruby_response[ruby_data_key]: + error_file.write(json.dumps(ruby_response) + '\n') + else: + ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key] + + error_file.close() + + with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file: + ruby_data_file.write(json.dumps(ruby_data)) + + return ruby_data + + +def fix_location_models(): try: - with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file: - ruby_regions_list = json.loads(file.read()) - except: - ruby_regions_list = {} + ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r") + ruby_data = json.loads(ruby_data_file.read()) + except FileNotFoundError: + ruby_data = get_ruby_data() - for child_region in child_regions_list: - mysql_region = transfer_models.Cities.objects.get(id=child_region.old_id) + for mysql_id, city_object in ruby_data.items(): + country_data = city_object["country"] + try: + country = Country.objects.get(code=country_data['code'], mysql_ids__isnull=False) + country.mysql_ids.append(mysql_id) + country.save() + except Country.DoesNotExist: + country_data['mysql_ids'] = [mysql_id] + country = Country.objects.create(**country_data) + city_object['country'] = country - if child_region.id in ruby_regions_list: - ruby_data = ruby_regions_list[child_region.id] - else: - ruby_params = { - "country_code": mysql_region.country_code, - "country_code_2": mysql_region.country_code_2, - "region_code": mysql_region.region_code, - "region_name": mysql_region.name, - "subregion_code": mysql_region.subregion_code - } + if "region" in city_object: + region_data = city_object['region'] + region_data['country'] = country + try: + region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False) + region.mysql_ids.append(mysql_id) + region.save() + except Region.DoesNotExist: + region_data['mysql_ids'] = [mysql_id] + region = Region.objects.create(**region_data) - ruby_data = get_ruby_socket("subregion", ruby_params) - if "error" in ruby_data: - continue + if "subregion" in city_object: + subregion_data = city_object.pop('subregion') + subregion_data['country'] = country + try: + subregion = Region.objects.get(code=subregion_data['code'], mysql_ids__isnull=False) + subregion.mysql_ids.append(mysql_id) + subregion.save() + except Region.DoesNotExist: + subregion_data['parent_region'] = region + subregion_data['mysql_ids'] = [mysql_id] + subregion = Region.objects.create(**subregion_data) - ruby_data_to_write[child_region.id] = ruby_data + city_object['region'] = subregion + else: + city_object['region'] = region + + if "subregion" in city_object: + del(city_object['subregion']) try: - parent_region = Region.objects.get( - code=ruby_data['region'], - country__code=ruby_data['country'] - ) - country = parent_region.country - except MultipleObjectsReturned: - parent_region = Region.objects.filter( - code=ruby_data['region'], - country__code=ruby_data['country'] - ).first() - country = parent_region.country - except Region.DoesNotExist: - try: - parent_region = Region.objects.get(code=ruby_data['region']) - country = Country.objects.get(code=ruby_data['country']) - except MultipleObjectsReturned: - parent_region = Region.objects.filter(code=ruby_data['region']).first() - country = parent_region.country - - child_region.parent_region = parent_region - child_region.country = country - child_region.save() - - if len(ruby_data_to_write) > 0: - ruby_data_to_write = {**ruby_data_to_write, **ruby_regions_list} - with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/child_regions.py", "w") as file: - file.write(json.dumps(ruby_data_to_write)) - - -# def update_relations(old_instance, new_instance, relations_dict): -# for relation_name, relation_field in relations_dict.items(): -# relations_to_update = getattr(old_instance, relation_name, None) -# print(relations_to_update) -# if relations_to_update is None: -# continue -# # elif isinstance(relations_to_update) -# -# # relations_to_update = relations_to_update.all() -# -# # if len(relations_to_update) > 0: -# -# -# def update_country_from_ruby(): -# countries = Country.objects.filter(old_id__isnull=False).distinct('code') -# -# json_countries_list = [] -# for country in countries: -# json_country_dict = { -# "old_id": [country.old_id] -# } -# mysql_country = transfer_models.Cities.objects.get(id=country.old_id) -# -# ruby_params = { -# "country_code": mysql_country.country_code, -# "country_code_2": mysql_country.country_code_2, -# "region_code": mysql_country.region_code -# } -# -# ruby_data = get_ruby_socket("country", ruby_params) -# -# if "error" in ruby_data: -# print("ERROR: ") -# print(f"{ruby_data['error']}: {ruby_params}") -# continue -# -# json_country_dict["data"] = ruby_data -# -# duplicates = Country.objects.\ -# filter( -# old_id__isnull=False, -# code=country.code -# ).exclude(old_id=country.old_id) -# -# for duplicate in duplicates: -# json_country_dict["old_id"].append(duplicate.old_id) -# -# json_countries_list.append(json_country_dict) -# -# with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data/countries.py", "w") as file: -# file.write(json.dumps(json_countries_list)) -# -# - - - + City.objects.create(**city_object) + except Exception as e: + print(city_object) + print(e) + break data_types = { @@ -375,7 +383,7 @@ data_types = { migrate_city_photos ], "tmp": [ - update_child_regions + fix_location_models ] } diff --git a/project/settings/local.py b/project/settings/local.py index f9a096fe..04dbafb6 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -43,8 +43,7 @@ INSTALLED_APPS.append('transfer.apps.TransferConfig') DATABASES.update({ 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.22.0.1', - # 'HOST': 'mysql_db', + 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev',