From 106ddb044e8fa4d183925885e036fdd1808a223d Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 07:48:29 +0300 Subject: [PATCH 01/17] Add country transfer --- apps/location/transfer_data.py | 24 ++++++++++-------------- apps/transfer/serializers/location.py | 26 ++++++++++++++++++++++++++ project/settings/base.py | 4 ++-- 3 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 apps/transfer/serializers/location.py diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index deee2fe5..04c2b163 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,25 +1,21 @@ -from transfer.serializers.news import NewsSerializer -from transfer.models import PageTexts -from news.models import NewsType -from django.db.models import Value, IntegerField, F +from transfer.serializers.location import CountrySerializer +from transfer.models import Cities from pprint import pprint -def transfer_news(): - news_type, _ = NewsType.objects.get_or_create(name="News") +def transfer_countries(): + queryset = Cities.objects.exclude(country_code_2__isnull=True).values_list("country_code_2", flat=True).distinct() - queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), - playlist=Value(1, output_field=IntegerField())) - queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) - queryset = queryset.annotate(template=F('page__template')) - - serialized_data = NewsSerializer(data=list(queryset.values()), many=True) + serialized_data = CountrySerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + pprint(f"Country serializer errors: {serialized_data.errors}") data_types = { - "news": [transfer_news] + "dictionaries": [ + transfer_countries, + + ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py new file mode 100644 index 00000000..f3fe44e6 --- /dev/null +++ b/apps/transfer/serializers/location.py @@ -0,0 +1,26 @@ +from rest_framework import serializers +from location.models import Country + + +class CountrySerializer(serializers.ModelSerializer): + country_code_2 = serializers.CharField() + + class Meta: + model = Country + fields = ( + "country_code_2", + ) + + def validate(self, data): + data["code"] = self.get_country_code(data) + del(data['country_code_2']) + return data + + def create(self, validated_data): + # Some countries already in database + country, _ = Country.objects.get_or_create(**validated_data) + return Country + + def get_country_code(self, obj): + print(f"OBJECT: {obj}") + return obj.get("country_code_2") diff --git a/project/settings/base.py b/project/settings/base.py index 6c52afe6..cd797225 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -154,8 +154,8 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - # 'HOST': '172.23.0.1', - 'HOST': '127.0.0.1', + 'HOST': '172.20.0.1', + # 'HOST': '127.0.0.1', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From c61d5321a99e674d7b1f2ae4039e719851fcf63c Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 09:52:35 +0300 Subject: [PATCH 02/17] Tmp commit --- apps/location/transfer_data.py | 16 +++++++++++- apps/transfer/management/commands/transfer.py | 4 ++- apps/transfer/serializers/location.py | 25 +++++++++++++++++-- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 04c2b163..745ff23c 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,4 +1,4 @@ -from transfer.serializers.location import CountrySerializer +from transfer.serializers.location import CountrySerializer, RegionSerializer from transfer.models import Cities from pprint import pprint @@ -12,10 +12,24 @@ def transfer_countries(): else: pprint(f"Country serializer errors: {serialized_data.errors}") +def transfer_regions(): + regions_without_subregion_queryset = Cities.objects.\ + exclude(subregion_code__isnull=False).values('region_code', 'country_code_2', 'subregion_code').distinct() + + serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) + if serialized_without_subregion.is_valid(): + serialized_without_subregion.save() + else: + pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") + + data_types = { "dictionaries": [ transfer_countries, + ], + "tmp": [ + transfer_regions ] } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index d4a1c213..96b4155a 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -15,7 +15,9 @@ class Command(BaseCommand): 'subscriber', 'recipe', 'partner', - 'gallery' + 'gallery', + + 'tmp' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index f3fe44e6..a6d000ef 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from location.models import Country +from location.models import Country, Region class CountrySerializer(serializers.ModelSerializer): @@ -22,5 +22,26 @@ class CountrySerializer(serializers.ModelSerializer): return Country def get_country_code(self, obj): - print(f"OBJECT: {obj}") return obj.get("country_code_2") + + +class RegionSerializer(serializers.ModelSerializer): + region_code = serializers.CharField() + subregion_code = serializers.CharField() + country_code_2 = serializers.CharField() + + class Meta: + model = Region + fields = ( + "region_code", + "country_code_2", + "subregion_code" + ) + + def validate(self, data): + return data + + def create(self, validated_data): + # Some regions may be already in database + region, _ = Region.objects.get_or_create(**validated_data) + return region From bf375f33707bf662ae73805c434eb8cefda823a9 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 10:01:30 +0300 Subject: [PATCH 03/17] Remove errors Add version to timezonefinder --- apps/gallery/models.py | 6 ------ requirements/base.txt | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/gallery/models.py b/apps/gallery/models.py index 1a11bb35..bf5bc51c 100644 --- a/apps/gallery/models.py +++ b/apps/gallery/models.py @@ -27,13 +27,7 @@ class Image(ProjectBaseMixin, SORLImageMixin, PlatformMixin): blank=True, null=True, default=None, verbose_name=_('image orientation')) title = models.CharField(_('title'), max_length=255, default='') - -<<<<<<< HEAD - image = ThumbnailerImageField(upload_to=image_path, - verbose_name=_('Image file'), max_length=255) -======= objects = ImageQuerySet.as_manager() ->>>>>>> develop class Meta: """Meta class.""" diff --git a/requirements/base.txt b/requirements/base.txt index bd073996..b9fbcdfc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,7 +9,7 @@ fcm-django django-easy-select2 bootstrap-admin drf-yasg==1.16.0 -timezonefinder +timezonefinder==4.1.0 PySocks!=1.5.7,>=1.5.6; djangorestframework==3.9.4 From abe8eae6551ad4c6a5cc029869f001d73009b7ad Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 12:27:38 +0300 Subject: [PATCH 04/17] Tmp --- apps/location/transfer_data.py | 7 +++++-- apps/transfer/serializers/location.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 745ff23c..885c47e0 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -12,9 +12,13 @@ def transfer_countries(): else: pprint(f"Country serializer errors: {serialized_data.errors}") + def transfer_regions(): regions_without_subregion_queryset = Cities.objects.\ - exclude(subregion_code__isnull=False).values('region_code', 'country_code_2', 'subregion_code').distinct() + exclude(subregion_code__isnull=True).\ + exclude(region_code__isnull=True).\ + exclude(country_code_2__isnull=True).\ + values('region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) if serialized_without_subregion.is_valid(): @@ -23,7 +27,6 @@ def transfer_regions(): pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - data_types = { "dictionaries": [ transfer_countries, diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index a6d000ef..61df6baf 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -27,7 +27,7 @@ class CountrySerializer(serializers.ModelSerializer): class RegionSerializer(serializers.ModelSerializer): region_code = serializers.CharField() - subregion_code = serializers.CharField() + subregion_code = serializers.CharField(allow_null=True) country_code_2 = serializers.CharField() class Meta: @@ -39,6 +39,17 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): + data['code'] = data.pop('region_code') + if "country_code_2" in data and data["country_code_2"] is not None: + try: + country = Country.objects.get(code=data['country_code_2']) + data.country = country + del (data['country_code_2']) + + except Country.DoesNotExist as e: + print(f"Country error: {e}") + + del(data['subregion_code']) return data def create(self, validated_data): From e72e92244b8d465e40bff68cd3dfd2daebed48ff Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 08:42:00 +0300 Subject: [PATCH 05/17] Update country serializer --- apps/location/transfer_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 885c47e0..e4ac90a7 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,3 +1,5 @@ +from django.db.models import Q + from transfer.serializers.location import CountrySerializer, RegionSerializer from transfer.models import Cities from pprint import pprint @@ -15,9 +17,7 @@ def transfer_countries(): def transfer_regions(): regions_without_subregion_queryset = Cities.objects.\ - exclude(subregion_code__isnull=True).\ - exclude(region_code__isnull=True).\ - exclude(country_code_2__isnull=True).\ + exclude(Q(subregion_code__isnull=False) | Q(region_code__isnull=True) | Q(country_code_2__isnull=True)).\ values('region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) From 06b12f94279f8132daa8ca46e3b387d5da4a947a Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 09:18:45 +0300 Subject: [PATCH 06/17] Add country transfer --- apps/location/transfer_data.py | 9 +++++++-- apps/transfer/serializers/location.py | 20 +++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index e4ac90a7..db020e7a 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -6,7 +6,8 @@ from pprint import pprint def transfer_countries(): - queryset = Cities.objects.exclude(country_code_2__isnull=True).values_list("country_code_2", flat=True).distinct() + queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2=""))\ + .values_list("country_code_2", flat=True).distinct() serialized_data = CountrySerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): @@ -17,7 +18,11 @@ def transfer_countries(): def transfer_regions(): regions_without_subregion_queryset = Cities.objects.\ - exclude(Q(subregion_code__isnull=False) | Q(region_code__isnull=True) | Q(country_code_2__isnull=True)).\ + exclude(Q(subregion_code__isnull=False) | + Q(region_code__isnull=True) | + Q(country_code_2__isnull=True) | + Q(region_code="") | + Q(country_code_2="")).\ values('region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 61df6baf..b53cad6a 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -40,19 +40,21 @@ class RegionSerializer(serializers.ModelSerializer): def validate(self, data): data['code'] = data.pop('region_code') - if "country_code_2" in data and data["country_code_2"] is not None: - try: - country = Country.objects.get(code=data['country_code_2']) - data.country = country - del (data['country_code_2']) - - except Country.DoesNotExist as e: - print(f"Country error: {e}") + try: + country = Country.objects.get(code=data['country_code_2']) + except Exception as e: + raise ValueError(f"{data}: {e}") + data["country"] = country + del (data['country_code_2']) del(data['subregion_code']) + return data def create(self, validated_data): # Some regions may be already in database - region, _ = Region.objects.get_or_create(**validated_data) + try: + region, _ = Region.objects.get_or_create(**validated_data) + except Exception as e: + raise ValueError(f"{validated_data}: {e}") return region From cd724de65fc532a0643c2f1ae6272f0c7b83d050 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 09:19:34 +0300 Subject: [PATCH 07/17] Move countries to dictionaries --- apps/location/transfer_data.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index db020e7a..a5a659c0 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -35,9 +35,6 @@ def transfer_regions(): data_types = { "dictionaries": [ transfer_countries, - - ], - "tmp": [ - transfer_regions + transfer_regions, ] } From 8bcf4c9ca00472952250e6d2649880c15e304146 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 09:35:09 +0300 Subject: [PATCH 08/17] Add child regions --- apps/location/transfer_data.py | 25 ++++++++++++++++++++++++- apps/transfer/serializers/location.py | 17 +++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index a5a659c0..cbf5954f 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,6 +1,6 @@ from django.db.models import Q -from transfer.serializers.location import CountrySerializer, RegionSerializer +from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer from transfer.models import Cities from pprint import pprint @@ -31,10 +31,33 @@ def transfer_regions(): else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") + regions_with_subregion_queryset = Cities.objects. \ + exclude(Q(subregion_code__isnull=True) | + Q(subregion_code="") | + Q(region_code__isnull=True) | + Q(country_code_2__isnull=True) | + Q(region_code="") | + Q(country_code_2="")). \ + values('region_code', 'country_code_2', 'subregion_code').distinct() + + serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) + if serialized_with_subregion.is_valid(): + serialized_with_subregion.save() + else: + pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + + + +def transfer_cities(): + pass + data_types = { "dictionaries": [ transfer_countries, + ], + "tmp": [ transfer_regions, + # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index b53cad6a..654dc246 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -39,11 +39,20 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): - data['code'] = data.pop('region_code') + if "subregion" in data and data["subregion"] is not None: + try: + parent_region = Region.objects.get(code=data['region_code']) + except Exception as e: + raise ValueError(f"Parent region error with {data}: {e}") + data['parent_region'] = parent_region + data['code'] = data.pop('subregion_code') + + else: + data['code'] = data.pop('region_code') try: country = Country.objects.get(code=data['country_code_2']) except Exception as e: - raise ValueError(f"{data}: {e}") + raise ValueError(f"Country error with {data}: {e}") data["country"] = country del (data['country_code_2']) @@ -58,3 +67,7 @@ class RegionSerializer(serializers.ModelSerializer): except Exception as e: raise ValueError(f"{validated_data}: {e}") return region + + +class CitySerializer(serializers.ModelSerializer): + pass From 7eb241ffeda79a35029d902323ea8f74307f3bda Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 12:47:45 +0300 Subject: [PATCH 09/17] Tmp commit --- apps/location/models.py | 1 + apps/location/transfer.py | 6 -- apps/location/transfer_data.py | 19 ++++-- apps/transfer/serializers/location.py | 83 ++++++++++++++++++++++++--- 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index da645de6..c562d300 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -21,6 +21,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): low_price = models.IntegerField(default=25, verbose_name=_('Low price')) high_price = models.IntegerField(default=50, verbose_name=_('High price')) languages = models.ManyToManyField(Language, verbose_name=_('Languages')) + old_id = models.IntegerField(null=True, blank=True, default=None) @property def country_id(self): diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 74099303..9ad1c15a 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -112,12 +112,6 @@ card = { "fields": { "name": "name", } - }, - "CityPhotos": { - "key": "city", - "fields": { - "coordinates": "geometries" - } } } }, diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index cbf5954f..681efa51 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,4 +1,4 @@ -from django.db.models import Q +from django.db.models import Q, F from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer from transfer.models import Cities @@ -47,17 +47,26 @@ def transfer_regions(): pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") - def transfer_cities(): - pass + queryset = Cities.objects.exclude(Q(region_code__isnull=True) | + Q(country_code_2__isnull=True) | + Q(region_code="") | + Q(country_code_2="")).only("name", "latitude", "longitude", "subregion_code", + "country_code_2", "zip_code", "is_island") + + serialized_data = CitySerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"City serializer errors: {serialized_data.errors}") data_types = { "dictionaries": [ - transfer_countries, + transfer_regions, ], "tmp": [ - transfer_regions, + transfer_countries, # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 654dc246..93437187 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -1,25 +1,33 @@ from rest_framework import serializers -from location.models import Country, Region +from location.models import Country, Region, City class CountrySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() + id = serializers.IntegerField() class Meta: model = Country fields = ( + "id", "country_code_2", ) def validate(self, data): data["code"] = self.get_country_code(data) del(data['country_code_2']) + + data['old_id'] = data.pop('id') + return data def create(self, validated_data): # Some countries already in database - country, _ = Country.objects.get_or_create(**validated_data) - return Country + try: + country = Country.objects.get(code=validated_data['code']) + except Country.DoesNotExists: + country = Country.objects.create(**validated_data) + return country def get_country_code(self, obj): return obj.get("country_code_2") @@ -41,14 +49,15 @@ class RegionSerializer(serializers.ModelSerializer): def validate(self, data): if "subregion" in data and data["subregion"] is not None: try: - parent_region = Region.objects.get(code=data['region_code']) + parent_region = Region.objects.get(code=str(data['region_code'])) except Exception as e: raise ValueError(f"Parent region error with {data}: {e}") data['parent_region'] = parent_region - data['code'] = data.pop('subregion_code') + data['code'] = str(data.pop('subregion_code')) else: - data['code'] = data.pop('region_code') + data['code'] = str(data.pop('region_code')) + try: country = Country.objects.get(code=data['country_code_2']) except Exception as e: @@ -70,4 +79,64 @@ class RegionSerializer(serializers.ModelSerializer): class CitySerializer(serializers.ModelSerializer): - pass + country_code_2 = serializers.CharField() + subregion_code = serializers.CharField() + zip_code = serializers.CharField() + is_island = serializers.IntegerField() + name = serializers.CharField() + + class Meta: + model = City + fields = ( + "country_code_2", + "subregion_code", + "zip_code", + "is_island", + "name", + ) + + def validate(self, data): + data = self.set_is_island(data) + data = self.set_code(data) + data = self.set_zip_code(data) + data = self.set_relations(data) + return data + + def create(self, validated_data): + return City.objects.create(**validated_data) + + def set_is_island(self, data): + data['is_island'] = True if "is_island" in data \ + and data['is_island'] is not None \ + and data['is_island'] > 0 \ + else False + return data + + def set_code(self, data): + data['code'] = data.pop('country_code_2') + return data + + def set_relations(self, data): + try: + region = Region.objects.get(code=data['code']) + except: + try: + region = Region.objects.get(code=data['subregion_code']) + except Exception as e: + raise ValueError(f"Region not found with {data}: {e}") + data['region'] = region + del(data['subregion_code']) + + try: + country = Country.objects.get(code=data['code']) + except Exception as e: + raise ValueError(f"Region not found with {data}: {e}") + + data['country'] = country + + return data + + def set_zip_code(self, data): + data['postal_code'] = data.pop('zip_code') + return data + From 0db2638eb8c547eabad8d685d80b73b426016be2 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 17:23:04 +0300 Subject: [PATCH 10/17] Fix country queryset --- apps/location/models.py | 2 ++ apps/location/transfer_data.py | 22 +++++++++++++++------- apps/transfer/serializers/location.py | 19 ++++++++++++++----- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index c562d300..abf58da2 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -44,6 +44,7 @@ class Region(models.Model): blank=True, default=None, on_delete=models.CASCADE) country = models.ForeignKey( Country, verbose_name=_('country'), on_delete=models.CASCADE) + old_id = models.IntegerField(null=True, blank=True, default=None) class Meta: """Meta class.""" @@ -77,6 +78,7 @@ class City(models.Model): _('postal code'), max_length=10, default='', help_text=_('Ex.: 350018')) is_island = models.BooleanField(_('is island'), default=False) + old_id = models.IntegerField(null=True, blank=True, default=None) objects = CityQuerySet.as_manager() diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 681efa51..6b001e46 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,4 +1,4 @@ -from django.db.models import Q, F +from django.db.models import Q, QuerySet from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer from transfer.models import Cities @@ -6,10 +6,18 @@ from pprint import pprint def transfer_countries(): - queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2=""))\ - .values_list("country_code_2", flat=True).distinct() + # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \ + # .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2") + queryset = Cities.objects.raw("SELECT cities.id, cities.country_code_2 " + "FROM cities " + "WHERE NOT ((cities.country_code_2 IS NULL " + "OR (cities.country_code_2 = '' " + "AND cities.country_code_2 IS NOT NULL))) " + "GROUP BY cities.country_code_2") - serialized_data = CountrySerializer(data=list(queryset.values()), many=True) + queryset = [vars(query) for query in queryset] + + serialized_data = CountrySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: @@ -23,7 +31,7 @@ def transfer_regions(): Q(country_code_2__isnull=True) | Q(region_code="") | Q(country_code_2="")).\ - values('region_code', 'country_code_2', 'subregion_code').distinct() + values('id', 'region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) if serialized_without_subregion.is_valid(): @@ -51,7 +59,7 @@ def transfer_cities(): queryset = Cities.objects.exclude(Q(region_code__isnull=True) | Q(country_code_2__isnull=True) | Q(region_code="") | - Q(country_code_2="")).only("name", "latitude", "longitude", "subregion_code", + Q(country_code_2="")).only("id", "name", "latitude", "longitude", "subregion_code", "country_code_2", "zip_code", "is_island") serialized_data = CitySerializer(data=list(queryset.values()), many=True) @@ -63,10 +71,10 @@ def transfer_cities(): data_types = { "dictionaries": [ - transfer_regions, ], "tmp": [ transfer_countries, + # transfer_regions, # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 93437187..c9795957 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -25,7 +25,7 @@ class CountrySerializer(serializers.ModelSerializer): # Some countries already in database try: country = Country.objects.get(code=validated_data['code']) - except Country.DoesNotExists: + except Country.DoesNotExist: country = Country.objects.create(**validated_data) return country @@ -37,13 +37,15 @@ class RegionSerializer(serializers.ModelSerializer): region_code = serializers.CharField() subregion_code = serializers.CharField(allow_null=True) country_code_2 = serializers.CharField() + id = serializers.IntegerField() class Meta: model = Region fields = ( "region_code", "country_code_2", - "subregion_code" + "subregion_code", + "id" ) def validate(self, data): @@ -67,14 +69,16 @@ class RegionSerializer(serializers.ModelSerializer): del (data['country_code_2']) del(data['subregion_code']) + data['old_id'] = data.pop("id") + return data def create(self, validated_data): # Some regions may be already in database try: - region, _ = Region.objects.get_or_create(**validated_data) - except Exception as e: - raise ValueError(f"{validated_data}: {e}") + region = Region.objects.get(old_id=validated_data['old_id']) + except Region.DoesNotExist: + region = Region.objects.create(**validated_data) return region @@ -84,6 +88,7 @@ class CitySerializer(serializers.ModelSerializer): zip_code = serializers.CharField() is_island = serializers.IntegerField() name = serializers.CharField() + id = serializers.IntegerField() class Meta: model = City @@ -93,6 +98,7 @@ class CitySerializer(serializers.ModelSerializer): "zip_code", "is_island", "name", + "id" ) def validate(self, data): @@ -100,6 +106,7 @@ class CitySerializer(serializers.ModelSerializer): data = self.set_code(data) data = self.set_zip_code(data) data = self.set_relations(data) + data = self.set_old_id(data) return data def create(self, validated_data): @@ -140,3 +147,5 @@ class CitySerializer(serializers.ModelSerializer): data['postal_code'] = data.pop('zip_code') return data + def set_old_id(self, data): + data['old_id'] = data.pop('id') From 07f226604e813b172a8265b66608e08d857207f7 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 20:31:41 +0300 Subject: [PATCH 11/17] Fix regions without subregion --- apps/location/transfer_data.py | 68 +++++++++++++++------------ apps/transfer/serializers/location.py | 55 +++++++++++++--------- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 6b001e46..657a8853 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -8,12 +8,12 @@ from pprint import pprint def transfer_countries(): # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \ # .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2") - queryset = Cities.objects.raw("SELECT cities.id, cities.country_code_2 " - "FROM cities " - "WHERE NOT ((cities.country_code_2 IS NULL " - "OR (cities.country_code_2 = '' " - "AND cities.country_code_2 IS NOT NULL))) " - "GROUP BY cities.country_code_2") + queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 + FROM cities + WHERE NOT ((cities.country_code_2 IS NULL + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL))) + GROUP BY cities.country_code_2""") queryset = [vars(query) for query in queryset] @@ -25,34 +25,40 @@ def transfer_countries(): def transfer_regions(): - regions_without_subregion_queryset = Cities.objects.\ - exclude(Q(subregion_code__isnull=False) | - Q(region_code__isnull=True) | - Q(country_code_2__isnull=True) | - Q(region_code="") | - Q(country_code_2="")).\ - values('id', 'region_code', 'country_code_2', 'subregion_code').distinct() + regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities + WHERE NOT ((cities.subregion_code IS NOT NULL + OR cities.region_code IS NULL + OR cities.country_code_2 IS NULL + OR (cities.region_code = '' + AND cities.region_code IS NOT NULL) + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL))) + GROUP BY region_code""") - serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) + regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] + + serialized_without_subregion = RegionSerializer(data=regions_without_subregion_queryset, many=True) if serialized_without_subregion.is_valid(): serialized_without_subregion.save() else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - - regions_with_subregion_queryset = Cities.objects. \ - exclude(Q(subregion_code__isnull=True) | - Q(subregion_code="") | - Q(region_code__isnull=True) | - Q(country_code_2__isnull=True) | - Q(region_code="") | - Q(country_code_2="")). \ - values('region_code', 'country_code_2', 'subregion_code').distinct() - - serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) - if serialized_with_subregion.is_valid(): - serialized_with_subregion.save() - else: - pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + # + # regions_with_subregion_queryset = Cities.objects. \ + # exclude(Q(subregion_code__isnull=True) | + # Q(subregion_code="") | + # Q(region_code__isnull=True) | + # Q(country_code_2__isnull=True) | + # Q(region_code="") | + # Q(country_code_2="")). \ + # values('region_code', 'country_code_2', 'subregion_code').distinct() + # + # serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) + # if serialized_with_subregion.is_valid(): + # serialized_with_subregion.save() + # else: + # pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") def transfer_cities(): @@ -71,10 +77,10 @@ def transfer_cities(): data_types = { "dictionaries": [ + transfer_countries, ], "tmp": [ - transfer_countries, - # transfer_regions, + transfer_regions, # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index c9795957..4895b85e 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -49,27 +49,9 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): - if "subregion" in data and data["subregion"] is not None: - try: - parent_region = Region.objects.get(code=str(data['region_code'])) - except Exception as e: - raise ValueError(f"Parent region error with {data}: {e}") - data['parent_region'] = parent_region - data['code'] = str(data.pop('subregion_code')) - - else: - data['code'] = str(data.pop('region_code')) - - try: - country = Country.objects.get(code=data['country_code_2']) - except Exception as e: - raise ValueError(f"Country error with {data}: {e}") - data["country"] = country - - del (data['country_code_2']) - del(data['subregion_code']) - - data['old_id'] = data.pop("id") + data = self.set_code(data) + data = self.set_country(data) + data = self.set_old_id(data) return data @@ -81,6 +63,37 @@ class RegionSerializer(serializers.ModelSerializer): region = Region.objects.create(**validated_data) return region + def set_code(self, data): + if "subregion_code" in data and data["subregion_code"] is not None: + try: + parent_region = Region.objects.get(code=str(data['region_code'])) + except Exception as e: + raise ValueError(f"Parent region error with {data}: {e}") + + data['parent_region'] = parent_region + data['code'] = data.pop('subregion_code') + + else: + data['code'] = data.pop('region_code') + del(data['subregion_code']) + + return data + + def set_country(self, data): + try: + country = Country.objects.get(code=data['country_code_2']) + except Exception as e: + raise ValueError(f"Country error with {data}: {e}") + + data["country"] = country + del(data['country_code_2']) + + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + class CitySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() From 0d30543664ff0c5bc4c610ad8cf6395640f40fb4 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 21:43:34 +0300 Subject: [PATCH 12/17] Fix regions with subregions --- apps/location/transfer_data.py | 57 +++++++++++++++------------ apps/transfer/serializers/location.py | 5 ++- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 657a8853..3dc52c05 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -25,16 +25,16 @@ def transfer_countries(): def transfer_regions(): - regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, - cities.country_code_2, cities.subregion_code - FROM cities - WHERE NOT ((cities.subregion_code IS NOT NULL - OR cities.region_code IS NULL - OR cities.country_code_2 IS NULL - OR (cities.region_code = '' - AND cities.region_code IS NOT NULL) - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL))) + regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities + WHERE NOT ((cities.subregion_code IS NOT NULL + OR cities.region_code IS NULL + OR cities.country_code_2 IS NULL + OR (cities.region_code = '' + AND cities.region_code IS NOT NULL) + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL))) GROUP BY region_code""") regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] @@ -44,21 +44,28 @@ def transfer_regions(): serialized_without_subregion.save() else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - # - # regions_with_subregion_queryset = Cities.objects. \ - # exclude(Q(subregion_code__isnull=True) | - # Q(subregion_code="") | - # Q(region_code__isnull=True) | - # Q(country_code_2__isnull=True) | - # Q(region_code="") | - # Q(country_code_2="")). \ - # values('region_code', 'country_code_2', 'subregion_code').distinct() - # - # serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) - # if serialized_with_subregion.is_valid(): - # serialized_with_subregion.save() - # else: - # pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + + regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities + WHERE NOT ((cities.subregion_code IS NULL + OR (cities.subregion_code = '' + OR cities.region_code IS NULL + OR cities.country_code_2 IS NULL + OR (cities.region_code = '' + AND cities.region_code IS NOT NULL) + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL)))) + AND cities.subregion_code in (SELECT region_code FROM cities) + GROUP BY region_code""") + + regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] + + serialized_with_subregion = RegionSerializer(data=regions_with_subregion_queryset, many=True) + if serialized_with_subregion.is_valid(): + serialized_with_subregion.save() + else: + pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") def transfer_cities(): diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 4895b85e..00c2e834 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -49,9 +49,9 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): + data = self.set_old_id(data) data = self.set_code(data) data = self.set_country(data) - data = self.set_old_id(data) return data @@ -61,6 +61,8 @@ class RegionSerializer(serializers.ModelSerializer): region = Region.objects.get(old_id=validated_data['old_id']) except Region.DoesNotExist: region = Region.objects.create(**validated_data) + except Exception as e: + raise ValueError(f"REGION ERROR: {validated_data}: {e}") return region def set_code(self, data): @@ -72,6 +74,7 @@ class RegionSerializer(serializers.ModelSerializer): data['parent_region'] = parent_region data['code'] = data.pop('subregion_code') + del(data['region_code']) else: data['code'] = data.pop('region_code') From 255b1ad5f1c193ead0a6c786970762f0ca131f35 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 00:15:37 +0300 Subject: [PATCH 13/17] Fix queryset conditions --- apps/location/transfer.py | 10 +---- apps/location/transfer_data.py | 61 ++++++++++++++------------- apps/transfer/serializers/location.py | 28 +++++------- 3 files changed, 44 insertions(+), 55 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 9ad1c15a..fe4e8084 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -101,18 +101,10 @@ card = { "dependencies": ("Country", "Region"), "fields": { "Cities": { - "coordinates": (("lat", "latitude"), ("long", "longitude")), + "name": "name", "code": "country_code_2", "postal_code": "zip_code", "is_island": ("is_island", "is_island", "django.db.models.Boolean") - }, - "relations": { - "CityNames": { - "key": "city", - "fields": { - "name": "name", - } - } } }, "relations": { diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 3dc52c05..6d6861cc 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -6,13 +6,10 @@ from pprint import pprint def transfer_countries(): - # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \ - # .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2") queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 - FROM cities - WHERE NOT ((cities.country_code_2 IS NULL - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL))) + FROM cities WHERE + country_code_2 IS NOT NULL AND + country_code_2 != "" GROUP BY cities.country_code_2""") queryset = [vars(query) for query in queryset] @@ -27,14 +24,13 @@ def transfer_countries(): def transfer_regions(): regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, cities.country_code_2, cities.subregion_code - FROM cities - WHERE NOT ((cities.subregion_code IS NOT NULL - OR cities.region_code IS NULL - OR cities.country_code_2 IS NULL - OR (cities.region_code = '' - AND cities.region_code IS NOT NULL) - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL))) + FROM cities WHERE + (subregion_code IS NULL OR + subregion_code = "") AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" GROUP BY region_code""") regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] @@ -45,18 +41,24 @@ def transfer_regions(): else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, - cities.country_code_2, cities.subregion_code - FROM cities - WHERE NOT ((cities.subregion_code IS NULL - OR (cities.subregion_code = '' - OR cities.region_code IS NULL - OR cities.country_code_2 IS NULL - OR (cities.region_code = '' - AND cities.region_code IS NOT NULL) - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL)))) - AND cities.subregion_code in (SELECT region_code FROM cities) + regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities WHERE + subregion_code IS NOT NULL AND + subregion_code != "" AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + AND cities.subregion_code in ( + SELECT region_code FROM cities WHERE + (subregion_code IS NULL OR + subregion_code = "") AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + ) GROUP BY region_code""") regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] @@ -72,12 +74,13 @@ def transfer_cities(): queryset = Cities.objects.exclude(Q(region_code__isnull=True) | Q(country_code_2__isnull=True) | Q(region_code="") | - Q(country_code_2="")).only("id", "name", "latitude", "longitude", "subregion_code", - "country_code_2", "zip_code", "is_island") + Q(country_code_2="")).only("id", "name", "region_code", + "country_code_2", "zip_code", "is_island").distinct() serialized_data = CitySerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): - serialized_data.save() + # serialized_data.save() + print(serialized_data.validated_data) else: pprint(f"City serializer errors: {serialized_data.errors}") diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 00c2e834..5a95e223 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -35,7 +35,7 @@ class CountrySerializer(serializers.ModelSerializer): class RegionSerializer(serializers.ModelSerializer): region_code = serializers.CharField() - subregion_code = serializers.CharField(allow_null=True) + subregion_code = serializers.CharField(allow_null=True, allow_blank=True) country_code_2 = serializers.CharField() id = serializers.IntegerField() @@ -52,7 +52,6 @@ class RegionSerializer(serializers.ModelSerializer): data = self.set_old_id(data) data = self.set_code(data) data = self.set_country(data) - return data def create(self, validated_data): @@ -66,7 +65,8 @@ class RegionSerializer(serializers.ModelSerializer): return region def set_code(self, data): - if "subregion_code" in data and data["subregion_code"] is not None: + print(data) + if "subregion_code" in data and data["subregion_code"] is not None and data["subregion_code"].strip() != "": try: parent_region = Region.objects.get(code=str(data['region_code'])) except Exception as e: @@ -75,7 +75,6 @@ class RegionSerializer(serializers.ModelSerializer): data['parent_region'] = parent_region data['code'] = data.pop('subregion_code') del(data['region_code']) - else: data['code'] = data.pop('region_code') del(data['subregion_code']) @@ -100,7 +99,6 @@ class RegionSerializer(serializers.ModelSerializer): class CitySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() - subregion_code = serializers.CharField() zip_code = serializers.CharField() is_island = serializers.IntegerField() name = serializers.CharField() @@ -110,7 +108,6 @@ class CitySerializer(serializers.ModelSerializer): model = City fields = ( "country_code_2", - "subregion_code", "zip_code", "is_island", "name", @@ -118,11 +115,11 @@ class CitySerializer(serializers.ModelSerializer): ) def validate(self, data): + data = self.set_old_id(data) + data = self.set_relations(data) data = self.set_is_island(data) data = self.set_code(data) data = self.set_zip_code(data) - data = self.set_relations(data) - data = self.set_old_id(data) return data def create(self, validated_data): @@ -141,18 +138,14 @@ class CitySerializer(serializers.ModelSerializer): def set_relations(self, data): try: - region = Region.objects.get(code=data['code']) - except: - try: - region = Region.objects.get(code=data['subregion_code']) - except Exception as e: - raise ValueError(f"Region not found with {data}: {e}") + region = Region.objects.get(old_id=data['old_id']) + except Region.DoesNotExist as e: + raise ValueError(f"Region not found with {data}: {e}") data['region'] = region - del(data['subregion_code']) try: - country = Country.objects.get(code=data['code']) - except Exception as e: + country = Country.objects.get(old_id=data['old_id']) + except Country.DoesNotExist as e: raise ValueError(f"Region not found with {data}: {e}") data['country'] = country @@ -165,3 +158,4 @@ class CitySerializer(serializers.ModelSerializer): def set_old_id(self, data): data['old_id'] = data.pop('id') + return data From 25d58a8bc50d385f1decd5b839a4d5b349ac2920 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 07:19:17 +0300 Subject: [PATCH 14/17] Fix and add city transfer --- apps/location/models.py | 1 - apps/location/transfer_data.py | 24 ++++++++++++++---------- apps/transfer/serializers/location.py | 26 +++++++++++++++++++------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index abf58da2..0b650bce 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -79,7 +79,6 @@ class City(models.Model): is_island = models.BooleanField(_('is island'), default=False) old_id = models.IntegerField(null=True, blank=True, default=None) - objects = CityQuerySet.as_manager() class Meta: diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 6d6861cc..dca432e7 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -71,16 +71,20 @@ def transfer_regions(): def transfer_cities(): - queryset = Cities.objects.exclude(Q(region_code__isnull=True) | - Q(country_code_2__isnull=True) | - Q(region_code="") | - Q(country_code_2="")).only("id", "name", "region_code", - "country_code_2", "zip_code", "is_island").distinct() + queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, + cities.is_island, cities.region_code, cities.subregion_code + FROM cities WHERE + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + """) - serialized_data = CitySerializer(data=list(queryset.values()), many=True) + queryset = [vars(query) for query in queryset] + + serialized_data = CitySerializer(data=queryset, many=True) if serialized_data.is_valid(): - # serialized_data.save() - print(serialized_data.validated_data) + serialized_data.save() else: pprint(f"City serializer errors: {serialized_data.errors}") @@ -88,9 +92,9 @@ def transfer_cities(): data_types = { "dictionaries": [ transfer_countries, + transfer_regions, + transfer_cities ], "tmp": [ - transfer_regions, - # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 5a95e223..11d809ec 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -99,8 +99,10 @@ class RegionSerializer(serializers.ModelSerializer): class CitySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() - zip_code = serializers.CharField() - is_island = serializers.IntegerField() + region_code = serializers.CharField() + subregion_code = serializers.CharField(allow_null=True, allow_blank=True) + zip_code = serializers.CharField(allow_null=True, allow_blank=True) + is_island = serializers.IntegerField(allow_null=True) name = serializers.CharField() id = serializers.IntegerField() @@ -108,6 +110,8 @@ class CitySerializer(serializers.ModelSerializer): model = City fields = ( "country_code_2", + "region_code", + "subregion_code", "zip_code", "is_island", "name", @@ -133,27 +137,35 @@ class CitySerializer(serializers.ModelSerializer): return data def set_code(self, data): - data['code'] = data.pop('country_code_2') + data['code'] = data.pop('region_code') return data def set_relations(self, data): try: - region = Region.objects.get(old_id=data['old_id']) + region = Region.objects.filter(code=data['region_code']).first() except Region.DoesNotExist as e: - raise ValueError(f"Region not found with {data}: {e}") + try: + region = Region.objects.filter(code=data['subregion_code']).first() + except Region.DoesNotExist as e: + raise ValueError(f"Region not found with {data}: {e}") + data['region'] = region + del(data['subregion_code']) try: - country = Country.objects.get(old_id=data['old_id']) + country = Country.objects.get(code=data['country_code_2']) except Country.DoesNotExist as e: - raise ValueError(f"Region not found with {data}: {e}") + raise ValueError(f"Country not found with {data}: {e}") data['country'] = country + del(data['country_code_2']) return data def set_zip_code(self, data): data['postal_code'] = data.pop('zip_code') + if data['postal_code'] is None: + data['postal_code'] = "" return data def set_old_id(self, data): From 7d556df8d553e4d9092318271463dc05f477c10b Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 10:13:01 +0300 Subject: [PATCH 15/17] Add address serializer --- apps/location/models.py | 1 + apps/location/transfer_data.py | 29 +++++++- apps/transfer/models.py | 2 +- apps/transfer/serializers/location.py | 97 ++++++++++++++++++++++++++- 4 files changed, 125 insertions(+), 4 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index 0b650bce..5b440560 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -102,6 +102,7 @@ class Address(models.Model): default='', help_text=_('Ex.: 350018')) coordinates = models.PointField( _('Coordinates'), blank=True, null=True, default=None) + old_id = models.IntegerField(null=True, blank=True, default=None) class Meta: """Meta class.""" diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index dca432e7..c079f97b 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,7 +1,7 @@ from django.db.models import Q, QuerySet -from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer -from transfer.models import Cities +from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer, AddressSerializer +from transfer.models import Cities, Locations from pprint import pprint @@ -89,6 +89,30 @@ def transfer_cities(): pprint(f"City serializer errors: {serialized_data.errors}") +def transfer_addresses(): + queryset = Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, + locations.latitude, locations.address, locations.city_id + FROM locations WHERE + locations.address != "" AND + locations.address IS NOT NULL AND + locations.city_id IS NOT NULL AND + locations.city_id IN (SELECT cities.id + FROM cities WHERE + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "")""") + + queryset = [vars(query) for query in queryset] + + serialized_data = AddressSerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Address serializer errors: {serialized_data.errors}") + + + data_types = { "dictionaries": [ transfer_countries, @@ -96,5 +120,6 @@ data_types = { transfer_cities ], "tmp": [ + transfer_addresses ] } diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f6dd2dbc..05f1e175 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -232,7 +232,7 @@ class CityPhotos(MigrateMixin): db_table = 'city_photos' -class Locations(models.Model): +class Locations(MigrateMixin): using = 'legacy' country_code = models.CharField(max_length=3) diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 11d809ec..d4ea51cc 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -1,5 +1,8 @@ from rest_framework import serializers -from location.models import Country, Region, City +from location.models import Country, Region, City, Address +from django.contrib.gis.geos import Point +from django.contrib.gis.geos import GEOSGeometry +import json class CountrySerializer(serializers.ModelSerializer): @@ -171,3 +174,95 @@ class CitySerializer(serializers.ModelSerializer): def set_old_id(self, data): data['old_id'] = data.pop('id') return data + + +class AddressSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + city_id = serializers.IntegerField() + zip_code = serializers.CharField(allow_null=True, allow_blank=True) + latitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True) + longitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True) + address = serializers.CharField() + + class Meta: + model = Address + fields = ( + "id", + "city_id", + "zip_code", + "latitude", + "longitude", + "address" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_address(data) + data = self.set_postal_code(data) + data = self.set_city(data) + data = self.set_point(data) + return data + + def create(self, validated_data): + return Address.objects.create(**validated_data) + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + + def set_postal_code(self, data): + data['postal_code'] = data.pop('zip_code', None) + if data['postal_code'] is None: + data['postal_code'] = "" + return data + + def set_city(self, data): + try: + city = City.objects.filter(old_id=data['city_id']).first() + except City.DoesNotExist as e: + raise ValueError(f"City not found with {data}: {e}") + + data['city'] = city + del(data['city_id']) + return data + + def set_address(self, data): + address_list = data.pop('address').split(' ') + is_first_street = False + data['street_name_1'] = [] + data['street_name_2'] = [] + while len(address_list) > 0: + address_part = address_list.pop() + try: + address_part = int(address_part) + data['number'] = address_part + is_first_street = True + except: + if is_first_street: + data['street_name_1'].append(address_part) + else: + data['street_name_2'].append(address_part) + + data['street_name_1'] = " ".join(data['street_name_1']) + data['street_name_2'] = " ".join(data['street_name_2']) + if "number" not in data: + data['number'] = 0 + + return data + + def set_point(self, data): + if data['latitude'] is not None and data['longitude'] is not None: + data['coordinates'] = Point(float(data['longitude']), float(data['latitude'])) + # data['coordinates'] = GEOSGeometry( + # json.dumps({ + # "type": "Point", + # "coordinates": [data['longitude'], data['latitude']] + # }, ensure_ascii=False, default=str) + # ) + else: + data['coordinates'] = None + + del(data['latitude']) + del(data['longitude']) + + return data From 3622ac22b9adf031849125fcc83b4f464ddb2aab Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 10:13:43 +0300 Subject: [PATCH 16/17] Add migrations --- .../location/migrations/0013_country_old_id.py | 18 ++++++++++++++++++ .../migrations/0014_remove_country_old_id.py | 17 +++++++++++++++++ .../location/migrations/0015_country_old_id.py | 18 ++++++++++++++++++ apps/location/migrations/0016_region_old_id.py | 18 ++++++++++++++++++ apps/location/migrations/0017_city_old_id.py | 18 ++++++++++++++++++ .../location/migrations/0018_address_old_id.py | 18 ++++++++++++++++++ 6 files changed, 107 insertions(+) create mode 100644 apps/location/migrations/0013_country_old_id.py create mode 100644 apps/location/migrations/0014_remove_country_old_id.py create mode 100644 apps/location/migrations/0015_country_old_id.py create mode 100644 apps/location/migrations/0016_region_old_id.py create mode 100644 apps/location/migrations/0017_city_old_id.py create mode 100644 apps/location/migrations/0018_address_old_id.py diff --git a/apps/location/migrations/0013_country_old_id.py b/apps/location/migrations/0013_country_old_id.py new file mode 100644 index 00000000..7a6a3029 --- /dev/null +++ b/apps/location/migrations/0013_country_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0012_data_migrate'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0014_remove_country_old_id.py b/apps/location/migrations/0014_remove_country_old_id.py new file mode 100644 index 00000000..6b04c9b4 --- /dev/null +++ b/apps/location/migrations/0014_remove_country_old_id.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0013_country_old_id'), + ] + + operations = [ + migrations.RemoveField( + model_name='country', + name='old_id', + ), + ] diff --git a/apps/location/migrations/0015_country_old_id.py b/apps/location/migrations/0015_country_old_id.py new file mode 100644 index 00000000..56e9b251 --- /dev/null +++ b/apps/location/migrations/0015_country_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0014_remove_country_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0016_region_old_id.py b/apps/location/migrations/0016_region_old_id.py new file mode 100644 index 00000000..c4bd0d34 --- /dev/null +++ b/apps/location/migrations/0016_region_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 10:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0015_country_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='region', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0017_city_old_id.py b/apps/location/migrations/0017_city_old_id.py new file mode 100644 index 00000000..b38a30bd --- /dev/null +++ b/apps/location/migrations/0017_city_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-28 04:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0016_region_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='city', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0018_address_old_id.py b/apps/location/migrations/0018_address_old_id.py new file mode 100644 index 00000000..549375f4 --- /dev/null +++ b/apps/location/migrations/0018_address_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-28 05:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0017_city_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='address', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] From 7feed3ac8f002812d243ff20f50e32d5e30c1d0d Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 10:14:11 +0300 Subject: [PATCH 17/17] Move address transfer to dictionaries --- apps/location/transfer_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index c079f97b..b19eb42b 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -117,9 +117,7 @@ data_types = { "dictionaries": [ transfer_countries, transfer_regions, - transfer_cities - ], - "tmp": [ + transfer_cities, transfer_addresses ] }