Fix country queryset

apps/location/models.py

@@ -44,6 +44,7 @@ class Region(models.Model):
         blank=True, default=None, on_delete=models.CASCADE)
     country = models.ForeignKey(
         Country, verbose_name=_('country'), on_delete=models.CASCADE)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     class Meta:
         """Meta class."""
@@ -77,6 +78,7 @@ class City(models.Model):
         _('postal code'), max_length=10, default='', help_text=_('Ex.: 350018'))
 
     is_island = models.BooleanField(_('is island'), default=False)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     objects = CityQuerySet.as_manager()
 

apps/location/transfer_data.py

@@ -1,4 +1,4 @@
-from django.db.models import Q, F
+from django.db.models import Q, QuerySet
 
 from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer
 from transfer.models import Cities
@@ -6,10 +6,18 @@ from pprint import pprint
 
 
 def transfer_countries():
-    queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2=""))\
+    # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \
-        .values_list("country_code_2", flat=True).distinct()
+    #     .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2")
+    queryset = Cities.objects.raw("SELECT cities.id, cities.country_code_2 "
+                                  "FROM cities "
+                                  "WHERE NOT ((cities.country_code_2 IS NULL "
+                                  "OR (cities.country_code_2 = '' "
+                                  "AND cities.country_code_2 IS NOT NULL))) "
+                                  "GROUP BY cities.country_code_2")
 
-    serialized_data = CountrySerializer(data=list(queryset.values()), many=True)
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = CountrySerializer(data=queryset, many=True)
     if serialized_data.is_valid():
         serialized_data.save()
     else:
@@ -23,7 +31,7 @@ def transfer_regions():
                 Q(country_code_2__isnull=True) |
                 Q(region_code="") |
                 Q(country_code_2="")).\
-        values('region_code', 'country_code_2', 'subregion_code').distinct()
+        values('id', 'region_code', 'country_code_2', 'subregion_code').distinct()
 
     serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True)
     if serialized_without_subregion.is_valid():
@@ -51,7 +59,7 @@ def transfer_cities():
     queryset = Cities.objects.exclude(Q(region_code__isnull=True) |
                 Q(country_code_2__isnull=True) |
                 Q(region_code="") |
-                Q(country_code_2="")).only("name", "latitude", "longitude", "subregion_code",
+                Q(country_code_2="")).only("id", "name", "latitude", "longitude", "subregion_code",
                                            "country_code_2", "zip_code", "is_island")
 
     serialized_data = CitySerializer(data=list(queryset.values()), many=True)
@@ -63,10 +71,10 @@ def transfer_cities():
 
 data_types = {
     "dictionaries": [
-        transfer_regions,
     ],
     "tmp": [
         transfer_countries,
+        # transfer_regions,
         # transfer_cities
     ]
 }

apps/transfer/serializers/location.py

@@ -25,7 +25,7 @@ class CountrySerializer(serializers.ModelSerializer):
         # Some countries already in database
         try:
             country = Country.objects.get(code=validated_data['code'])
-        except Country.DoesNotExists:
+        except Country.DoesNotExist:
             country = Country.objects.create(**validated_data)
         return country
 
@@ -37,13 +37,15 @@ class RegionSerializer(serializers.ModelSerializer):
     region_code = serializers.CharField()
     subregion_code = serializers.CharField(allow_null=True)
     country_code_2 = serializers.CharField()
+    id = serializers.IntegerField()
 
     class Meta:
         model = Region
         fields = (
             "region_code",
             "country_code_2",
-            "subregion_code"
+            "subregion_code",
+            "id"
         )
 
     def validate(self, data):
@@ -67,14 +69,16 @@ class RegionSerializer(serializers.ModelSerializer):
         del (data['country_code_2'])
         del(data['subregion_code'])
 
+        data['old_id'] = data.pop("id")
+
         return data
 
     def create(self, validated_data):
         # Some regions may be already in database
         try:
-            region, _ = Region.objects.get_or_create(**validated_data)
+            region = Region.objects.get(old_id=validated_data['old_id'])
-        except Exception as e:
+        except Region.DoesNotExist:
-            raise ValueError(f"{validated_data}: {e}")
+            region = Region.objects.create(**validated_data)
         return region
 
 
@@ -84,6 +88,7 @@ class CitySerializer(serializers.ModelSerializer):
     zip_code = serializers.CharField()
     is_island = serializers.IntegerField()
     name = serializers.CharField()
+    id = serializers.IntegerField()
 
     class Meta:
         model = City
@@ -93,6 +98,7 @@ class CitySerializer(serializers.ModelSerializer):
             "zip_code",
             "is_island",
             "name",
+            "id"
         )
 
     def validate(self, data):
@@ -100,6 +106,7 @@ class CitySerializer(serializers.ModelSerializer):
         data = self.set_code(data)
         data = self.set_zip_code(data)
         data = self.set_relations(data)
+        data = self.set_old_id(data)
         return data
 
     def create(self, validated_data):
@@ -140,3 +147,5 @@ class CitySerializer(serializers.ModelSerializer):
         data['postal_code'] = data.pop('zip_code')
         return data
 
+    def set_old_id(self, data):
+        data['old_id'] = data.pop('id')