Merge remote-tracking branch 'origin/migrate-location-app' into origin/feature/migrate-etablishment

apps/location/migrations/0013_country_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 06:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0012_data_migrate'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='country',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0014_remove_country_old_id.py

@@ -0,0 +1,17 @@
+# Generated by Django 2.2.4 on 2019-10-27 06:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0013_country_old_id'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='country',
+            name='old_id',
+        ),
+    ]

apps/location/migrations/0015_country_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 06:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0014_remove_country_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='country',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0016_region_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-27 10:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0015_country_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='region',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0017_city_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-28 04:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0016_region_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='city',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/migrations/0018_address_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-10-28 05:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('location', '0017_city_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='address',
+            name='old_id',
+            field=models.IntegerField(blank=True, default=None, null=True),
+        ),
+    ]

apps/location/models.py

@@ -21,6 +21,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
     low_price = models.IntegerField(default=25, verbose_name=_('Low price'))
     high_price = models.IntegerField(default=50, verbose_name=_('High price'))
     languages = models.ManyToManyField(Language, verbose_name=_('Languages'))
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     @property
     def country_id(self):
@@ -43,6 +44,7 @@ class Region(models.Model):
         blank=True, default=None, on_delete=models.CASCADE)
     country = models.ForeignKey(
         Country, verbose_name=_('country'), on_delete=models.CASCADE)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     class Meta:
         """Meta class."""
@@ -76,7 +78,7 @@ class City(models.Model):
         _('postal code'), max_length=10, default='', help_text=_('Ex.: 350018'))
 
     is_island = models.BooleanField(_('is island'), default=False)
-
+    old_id = models.IntegerField(null=True, blank=True, default=None)
     objects = CityQuerySet.as_manager()
 
     class Meta:
@@ -100,6 +102,7 @@ class Address(models.Model):
         default='', help_text=_('Ex.: 350018'))
     coordinates = models.PointField(
         _('Coordinates'), blank=True, null=True, default=None)
+    old_id = models.IntegerField(null=True, blank=True, default=None)
 
     class Meta:
         """Meta class."""

apps/location/transfer.py

@@ -101,24 +101,10 @@ card = {
         "dependencies": ("Country", "Region"),
         "fields": {
             "Cities": {
-                "coordinates": (("lat", "latitude"), ("long", "longitude")),
+                "name": "name",
                 "code": "country_code_2",
                 "postal_code": "zip_code",
                 "is_island": ("is_island", "is_island", "django.db.models.Boolean")
-            },
-            "relations": {
-                "CityNames": {
-                    "key": "city",
-                    "fields": {
-                        "name": "name",
-                    }
-                },
-                "CityPhotos": {
-                    "key": "city",
-                    "fields": {
-                        "coordinates": "geometries"
-                    }
-                }
             }
         },
         "relations": {

apps/location/transfer_data.py

@@ -1,25 +1,123 @@
-from transfer.serializers.news import NewsSerializer
+from django.db.models import Q, QuerySet
-from transfer.models import PageTexts
+
-from news.models import NewsType
+from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer, AddressSerializer
-from django.db.models import Value, IntegerField, F
+from transfer.models import Cities, Locations
 from pprint import pprint
 
 
-# def transfer_news():
+def transfer_countries():
-#     news_type, _ = NewsType.objects.get_or_create(name="News")
+    queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 
-#
+                                  FROM cities WHERE 
-#     queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()),
+                                  country_code_2 IS NOT NULL AND 
-#                                                                     playlist=Value(1, output_field=IntegerField()))
+                                  country_code_2 != ""
-#     queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name'))
+                                  GROUP BY cities.country_code_2""")
-#     queryset = queryset.annotate(template=F('page__template'))
+
-#
+    queryset = [vars(query) for query in queryset]
-#     serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
+
-#     if serialized_data.is_valid():
+    serialized_data = CountrySerializer(data=queryset, many=True)
-#         serialized_data.save()
+    if serialized_data.is_valid():
-#     else:
+        serialized_data.save()
-#         pprint(f"News serializer errors: {serialized_data.errors}")
+    else:
-#
+        pprint(f"Country serializer errors: {serialized_data.errors}")
-#
+
-# data_types = {
+
-#     "news": [transfer_news]
+def transfer_regions():
-# }
+    regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code,
+                                                            cities.country_code_2, cities.subregion_code
+                                                            FROM cities WHERE
+                                                            (subregion_code IS NULL OR 
+                                                            subregion_code = "") AND
+                                                            region_code IS NOT NULL AND 
+                                                            region_code != "" AND
+                                                            country_code_2 IS NOT NULL AND 
+                                                            country_code_2 != ""
+                                                            GROUP BY region_code""")
+
+    regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]
+
+    serialized_without_subregion = RegionSerializer(data=regions_without_subregion_queryset, many=True)
+    if serialized_without_subregion.is_valid():
+        serialized_without_subregion.save()
+    else:
+        pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")
+
+    regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code,
+                                                        cities.country_code_2, cities.subregion_code
+                                                        FROM cities WHERE
+                                                        subregion_code IS NOT NULL AND
+                                                        subregion_code != "" AND
+                                                        region_code IS NOT NULL AND
+                                                        region_code != "" AND
+                                                        country_code_2 IS NOT NULL AND
+                                                        country_code_2 != ""
+                                                        AND cities.subregion_code in (
+                                                        SELECT region_code FROM cities WHERE
+                                                            (subregion_code IS NULL OR
+                                                            subregion_code = "") AND
+                                                            region_code IS NOT NULL AND
+                                                            region_code != "" AND
+                                                            country_code_2 IS NOT NULL AND
+                                                            country_code_2 != ""
+                                                        )
+                                                        GROUP BY region_code""")
+
+    regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]
+
+    serialized_with_subregion = RegionSerializer(data=regions_with_subregion_queryset, many=True)
+    if serialized_with_subregion.is_valid():
+        serialized_with_subregion.save()
+    else:
+        pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")
+
+
+def transfer_cities():
+    queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
+                                    cities.is_island, cities.region_code, cities.subregion_code
+                                    FROM cities WHERE
+                                    region_code IS NOT NULL AND 
+                                    region_code != "" AND
+                                    country_code_2 IS NOT NULL AND 
+                                    country_code_2 != ""
+                                        """)
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = CitySerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"City serializer errors: {serialized_data.errors}")
+
+
+def transfer_addresses():
+    queryset = Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, 
+                                      locations.latitude, locations.address, locations.city_id 
+                                      FROM locations WHERE 
+                                      locations.address != "" AND 
+                                      locations.address IS NOT NULL AND 
+                                      locations.city_id IS NOT NULL AND 
+                                      locations.city_id IN (SELECT cities.id
+                                        FROM cities WHERE
+                                        region_code IS NOT NULL AND 
+                                        region_code != "" AND
+                                        country_code_2 IS NOT NULL AND 
+                                        country_code_2 != "")""")
+
+    queryset = [vars(query) for query in queryset]
+
+    serialized_data = AddressSerializer(data=queryset, many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        pprint(f"Address serializer errors: {serialized_data.errors}")
+
+
+
+data_types = {
+    "dictionaries": [
+        transfer_countries,
+        transfer_regions,
+        transfer_cities,
+        transfer_addresses
+    ]
+}

apps/main/migrations/0020_merge_20191025_0423.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 04:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0019_auto_20191022_1359'),
+        ('main', '0019_award_image_url'),
+    ]
+
+    operations = [
+    ]

apps/news/migrations/0023_merge_20191025_0423.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.2.4 on 2019-10-25 04:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('news', '0021_auto_20191021_1120'),
+        ('news', '0022_auto_20191021_1306'),
+    ]
+
+    operations = [
+    ]

apps/transfer/management/commands/transfer.py

@@ -17,7 +17,8 @@ class Command(BaseCommand):
         'partner',
         'establishment',
         'gallery',
-        'commercial'
+        'commercial',
+        'tmp'
     ]
 
     def handle(self, *args, **options):

apps/transfer/models.py

@@ -232,7 +232,7 @@ class CityPhotos(MigrateMixin):
         db_table = 'city_photos'
 
 
-class Locations(models.Model):
+class Locations(MigrateMixin):
     using = 'legacy'
 
     country_code = models.CharField(max_length=3)

apps/transfer/serializers/location.py

@@ -0,0 +1,268 @@
+from rest_framework import serializers
+from location.models import Country, Region, City, Address
+from django.contrib.gis.geos import Point
+from django.contrib.gis.geos import GEOSGeometry
+import json
+
+
+class CountrySerializer(serializers.ModelSerializer):
+    country_code_2 = serializers.CharField()
+    id = serializers.IntegerField()
+
+    class Meta:
+        model = Country
+        fields = (
+            "id",
+            "country_code_2",
+        )
+
+    def validate(self, data):
+        data["code"] = self.get_country_code(data)
+        del(data['country_code_2'])
+
+        data['old_id'] = data.pop('id')
+
+        return data
+
+    def create(self, validated_data):
+        # Some countries already in database
+        try:
+            country = Country.objects.get(code=validated_data['code'])
+        except Country.DoesNotExist:
+            country = Country.objects.create(**validated_data)
+        return country
+
+    def get_country_code(self, obj):
+        return obj.get("country_code_2")
+
+
+class RegionSerializer(serializers.ModelSerializer):
+    region_code = serializers.CharField()
+    subregion_code = serializers.CharField(allow_null=True, allow_blank=True)
+    country_code_2 = serializers.CharField()
+    id = serializers.IntegerField()
+
+    class Meta:
+        model = Region
+        fields = (
+            "region_code",
+            "country_code_2",
+            "subregion_code",
+            "id"
+        )
+
+    def validate(self, data):
+        data = self.set_old_id(data)
+        data = self.set_code(data)
+        data = self.set_country(data)
+        return data
+
+    def create(self, validated_data):
+        # Some regions may be already in database
+        try:
+            region = Region.objects.get(old_id=validated_data['old_id'])
+        except Region.DoesNotExist:
+            region = Region.objects.create(**validated_data)
+        except Exception as e:
+            raise ValueError(f"REGION ERROR: {validated_data}: {e}")
+        return region
+
+    def set_code(self, data):
+        print(data)
+        if "subregion_code" in data and data["subregion_code"] is not None and data["subregion_code"].strip() != "":
+            try:
+                parent_region = Region.objects.get(code=str(data['region_code']))
+            except Exception as e:
+                raise ValueError(f"Parent region error with {data}: {e}")
+
+            data['parent_region'] = parent_region
+            data['code'] = data.pop('subregion_code')
+            del(data['region_code'])
+        else:
+            data['code'] = data.pop('region_code')
+            del(data['subregion_code'])
+
+        return data
+
+    def set_country(self, data):
+        try:
+            country = Country.objects.get(code=data['country_code_2'])
+        except Exception as e:
+            raise ValueError(f"Country error with {data}: {e}")
+
+        data["country"] = country
+        del(data['country_code_2'])
+
+        return data
+
+    def set_old_id(self, data):
+        data['old_id'] = data.pop("id")
+        return data
+
+
+class CitySerializer(serializers.ModelSerializer):
+    country_code_2 = serializers.CharField()
+    region_code = serializers.CharField()
+    subregion_code = serializers.CharField(allow_null=True, allow_blank=True)
+    zip_code = serializers.CharField(allow_null=True, allow_blank=True)
+    is_island = serializers.IntegerField(allow_null=True)
+    name = serializers.CharField()
+    id = serializers.IntegerField()
+
+    class Meta:
+        model = City
+        fields = (
+            "country_code_2",
+            "region_code",
+            "subregion_code",
+            "zip_code",
+            "is_island",
+            "name",
+            "id"
+        )
+
+    def validate(self, data):
+        data = self.set_old_id(data)
+        data = self.set_relations(data)
+        data = self.set_is_island(data)
+        data = self.set_code(data)
+        data = self.set_zip_code(data)
+        return data
+
+    def create(self, validated_data):
+        return City.objects.create(**validated_data)
+
+    def set_is_island(self, data):
+        data['is_island'] = True if "is_island" in data \
+                                    and data['is_island'] is not None \
+                                    and data['is_island'] > 0 \
+            else False
+        return data
+
+    def set_code(self, data):
+        data['code'] = data.pop('region_code')
+        return data
+
+    def set_relations(self, data):
+        try:
+            region = Region.objects.filter(code=data['region_code']).first()
+        except Region.DoesNotExist as e:
+            try:
+                region = Region.objects.filter(code=data['subregion_code']).first()
+            except Region.DoesNotExist as e:
+                raise ValueError(f"Region not found with {data}: {e}")
+
+        data['region'] = region
+        del(data['subregion_code'])
+
+        try:
+            country = Country.objects.get(code=data['country_code_2'])
+        except Country.DoesNotExist as e:
+            raise ValueError(f"Country not found with {data}: {e}")
+
+        data['country'] = country
+        del(data['country_code_2'])
+
+        return data
+
+    def set_zip_code(self, data):
+        data['postal_code'] = data.pop('zip_code')
+        if data['postal_code'] is None:
+            data['postal_code'] = ""
+        return data
+
+    def set_old_id(self, data):
+        data['old_id'] = data.pop('id')
+        return data
+
+
+class AddressSerializer(serializers.ModelSerializer):
+    id = serializers.IntegerField()
+    city_id = serializers.IntegerField()
+    zip_code = serializers.CharField(allow_null=True, allow_blank=True)
+    latitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True)
+    longitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True)
+    address = serializers.CharField()
+
+    class Meta:
+        model = Address
+        fields = (
+            "id",
+            "city_id",
+            "zip_code",
+            "latitude",
+            "longitude",
+            "address"
+        )
+
+    def validate(self, data):
+        data = self.set_old_id(data)
+        data = self.set_address(data)
+        data = self.set_postal_code(data)
+        data = self.set_city(data)
+        data = self.set_point(data)
+        return data
+
+    def create(self, validated_data):
+        return Address.objects.create(**validated_data)
+
+    def set_old_id(self, data):
+        data['old_id'] = data.pop("id")
+        return data
+
+    def set_postal_code(self, data):
+        data['postal_code'] = data.pop('zip_code', None)
+        if data['postal_code'] is None:
+            data['postal_code'] = ""
+        return data
+
+    def set_city(self, data):
+        try:
+            city = City.objects.filter(old_id=data['city_id']).first()
+        except City.DoesNotExist as e:
+            raise ValueError(f"City not found with {data}: {e}")
+
+        data['city'] = city
+        del(data['city_id'])
+        return data
+
+    def set_address(self, data):
+        address_list = data.pop('address').split(' ')
+        is_first_street = False
+        data['street_name_1'] = []
+        data['street_name_2'] = []
+        while len(address_list) > 0:
+            address_part = address_list.pop()
+            try:
+                address_part = int(address_part)
+                data['number'] = address_part
+                is_first_street = True
+            except:
+                if is_first_street:
+                    data['street_name_1'].append(address_part)
+                else:
+                    data['street_name_2'].append(address_part)
+
+        data['street_name_1'] = " ".join(data['street_name_1'])
+        data['street_name_2'] = " ".join(data['street_name_2'])
+        if "number" not in data:
+            data['number'] = 0
+
+        return data
+
+    def set_point(self, data):
+        if data['latitude'] is not None and data['longitude'] is not None:
+            data['coordinates'] = Point(float(data['longitude']), float(data['latitude']))
+            # data['coordinates'] = GEOSGeometry(
+            #     json.dumps({
+            #         "type": "Point",
+            #         "coordinates": [data['longitude'], data['latitude']]
+            #     }, ensure_ascii=False, default=str)
+            # )
+        else:
+            data['coordinates'] = None
+
+        del(data['latitude'])
+        del(data['longitude'])
+
+        return data