phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/migrate-location-app' into origin/feature/migrate-etablishment

Browse Source
This commit is contained in:
alex 2019-10-28 10:22:33 +03:00
commit 49dbc9c72e
14 changed files with 531 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/location/migrations/0013_country_old_id.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-27 06:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('location', '0012_data_migrate'),
]
operations = [
migrations.AddField(
model_name='country',
name='old_id',
field=models.IntegerField(blank=True, default=None, null=True),
),
]

17
apps/location/migrations/0014_remove_country_old_id.py Normal file
View File

@ -0,0 +1,17 @@
# Generated by Django 2.2.4 on 2019-10-27 06:04
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('location', '0013_country_old_id'),
]
operations = [
migrations.RemoveField(
model_name='country',
name='old_id',
),
]

18
apps/location/migrations/0015_country_old_id.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-27 06:04
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('location', '0014_remove_country_old_id'),
]
operations = [
migrations.AddField(
model_name='country',
name='old_id',
field=models.IntegerField(blank=True, default=None, null=True),
),
]

18
apps/location/migrations/0016_region_old_id.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-27 10:02
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('location', '0015_country_old_id'),
]
operations = [
migrations.AddField(
model_name='region',
name='old_id',
field=models.IntegerField(blank=True, default=None, null=True),
),
]

18
apps/location/migrations/0017_city_old_id.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-28 04:11
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('location', '0016_region_old_id'),
]
operations = [
migrations.AddField(
model_name='city',
name='old_id',
field=models.IntegerField(blank=True, default=None, null=True),
),
]

18
apps/location/migrations/0018_address_old_id.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.2.4 on 2019-10-28 05:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('location', '0017_city_old_id'),
]
operations = [
migrations.AddField(
model_name='address',
name='old_id',
field=models.IntegerField(blank=True, default=None, null=True),
),
]

5
apps/location/models.py
View File

@ -21,6 +21,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin):
low_price = models.IntegerField(default=25, verbose_name=_('Low price'))
high_price = models.IntegerField(default=50, verbose_name=_('High price'))
languages = models.ManyToManyField(Language, verbose_name=_('Languages'))
old_id = models.IntegerField(null=True, blank=True, default=None)
@property
def country_id(self):
@ -43,6 +44,7 @@ class Region(models.Model):
blank=True, default=None, on_delete=models.CASCADE)
country = models.ForeignKey(
Country, verbose_name=_('country'), on_delete=models.CASCADE)
old_id = models.IntegerField(null=True, blank=True, default=None)
class Meta:
"""Meta class."""
@ -76,7 +78,7 @@ class City(models.Model):
_('postal code'), max_length=10, default='', help_text=_('Ex.: 350018'))
is_island = models.BooleanField(_('is island'), default=False)
old_id = models.IntegerField(null=True, blank=True, default=None)
objects = CityQuerySet.as_manager()
class Meta:
@ -100,6 +102,7 @@ class Address(models.Model):
default='', help_text=_('Ex.: 350018'))
coordinates = models.PointField(
_('Coordinates'), blank=True, null=True, default=None)
old_id = models.IntegerField(null=True, blank=True, default=None)
class Meta:
"""Meta class."""

16
apps/location/transfer.py
View File

@ -101,24 +101,10 @@ card = {
"dependencies": ("Country", "Region"),
"fields": {
"Cities": {
"coordinates": (("lat", "latitude"), ("long", "longitude")),
"name": "name",
"code": "country_code_2",
"postal_code": "zip_code",
"is_island": ("is_island", "is_island", "django.db.models.Boolean")
},
"relations": {
"CityNames": {
"key": "city",
"fields": {
"name": "name",
}
},
"CityPhotos": {
"key": "city",
"fields": {
"coordinates": "geometries"
}
}
}
},
"relations": {

142
apps/location/transfer_data.py
View File

@ -1,25 +1,123 @@
from transfer.serializers.news import NewsSerializer
from transfer.models import PageTexts
from news.models import NewsType
from django.db.models import Value, IntegerField, F
from django.db.models import Q, QuerySet
from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer, AddressSerializer
from transfer.models import Cities, Locations
from pprint import pprint
# def transfer_news():
# news_type, _ = NewsType.objects.get_or_create(name="News")
#
# queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()),
# playlist=Value(1, output_field=IntegerField()))
# queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name'))
# queryset = queryset.annotate(template=F('page__template'))
#
# serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
# if serialized_data.is_valid():
# serialized_data.save()
# else:
# pprint(f"News serializer errors: {serialized_data.errors}")
#
#
# data_types = {
# "news": [transfer_news]
# }
def transfer_countries():
queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2
FROM cities WHERE
country_code_2 IS NOT NULL AND
country_code_2 != ""
GROUP BY cities.country_code_2""")
queryset = [vars(query) for query in queryset]
serialized_data = CountrySerializer(data=queryset, many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"Country serializer errors: {serialized_data.errors}")
def transfer_regions():
regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code,
cities.country_code_2, cities.subregion_code
FROM cities WHERE
(subregion_code IS NULL OR
subregion_code = "") AND
region_code IS NOT NULL AND
region_code != "" AND
country_code_2 IS NOT NULL AND
country_code_2 != ""
GROUP BY region_code""")
regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]
serialized_without_subregion = RegionSerializer(data=regions_without_subregion_queryset, many=True)
if serialized_without_subregion.is_valid():
serialized_without_subregion.save()
else:
pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")
regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code,
cities.country_code_2, cities.subregion_code
FROM cities WHERE
subregion_code IS NOT NULL AND
subregion_code != "" AND
region_code IS NOT NULL AND
region_code != "" AND
country_code_2 IS NOT NULL AND
country_code_2 != ""
AND cities.subregion_code in (
SELECT region_code FROM cities WHERE
(subregion_code IS NULL OR
subregion_code = "") AND
region_code IS NOT NULL AND
region_code != "" AND
country_code_2 IS NOT NULL AND
country_code_2 != ""
)
GROUP BY region_code""")
regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]
serialized_with_subregion = RegionSerializer(data=regions_with_subregion_queryset, many=True)
if serialized_with_subregion.is_valid():
serialized_with_subregion.save()
else:
pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")
def transfer_cities():
queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
cities.is_island, cities.region_code, cities.subregion_code
FROM cities WHERE
region_code IS NOT NULL AND
region_code != "" AND
country_code_2 IS NOT NULL AND
country_code_2 != ""
""")
queryset = [vars(query) for query in queryset]
serialized_data = CitySerializer(data=queryset, many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"City serializer errors: {serialized_data.errors}")
def transfer_addresses():
queryset = Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude,
locations.latitude, locations.address, locations.city_id
FROM locations WHERE
locations.address != "" AND
locations.address IS NOT NULL AND
locations.city_id IS NOT NULL AND
locations.city_id IN (SELECT cities.id
FROM cities WHERE
region_code IS NOT NULL AND
region_code != "" AND
country_code_2 IS NOT NULL AND
country_code_2 != "")""")
queryset = [vars(query) for query in queryset]
serialized_data = AddressSerializer(data=queryset, many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"Address serializer errors: {serialized_data.errors}")
data_types = {
"dictionaries": [
transfer_countries,
transfer_regions,
transfer_cities,
transfer_addresses
]
}

14
apps/main/migrations/0020_merge_20191025_0423.py Normal file
View File

@ -0,0 +1,14 @@
# Generated by Django 2.2.4 on 2019-10-25 04:23
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0019_auto_20191022_1359'),
('main', '0019_award_image_url'),
]
operations = [
]

14
apps/news/migrations/0023_merge_20191025_0423.py Normal file
View File

@ -0,0 +1,14 @@
# Generated by Django 2.2.4 on 2019-10-25 04:23
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('news', '0021_auto_20191021_1120'),
('news', '0022_auto_20191021_1306'),
]
operations = [
]

3
apps/transfer/management/commands/transfer.py
View File

@ -17,7 +17,8 @@ class Command(BaseCommand):
'partner',
'establishment',
'gallery',
'commercial'
'commercial',
'tmp'
]
def handle(self, *args, **options):

2
apps/transfer/models.py
View File

@ -232,7 +232,7 @@ class CityPhotos(MigrateMixin):
db_table = 'city_photos'
class Locations(models.Model):
class Locations(MigrateMixin):
using = 'legacy'
country_code = models.CharField(max_length=3)

268
apps/transfer/serializers/location.py Normal file
View File

@ -0,0 +1,268 @@
from rest_framework import serializers
from location.models import Country, Region, City, Address
from django.contrib.gis.geos import Point
from django.contrib.gis.geos import GEOSGeometry
import json
class CountrySerializer(serializers.ModelSerializer):
country_code_2 = serializers.CharField()
id = serializers.IntegerField()
class Meta:
model = Country
fields = (
"id",
"country_code_2",
)
def validate(self, data):
data["code"] = self.get_country_code(data)
del(data['country_code_2'])
data['old_id'] = data.pop('id')
return data
def create(self, validated_data):
# Some countries already in database
try:
country = Country.objects.get(code=validated_data['code'])
except Country.DoesNotExist:
country = Country.objects.create(**validated_data)
return country
def get_country_code(self, obj):
return obj.get("country_code_2")
class RegionSerializer(serializers.ModelSerializer):
region_code = serializers.CharField()
subregion_code = serializers.CharField(allow_null=True, allow_blank=True)
country_code_2 = serializers.CharField()
id = serializers.IntegerField()
class Meta:
model = Region
fields = (
"region_code",
"country_code_2",
"subregion_code",
"id"
)
def validate(self, data):
data = self.set_old_id(data)
data = self.set_code(data)
data = self.set_country(data)
return data
def create(self, validated_data):
# Some regions may be already in database
try:
region = Region.objects.get(old_id=validated_data['old_id'])
except Region.DoesNotExist:
region = Region.objects.create(**validated_data)
except Exception as e:
raise ValueError(f"REGION ERROR: {validated_data}: {e}")
return region
def set_code(self, data):
print(data)
if "subregion_code" in data and data["subregion_code"] is not None and data["subregion_code"].strip() != "":
try:
parent_region = Region.objects.get(code=str(data['region_code']))
except Exception as e:
raise ValueError(f"Parent region error with {data}: {e}")
data['parent_region'] = parent_region
data['code'] = data.pop('subregion_code')
del(data['region_code'])
else:
data['code'] = data.pop('region_code')
del(data['subregion_code'])
return data
def set_country(self, data):
try:
country = Country.objects.get(code=data['country_code_2'])
except Exception as e:
raise ValueError(f"Country error with {data}: {e}")
data["country"] = country
del(data['country_code_2'])
return data
def set_old_id(self, data):
data['old_id'] = data.pop("id")
return data
class CitySerializer(serializers.ModelSerializer):
country_code_2 = serializers.CharField()
region_code = serializers.CharField()
subregion_code = serializers.CharField(allow_null=True, allow_blank=True)
zip_code = serializers.CharField(allow_null=True, allow_blank=True)
is_island = serializers.IntegerField(allow_null=True)
name = serializers.CharField()
id = serializers.IntegerField()
class Meta:
model = City
fields = (
"country_code_2",
"region_code",
"subregion_code",
"zip_code",
"is_island",
"name",
"id"
)
def validate(self, data):
data = self.set_old_id(data)
data = self.set_relations(data)
data = self.set_is_island(data)
data = self.set_code(data)
data = self.set_zip_code(data)
return data
def create(self, validated_data):
return City.objects.create(**validated_data)
def set_is_island(self, data):
data['is_island'] = True if "is_island" in data \
and data['is_island'] is not None \
and data['is_island'] > 0 \
else False
return data
def set_code(self, data):
data['code'] = data.pop('region_code')
return data
def set_relations(self, data):
try:
region = Region.objects.filter(code=data['region_code']).first()
except Region.DoesNotExist as e:
try:
region = Region.objects.filter(code=data['subregion_code']).first()
except Region.DoesNotExist as e:
raise ValueError(f"Region not found with {data}: {e}")
data['region'] = region
del(data['subregion_code'])
try:
country = Country.objects.get(code=data['country_code_2'])
except Country.DoesNotExist as e:
raise ValueError(f"Country not found with {data}: {e}")
data['country'] = country
del(data['country_code_2'])
return data
def set_zip_code(self, data):
data['postal_code'] = data.pop('zip_code')
if data['postal_code'] is None:
data['postal_code'] = ""
return data
def set_old_id(self, data):
data['old_id'] = data.pop('id')
return data
class AddressSerializer(serializers.ModelSerializer):
id = serializers.IntegerField()
city_id = serializers.IntegerField()
zip_code = serializers.CharField(allow_null=True, allow_blank=True)
latitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True)
longitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True)
address = serializers.CharField()
class Meta:
model = Address
fields = (
"id",
"city_id",
"zip_code",
"latitude",
"longitude",
"address"
)
def validate(self, data):
data = self.set_old_id(data)
data = self.set_address(data)
data = self.set_postal_code(data)
data = self.set_city(data)
data = self.set_point(data)
return data
def create(self, validated_data):
return Address.objects.create(**validated_data)
def set_old_id(self, data):
data['old_id'] = data.pop("id")
return data
def set_postal_code(self, data):
data['postal_code'] = data.pop('zip_code', None)
if data['postal_code'] is None:
data['postal_code'] = ""
return data
def set_city(self, data):
try:
city = City.objects.filter(old_id=data['city_id']).first()
except City.DoesNotExist as e:
raise ValueError(f"City not found with {data}: {e}")
data['city'] = city
del(data['city_id'])
return data
def set_address(self, data):
address_list = data.pop('address').split(' ')
is_first_street = False
data['street_name_1'] = []
data['street_name_2'] = []
while len(address_list) > 0:
address_part = address_list.pop()
try:
address_part = int(address_part)
data['number'] = address_part
is_first_street = True
except:
if is_first_street:
data['street_name_1'].append(address_part)
else:
data['street_name_2'].append(address_part)
data['street_name_1'] = " ".join(data['street_name_1'])
data['street_name_2'] = " ".join(data['street_name_2'])
if "number" not in data:
data['number'] = 0
return data
def set_point(self, data):
if data['latitude'] is not None and data['longitude'] is not None:
data['coordinates'] = Point(float(data['longitude']), float(data['latitude']))
# data['coordinates'] = GEOSGeometry(
# json.dumps({
# "type": "Point",
# "coordinates": [data['longitude'], data['latitude']]
# }, ensure_ascii=False, default=str)
# )
else:
data['coordinates'] = None
del(data['latitude'])
del(data['longitude'])
return data