gault-millau/apps/location/transfer_data.py

from transfer.serializers import location as location_serializers
from transfer import models as transfer_models
from location.models import Country, Region, City
from pprint import pprint
import json

from django.conf import settings
from django.core.exceptions import MultipleObjectsReturned

from requests import get


def transfer_countries():
    queryset = transfer_models.Cities.objects.raw("""
                                    SELECT cities.id, cities.country_code_2 
                                    FROM cities 
                                    WHERE country_code_2 IS NOT NULL AND 
                                          country_code_2 != ""
                                    GROUP BY cities.id, cities.country_code_2     
                                  """)

    queryset = [vars(query) for query in queryset]

    serialized_data = location_serializers.CountrySerializer(data=queryset, many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"Country serializer errors: {serialized_data.errors}")


def transfer_regions():
    regions_without_subregion_queryset = transfer_models.Cities.objects.raw("""
                                                            SELECT cities.id, 
                                                                   cities.region_code,
                                                                   cities.country_code_2, 
                                                                   cities.subregion_code
                                                            FROM cities 
                                                            WHERE (subregion_code IS NULL 
                                                                   OR subregion_code = ""
                                                                   ) 
                                                                AND region_code IS NOT NULL 
                                                                AND region_code != ""
                                                                AND country_code_2 IS NOT NULL 
                                                                AND country_code_2 != ""
                                                            GROUP BY cities.id, 
                                                                     cities.region_code,
                                                                     cities.country_code_2, 
                                                                     cities.subregion_code     
                                                                """)

    regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset]

    serialized_without_subregion = location_serializers.RegionSerializer(data=regions_without_subregion_queryset,
                                                                         many=True)
    if serialized_without_subregion.is_valid():
        serialized_without_subregion.save()
    else:
        pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}")

    regions_with_subregion_queryset = transfer_models.Cities.objects.raw("""
                                                            SELECT 
                                                                  cities.id, 
                                                                  cities.region_code, 
                                                                  cities.country_code_2, 
                                                                  cities.subregion_code
                                                            FROM cities 
                                                            WHERE subregion_code IS NOT NULL AND
                                                                subregion_code != "" AND
                                                                region_code IS NOT NULL AND
                                                                region_code != "" AND
                                                                country_code_2 IS NOT NULL AND
                                                                country_code_2 != ""
                                                                AND cities.subregion_code in 
                                                                (
                                                                    SELECT region_code FROM cities WHERE
                                                                        (subregion_code IS NULL OR
                                                                        subregion_code = "") AND
                                                                        region_code IS NOT NULL AND
                                                                        region_code != "" AND
                                                                        country_code_2 IS NOT NULL AND
                                                                        country_code_2 != ""
                                                                )
                                                                GROUP BY cities.id, 
                                                                         cities.region_code, 
                                                                         cities.country_code_2, 
                                                                         cities.subregion_code     
                                                                """)

    regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset]

    serialized_with_subregion = location_serializers.RegionSerializer(data=regions_with_subregion_queryset,
                                                                      many=True)
    if serialized_with_subregion.is_valid():
        serialized_with_subregion.save()
    else:
        pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}")


def transfer_cities():
    queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
                                    cities.is_island, cities.region_code, cities.subregion_code
                                    FROM cities WHERE
                                    region_code IS NOT NULL AND 
                                    region_code != "" AND
                                    country_code_2 IS NOT NULL AND 
                                    country_code_2 != ""
                                        """)

    queryset = [vars(query) for query in queryset]

    serialized_data = location_serializers.CitySerializer(data=queryset, many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"City serializer errors: {serialized_data.errors}")


def transfer_addresses():
    queryset = transfer_models.Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, 
                                      locations.latitude, locations.address, locations.city_id 
                                      FROM locations WHERE 
                                      locations.address != "" AND 
                                      locations.address IS NOT NULL AND 
                                      locations.city_id IS NOT NULL AND 
                                      locations.city_id IN (SELECT cities.id
                                        FROM cities WHERE
                                        region_code IS NOT NULL AND 
                                        region_code != "" AND
                                        country_code_2 IS NOT NULL AND 
                                        country_code_2 != "")""")

    queryset = [vars(query) for query in queryset]

    serialized_data = location_serializers.AddressSerializer(data=queryset, many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"Address serializer errors: {serialized_data.errors}")


def transfer_wine_region():
    queryset = transfer_models.WineLocations.objects.filter(type='WineRegion')
    serialized_data = location_serializers.WineRegion(
      data=list(queryset.values()),
      many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")


def transfer_wine_sub_region():
    queryset = transfer_models.WineLocations.objects.filter(type='WineSubRegion')
    serialized_data = location_serializers.WineSubRegion(
      data=list(queryset.values()),
      many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")


def transfer_wine_village():
    queryset = transfer_models.WineLocations.objects.filter(type='Village')
    serialized_data = location_serializers.WineVillage(
      data=list(queryset.values()),
      many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"WineStandardClassificationSerializer errors: {serialized_data.errors}")


def update_flags():
    queryset = Country.objects.only("id", "code", "svg_image").filter(old_id__isnull=False)
    link_to_request = "https://s3.eu-central-1.amazonaws.com/gm-test.com/media"

    for query in queryset:
        svg_link = f"/svg/country/10-31-2019/{query.code}.svg"
        resp = get(f"{link_to_request}{svg_link}")
        if resp.status_code == 200:
            query.svg_image = svg_link
            query.save()


def migrate_city_map_situation():
    queryset = transfer_models.Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code,
                                        cities.is_island, cities.region_code, cities.subregion_code
                                        FROM cities WHERE
                                        region_code IS NOT NULL AND 
                                        region_code != "" AND
                                        country_code_2 IS NOT NULL AND 
                                        country_code_2 != ""
                                            """)

    serialized_data = location_serializers.CityMapSerializer(data=queryset, many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"City info serializer errors: {serialized_data.errors}")


def migrate_city_photos():
    queryset = transfer_models.CityPhotos.objects.raw("""SELECT city_photos.id, city_photos.city_id, city_photos.attachment_file_name 
                                            FROM city_photos WHERE
                                            city_photos.attachment_file_name IS NOT NULL AND
                                            city_id IN(
                                              SELECT cities.id
                                                FROM cities WHERE
                                                region_code IS NOT NULL AND 
                                                region_code != "" AND
                                                country_code_2 IS NOT NULL AND 
                                                country_code_2 != ""
                                            )
                                            """)

    queryset = [vars(query) for query in queryset]

    serialized_data = location_serializers.CityGallerySerializer(data=queryset, many=True)
    if serialized_data.is_valid():
        serialized_data.save()
    else:
        pprint(f"Address serializer errors: {serialized_data.errors}")


"""
Update location models with ruby library
Utils functions defined before transfer functions
"""

def get_ruby_socket(params):
    url = 'http://172.21.0.1:5678'  # docker host
    response = get(url, params=params)
    try:
        data = json.loads(response.text)
    except Exception as e:
        print(f"{response.text} error: {e}")
        return None
    return data


def get_ruby_data():
    cities = City.objects.filter(old_id__isnull=False)
    ruby_data = {}
    error_file = open(f"{settings.PROJECT_ROOT}/ruby_error.txt", "w")

    for city in cities:
        try:
            mysql_city = transfer_models.Cities.objects.get(id=city.old_id)
        except transfer_models.Cities.DoesNotExist:
            print(f"City with id {city.old_id} not found")
            continue

        ruby_params = {}

        if mysql_city.country_code is not None:
            ruby_params['country_code'] = mysql_city.country_code

        if mysql_city.country_code_2 is not None:
            ruby_params['country_code_2'] = mysql_city.country_code_2

        if mysql_city.region_code is not None:
            ruby_params['region_code'] = mysql_city.region_code

        if mysql_city.subregion_code is not None:
            ruby_params['subregion_code'] = mysql_city.subregion_code

        ruby_response = get_ruby_socket(ruby_params)

        if ruby_response is None:
            continue

        if "error" in ruby_response or "country" not in ruby_response:
            error_file.write(f"{json.dumps(ruby_params)}: {json.dumps(ruby_response)}\n")
            continue

        ruby_data[mysql_city.id] = {
            "mysql_id": mysql_city.id,
            "name": mysql_city.name,
            "name_translated": '{"en-GB":"' + mysql_city.name + '"}',

            "map1": mysql_city.map1,
            "map2": mysql_city.map2,
            "map_ref": mysql_city.map_ref,
            "situation": mysql_city.situation,

            "is_island": True if mysql_city.is_island is not None and mysql_city.is_island > 0 else False,

            "postal_code": mysql_city.zip_code if mysql_city.zip_code is not None else '',
            "code": mysql_city.country_code_2.lower()

        }

        for ruby_data_key in ruby_response.keys():
            if "error" in ruby_response[ruby_data_key]:
                error_file.write(json.dumps(ruby_response) + '\n')
            else:
                ruby_data[mysql_city.id][ruby_data_key] = ruby_response[ruby_data_key]

    error_file.close()

    with open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "w") as ruby_data_file:
        ruby_data_file.write(json.dumps(ruby_data))

    return ruby_data


def fix_location_models():

    try:
        ruby_data_file = open(f"{settings.PROJECT_ROOT}/apps/location/ruby_data.py", "r")
        ruby_data = json.loads(ruby_data_file.read())
    except FileNotFoundError:
        ruby_data = get_ruby_data()

    for mysql_id, city_object in ruby_data.items():
        country_data = city_object["country"]
        try:
            country = Country.objects.get(code=country_data['code'], mysql_ids__isnull=False)
            country.mysql_ids.append(mysql_id)
            country.save()
        except Country.DoesNotExist:
            country_data['mysql_ids'] = [mysql_id]
            country = Country.objects.create(**country_data)

        city_object['country'] = country

        if "region" in city_object:
            region_data = city_object['region']
            region_data['country'] = country
            try:
                region = Region.objects.get(code=region_data['code'], mysql_ids__isnull=False)
                region.mysql_ids.append(mysql_id)
                region.save()
            except Region.DoesNotExist:
                region_data['mysql_ids'] = [mysql_id]
                region = Region.objects.create(**region_data)

            if "subregion" in city_object:
                subregion_data = city_object.pop('subregion')
                subregion_data['country'] = country
                try:
                    subregion = Region.objects.get(code=subregion_data['code'], mysql_ids__isnull=False)
                    subregion.mysql_ids.append(mysql_id)
                    subregion.save()
                except Region.DoesNotExist:
                    subregion_data['parent_region'] = region
                    subregion_data['mysql_ids'] = [mysql_id]
                    subregion = Region.objects.create(**subregion_data)

                city_object['region'] = subregion
            else:
                city_object['region'] = region

        if "subregion" in city_object:
            del(city_object['subregion'])

        try:
            City.objects.create(**city_object)
        except Exception as e:
            print(city_object)
            print(e)
            break


data_types = {
    "dictionaries": [
        transfer_countries,
        transfer_regions,
        transfer_cities,
        transfer_addresses,
        transfer_wine_region,
        transfer_wine_sub_region,
        transfer_wine_village,
    ],
    "update_country_flag": [
        update_flags
    ],
    "update_city_info": [
        migrate_city_map_situation
    ],
    "migrate_city_gallery": [
        migrate_city_photos
    ],
    "tmp": [
        fix_location_models
    ]

}