From c03792d0e32b389675eff3abefa462a1d9eda25e Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 30 Sep 2019 11:19:09 +0300 Subject: [PATCH 001/173] Add legacy database Add requirements --- project/settings/base.py | 9 +++++++++ requirements/base.txt | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/project/settings/base.py b/project/settings/base.py index cfea18a5..1735154c 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -147,6 +147,15 @@ DATABASES = { 'HOST': os.environ.get('DB_HOSTNAME'), 'PORT': os.environ.get('DB_PORT'), }, + 'legacy': { + # mysql://92.53.66.27:3306 + 'ENGINE': 'django.db.backends.mysql', + 'HOST': '92.53.66.27', + 'PORT': 3306, + 'NAME': 'dev', + 'USER': 'dev', + 'PASSWORD': 'octosecret123' + } } diff --git a/requirements/base.txt b/requirements/base.txt index 25749c4b..c8a82c5b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -32,4 +32,6 @@ djangorestframework-simplejwt==4.3.0 django-elasticsearch-dsl>=7.0.0,<8.0.0 django-elasticsearch-dsl-drf==0.20.2 -sentry-sdk==0.11.2 \ No newline at end of file +sentry-sdk==0.11.2 + +mysqlclient==1.4.4 \ No newline at end of file From 277418bf72e654b44d191a2100ef6d49fbb6c2eb Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 30 Sep 2019 13:56:12 +0300 Subject: [PATCH 002/173] Fix docker host --- project/settings/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/project/settings/base.py b/project/settings/base.py index 1735154c..6ea27a6c 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -148,9 +148,8 @@ DATABASES = { 'PORT': os.environ.get('DB_PORT'), }, 'legacy': { - # mysql://92.53.66.27:3306 'ENGINE': 'django.db.backends.mysql', - 'HOST': '92.53.66.27', + 'HOST': '172.19.0.1', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From 661a7b3b6f2b673909b8cbcf40b6db17e6dc9453 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 30 Sep 2019 15:10:27 +0300 Subject: [PATCH 003/173] Add reqs to docker --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 3564a933..ac63ed76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.7 ENV PYTHONUNBUFFERED 1 RUN apt-get update; apt-get --assume-yes install binutils libproj-dev gdal-bin gettext +RUN apt-get install python3-dev libmysqlclient-dev RUN mkdir /code WORKDIR /code ADD . /code/ From e3d26b5dd1cc6f7bd4d629fd9d2d034d8a6bc35b Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 1 Oct 2019 07:43:38 +0300 Subject: [PATCH 004/173] Temp commit --- apps/migrates/__init__.py | 0 apps/migrates/admin.py | 3 ++ apps/migrates/apps.py | 5 ++ apps/migrates/migrations/__init__.py | 0 apps/migrates/models.py | 68 ++++++++++++++++++++++++++++ apps/migrates/tests.py | 3 ++ apps/migrates/views.py | 3 ++ project/settings/base.py | 1 + 8 files changed, 83 insertions(+) create mode 100644 apps/migrates/__init__.py create mode 100644 apps/migrates/admin.py create mode 100644 apps/migrates/apps.py create mode 100644 apps/migrates/migrations/__init__.py create mode 100644 apps/migrates/models.py create mode 100644 apps/migrates/tests.py create mode 100644 apps/migrates/views.py diff --git a/apps/migrates/__init__.py b/apps/migrates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/migrates/admin.py b/apps/migrates/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/apps/migrates/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/apps/migrates/apps.py b/apps/migrates/apps.py new file mode 100644 index 00000000..60537525 --- /dev/null +++ b/apps/migrates/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MigratesConfig(AppConfig): + name = 'migrates' diff --git a/apps/migrates/migrations/__init__.py b/apps/migrates/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/migrates/models.py b/apps/migrates/models.py new file mode 100644 index 00000000..7ce617c5 --- /dev/null +++ b/apps/migrates/models.py @@ -0,0 +1,68 @@ +# This is an auto-generated Django model module. +# You'll have to do the following manually to clean this up: +# * Rearrange models' order +# * Make sure each model has one field with primary_key=True +# * Make sure each ForeignKey has `on_delete` set to the desired behavior. +# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table +# Feel free to rename the models, but don't rename db_table values or field names. +from django.contrib.gis.db import models + + +class Cities(models.Model): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + country_code_2 = models.CharField(max_length=2, blank=True, null=True) + region_code = models.CharField(max_length=255, blank=True, null=True) + subregion_code = models.CharField(max_length=255, blank=True, null=True) + is_island = models.IntegerField(blank=True, null=True) + zip_code = models.CharField(max_length=9, blank=True, null=True) + situation = models.CharField(max_length=255, blank=True, null=True) + map_ref = models.CharField(max_length=255, blank=True, null=True) + map1 = models.CharField(max_length=255, blank=True, null=True) + map2 = models.CharField(max_length=255, blank=True, null=True) + latitude = models.FloatField(blank=True, null=True) + longitude = models.FloatField(blank=True, null=True) + encima_id = models.IntegerField(blank=True, null=True) + related_city_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + index_name = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'cities' + unique_together = (('name', 'region_code', 'country_code'),) + + +class CityNames(models.Model): + using = 'legacy' + + name = models.CharField(max_length=100, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'city_names' + unique_together = (('city', 'name', 'locale'),) + + +class CityPhotos(models.Model): + using = 'legacy' + + city_id = models.IntegerField(blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'city_photos' diff --git a/apps/migrates/tests.py b/apps/migrates/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/apps/migrates/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/apps/migrates/views.py b/apps/migrates/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/apps/migrates/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/project/settings/base.py b/project/settings/base.py index 6ea27a6c..2fca1b46 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -91,6 +91,7 @@ EXTERNAL_APPS = [ 'rest_framework_simplejwt.token_blacklist', 'solo', 'phonenumber_field', + 'apps.migrates' ] From 9dcdb7575ea09f5ffb4bff95bafce3722ed93c5f Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 2 Oct 2019 11:41:14 +0300 Subject: [PATCH 005/173] Switch application Add application command Update mixin Add mixin to models --- Dockerfile | 1 - apps/migrates/apps.py | 5 ---- apps/{migrates => transfer}/__init__.py | 0 apps/{migrates => transfer}/admin.py | 0 apps/transfer/apps.py | 7 ++++++ .../management}/__init__.py | 0 apps/transfer/management/commands/__init__.py | 0 apps/transfer/management/commands/transfer.py | 20 ++++++++++++++++ apps/transfer/migrations/__init__.py | 0 apps/transfer/mixins.py | 24 +++++++++++++++++++ apps/{migrates => transfer}/models.py | 8 ++++--- apps/{migrates => transfer}/tests.py | 0 apps/{migrates => transfer}/views.py | 0 project/settings/base.py | 2 +- 14 files changed, 57 insertions(+), 10 deletions(-) delete mode 100644 apps/migrates/apps.py rename apps/{migrates => transfer}/__init__.py (100%) rename apps/{migrates => transfer}/admin.py (100%) create mode 100644 apps/transfer/apps.py rename apps/{migrates/migrations => transfer/management}/__init__.py (100%) create mode 100644 apps/transfer/management/commands/__init__.py create mode 100644 apps/transfer/management/commands/transfer.py create mode 100644 apps/transfer/migrations/__init__.py create mode 100644 apps/transfer/mixins.py rename apps/{migrates => transfer}/models.py (95%) rename apps/{migrates => transfer}/tests.py (100%) rename apps/{migrates => transfer}/views.py (100%) diff --git a/Dockerfile b/Dockerfile index ac63ed76..3564a933 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM python:3.7 ENV PYTHONUNBUFFERED 1 RUN apt-get update; apt-get --assume-yes install binutils libproj-dev gdal-bin gettext -RUN apt-get install python3-dev libmysqlclient-dev RUN mkdir /code WORKDIR /code ADD . /code/ diff --git a/apps/migrates/apps.py b/apps/migrates/apps.py deleted file mode 100644 index 60537525..00000000 --- a/apps/migrates/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class MigratesConfig(AppConfig): - name = 'migrates' diff --git a/apps/migrates/__init__.py b/apps/transfer/__init__.py similarity index 100% rename from apps/migrates/__init__.py rename to apps/transfer/__init__.py diff --git a/apps/migrates/admin.py b/apps/transfer/admin.py similarity index 100% rename from apps/migrates/admin.py rename to apps/transfer/admin.py diff --git a/apps/transfer/apps.py b/apps/transfer/apps.py new file mode 100644 index 00000000..321ffb83 --- /dev/null +++ b/apps/transfer/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ + + +class TransferConfig(AppConfig): + name = 'transfer' + verbose_name = _('Transfer') diff --git a/apps/migrates/migrations/__init__.py b/apps/transfer/management/__init__.py similarity index 100% rename from apps/migrates/migrations/__init__.py rename to apps/transfer/management/__init__.py diff --git a/apps/transfer/management/commands/__init__.py b/apps/transfer/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py new file mode 100644 index 00000000..66ed0a2c --- /dev/null +++ b/apps/transfer/management/commands/transfer.py @@ -0,0 +1,20 @@ +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = 'Transfer data between databases' + + def handle(self, *args, **options): + if options['short']: + import __hello__ + else: + import this + + def add_arguments(self, parser): + parser.add_argument( + '-s', + '--short', + action='store_true', + default=False, + help='Вывод короткого сообщения' + ) diff --git a/apps/transfer/migrations/__init__.py b/apps/transfer/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/mixins.py b/apps/transfer/mixins.py new file mode 100644 index 00000000..88cfe091 --- /dev/null +++ b/apps/transfer/mixins.py @@ -0,0 +1,24 @@ +from django.db import models + + +class SecondDbManager(models.Manager): + def get_queryset(self): + qs = super().get_queryset() + + # if `use_db` is set on model use that for choosing the DB + if hasattr(self.model, 'use_db'): + qs = qs.using(self.model.use_db) + + return qs + + +class MigrateMixin(models.Model): + """Mixin to data transfer from legacy database""" + use_db = 'legacy' + objects = SecondDbManager() + + def _parse_instance_fields(self): + print(getattr(self, "using", "Using not found")) + + class Meta: + abstract = True diff --git a/apps/migrates/models.py b/apps/transfer/models.py similarity index 95% rename from apps/migrates/models.py rename to apps/transfer/models.py index 7ce617c5..4a0931d8 100644 --- a/apps/migrates/models.py +++ b/apps/transfer/models.py @@ -7,8 +7,10 @@ # Feel free to rename the models, but don't rename db_table values or field names. from django.contrib.gis.db import models +from transfer.mixins import MigrateMixin -class Cities(models.Model): + +class Cities(MigrateMixin): using = 'legacy' name = models.CharField(max_length=255, blank=True, null=True) @@ -36,7 +38,7 @@ class Cities(models.Model): unique_together = (('name', 'region_code', 'country_code'),) -class CityNames(models.Model): +class CityNames(MigrateMixin): using = 'legacy' name = models.CharField(max_length=100, blank=True, null=True) @@ -51,7 +53,7 @@ class CityNames(models.Model): unique_together = (('city', 'name', 'locale'),) -class CityPhotos(models.Model): +class CityPhotos(MigrateMixin): using = 'legacy' city_id = models.IntegerField(blank=True, null=True) diff --git a/apps/migrates/tests.py b/apps/transfer/tests.py similarity index 100% rename from apps/migrates/tests.py rename to apps/transfer/tests.py diff --git a/apps/migrates/views.py b/apps/transfer/views.py similarity index 100% rename from apps/migrates/views.py rename to apps/transfer/views.py diff --git a/project/settings/base.py b/project/settings/base.py index 2fca1b46..98bede78 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -71,6 +71,7 @@ PROJECT_APPS = [ 'review.apps.ReviewConfig', 'comment.apps.CommentConfig', 'favorites.apps.FavoritesConfig', + 'transfer.apps.TransferConfig' ] EXTERNAL_APPS = [ @@ -91,7 +92,6 @@ EXTERNAL_APPS = [ 'rest_framework_simplejwt.token_blacklist', 'solo', 'phonenumber_field', - 'apps.migrates' ] From 59320f43332792c50652ae290352722275582845 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 2 Oct 2019 14:23:24 +0300 Subject: [PATCH 006/173] Add temp transfer options to apps.location --- apps/location/transfer.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 apps/location/transfer.py diff --git a/apps/location/transfer.py b/apps/location/transfer.py new file mode 100644 index 00000000..2ae7411b --- /dev/null +++ b/apps/location/transfer.py @@ -0,0 +1,25 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" + +City = { + "dependencies": ("Country", "Region"), + "fields": { + "Cities": { + "coordinates": (("lat", "latitude", "Boolean"), ("long", "longtitude")), + "code": "country_code_2" + }, + "CityNames": { + "name": "name", + + } + } +} \ No newline at end of file From c0384cf03700d72fb08e1aee8ed70e15105ab667 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 2 Oct 2019 16:30:39 +0500 Subject: [PATCH 007/173] first --- apps/location/transfer.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 apps/location/transfer.py diff --git a/apps/location/transfer.py b/apps/location/transfer.py new file mode 100644 index 00000000..65c54584 --- /dev/null +++ b/apps/location/transfer.py @@ -0,0 +1,29 @@ +Country = { + "dependencies": None, + "legacy": "Cities", + "fields": { + "code": "country_code_2" + } +} + +Region = { + "dependencies": ["Country", "self"], + "legacy": "Cities", + "fields": { + "code": "region_code", + "country": "Country", + "parent_region": "self", + } +} + +City = { + "dependencies": ["Country", "Region"], + "legacy": "Cities", + "fields": { + "name": "name", + "region": "Region", + "country": "Country", + "postal_code": "zip_code", + "is_island": "is_island", + } +} From 223447fc95df923f0f1838c4fbfbfe645b7c9b56 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 2 Oct 2019 18:11:16 +0500 Subject: [PATCH 008/173] added model dicts to location/transfer.py --- apps/location/transfer.py | 55 ++++++++++++++++++++++++++++++++++++--- apps/transfer/models.py | 23 ++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 2ae7411b..e9b51978 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -8,18 +8,65 @@ field[0] - название аргумента field[1] - название поля в таблице legacy Опционально: field[2] - тип данных для преобразования +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) """ + +Country = { + "dependencies": None, + "fields": { + "Cities": { + "code": "country_code_2", + } + } +} + + +Region = { + "dependencies": ("Country", "Region"), + "fields": { + # нету аналога для поля name + "Cities": { + "code": "region_code", + }, + "ForeignKeys": { + "parent_region": "Region", + "country": "Country" + } + } +} + + City = { "dependencies": ("Country", "Region"), "fields": { "Cities": { - "coordinates": (("lat", "latitude", "Boolean"), ("long", "longtitude")), - "code": "country_code_2" + "coordinates": (("lat", "latitude"), ("long", "longitude")), + "code": "country_code_2", + "postal_code": "zip_code", + "is_island": ("is_island", "Boolean") }, "CityNames": { "name": "name", - + }, + "ForeignKeys": { + "region": "Region", + "country": "Country" } } -} \ No newline at end of file +} + + +Address = { + "dependencies": ("City",), + "fields": { + # нету аналога для поля number + "Locations": { + "postal_code": "zip_code", + "coordinates": (("lat", "latitude"), ("long", "longitude")) + }, + "ForeignKeys": { + "city": "City" + } + } +} diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 4a0931d8..8c8cbb1f 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -68,3 +68,26 @@ class CityPhotos(MigrateMixin): class Meta: managed = False db_table = 'city_photos' + + +class Locations(models.Model): + using = 'legacy' + + country_code = models.CharField(max_length=3) + region_code = models.CharField(max_length=3, blank=True, null=True) + subregion_code = models.CharField(max_length=3, blank=True, null=True) + zip_code = models.CharField(max_length=9, blank=True, null=True) + district_name = models.CharField(max_length=255, blank=True, null=True) + longitude = models.DecimalField(max_digits=10, decimal_places=6, blank=True, null=True) + latitude = models.DecimalField(max_digits=10, decimal_places=6, blank=True, null=True) + timezone = models.CharField(max_length=32, blank=True, null=True) + transportation = models.CharField(max_length=255, blank=True, null=True) + address = models.TextField(blank=True, null=True) + city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) + map_ref = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'locations' From 288d6cd699f9168519150ae7edd49faffceaa89f Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 2 Oct 2019 18:18:32 +0300 Subject: [PATCH 009/173] Tmp --- apps/transfer/management/commands/transfer.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 66ed0a2c..d8e18adc 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -1,14 +1,17 @@ from django.core.management.base import BaseCommand - +from django.apps import apps class Command(BaseCommand): help = 'Transfer data between databases' def handle(self, *args, **options): - if options['short']: - import __hello__ - else: - import this + for app in apps.get_app_configs(): + if app.name == "location": + print(app.name, ":") + + print(dir(app)) + for model in app.get_models(): + print("\t", model) def add_arguments(self, parser): parser.add_argument( From cf7184d7ac2014afa8fdf3e476faf044a443e9d2 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 3 Oct 2019 00:29:19 +0300 Subject: [PATCH 010/173] Switch transfer standard Add utils Add -d key --- apps/location/transfer.py | 82 +++++++++---------- apps/transfer/management/commands/transfer.py | 25 ++++-- apps/transfer/utils.py | 6 ++ 3 files changed, 63 insertions(+), 50 deletions(-) create mode 100644 apps/transfer/utils.py diff --git a/apps/location/transfer.py b/apps/location/transfer.py index e9b51978..9de7448f 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -11,62 +11,62 @@ field[1] - название поля в таблице legacy NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) """ - -Country = { - "dependencies": None, - "fields": { - "Cities": { - "code": "country_code_2", +card = { + "Country": { + "dependencies": None, + "fields": { + "Cities": { + "code": "country_code_2", + } } - } -} + }, - -Region = { - "dependencies": ("Country", "Region"), - "fields": { - # нету аналога для поля name - "Cities": { - "code": "region_code", + "Region": { + "dependencies": ("Country", "Region"), + "fields": { + # нету аналога для поля name + "Cities": { + "code": "region_code", + }, }, - "ForeignKeys": { + "relations": { "parent_region": "Region", "country": "Country" } - } -} + }, - -City = { - "dependencies": ("Country", "Region"), - "fields": { - "Cities": { - "coordinates": (("lat", "latitude"), ("long", "longitude")), - "code": "country_code_2", - "postal_code": "zip_code", - "is_island": ("is_island", "Boolean") + "City": { + "dependencies": ("Country", "Region"), + "fields": { + "Cities": { + "coordinates": (("lat", "latitude"), ("long", "longitude")), + "code": "country_code_2", + "postal_code": "zip_code", + "is_island": ("is_island", "Boolean") + }, + "CityNames": { + "name": "name", + }, }, - "CityNames": { - "name": "name", - }, - "ForeignKeys": { + "relations": { "region": "Region", "country": "Country" } - } -} + }, -Address = { - "dependencies": ("City",), - "fields": { - # нету аналога для поля number - "Locations": { - "postal_code": "zip_code", - "coordinates": (("lat", "latitude"), ("long", "longitude")) + "Address": { + "dependencies": ("City",), + "fields": { + # нету аналога для поля number + "Locations": { + "postal_code": "zip_code", + "coordinates": (("lat", "latitude"), ("long", "longitude")) + }, }, - "ForeignKeys": { + "relations": { "city": "City" } } + } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index d8e18adc..13cfbf90 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -1,23 +1,30 @@ from django.core.management.base import BaseCommand from django.apps import apps +from os.path import exists +from importlib.machinery import SourceFileLoader + +from transfer.utils import transfer_objects + class Command(BaseCommand): help = 'Transfer data between databases' def handle(self, *args, **options): - for app in apps.get_app_configs(): - if app.name == "location": - print(app.name, ":") - - print(dir(app)) - for model in app.get_models(): - print("\t", model) + if options['dictionaries']: + self._transfer_dictionaries() def add_arguments(self, parser): parser.add_argument( - '-s', - '--short', + '-d', + '--dictionaries', action='store_true', default=False, help='Вывод короткого сообщения' ) + + def _transfer_dictionaries(self): + for app in apps.get_app_configs(): + if exists(f"{app.path}/transfer.py"): + card_module = SourceFileLoader("transfer", f"{app.path}/transfer.py").load_module() + transfer_objects(app, card_module.card) + diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py new file mode 100644 index 00000000..639a2ccd --- /dev/null +++ b/apps/transfer/utils.py @@ -0,0 +1,6 @@ +from collections import OrderedDict + +def transfer_objects(app, transfer_card): + for model, card in transfer_card.items(): + print(model) + print(f"\t{card}") From 018d9f6557eac94e137fb1c820c81a28f91f76fb Mon Sep 17 00:00:00 2001 From: michail Date: Thu, 3 Oct 2019 17:54:26 +0500 Subject: [PATCH 011/173] added establishment and collection transfer.py files --- apps/collection/transfer.py | 42 +++++ apps/establishment/transfer.py | 70 ++++++++ apps/transfer/models.py | 316 +++++++++++++++++++++++++++++++++ 3 files changed, 428 insertions(+) create mode 100644 apps/collection/transfer.py create mode 100644 apps/establishment/transfer.py diff --git a/apps/collection/transfer.py b/apps/collection/transfer.py new file mode 100644 index 00000000..2e7b1306 --- /dev/null +++ b/apps/collection/transfer.py @@ -0,0 +1,42 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" + +card = { + "Collection": { + "dependencies": ("Country", ), + "fields": { + "Collections": { + # нету аналогов для полей start и end + "name": "title", + "slug": "slug", + } + }, + "relations": { + "country": "Country", + } + }, + "Guide": { + "dependencies": ("Collection", ), + "fields": { + "Guides": { + # нету аналогов для полей start и end + "name": "title" + } + }, + "relations": { + "parent": "Guide", + "collection": "Collection" + } + } +} + +used_apps = ("location", ) \ No newline at end of file diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py new file mode 100644 index 00000000..bf9aca39 --- /dev/null +++ b/apps/establishment/transfer.py @@ -0,0 +1,70 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" + + +card = { + "Establishment": { + "dependencies": ("Address", "Collection", ), + # нету аналогов для establishment_type, establishment_subtypes, schedule, comments, tags + "fields": { + "Establishments": { + "name": "name", + "slug": "slug" + }, + "EstablishmentInfos": { + "website": "website", + "facebook": "facebook", + "twitter": "twitter", + "lafourchette": "lafourchette", + "booking": "booking_url" + }, + }, + "relations": { + "address": "Address", + # "establishment_type": "EstablishmentType", + # "establishment_subtypes": "EstablishmentSubType", + "collections": "Collection", + # "schedule": "Timetable", + "award": "Award", + # "tags": "MetaDataContent", + "reviews": "Review", + # "comments": "Comment", + # "favorites": "Favorites" + + } + }, + "Menu": { + "dependencies": ("Establishment", ), + "fields": { + "Menus": { + "category": ("name", "TJSONField") + } + }, + "relations": { + "establishment": "Establishment" + } + }, + "ContactPhone": { + "dependencies": ("Establishment", ), + "fields": { + "Establishments": { + "phone": "phone" + }, + "relations": { + "establishment": "Establishment" + } + } + } +} + + +used_apps = ("review", "location", "collection", "main", "timetable", "favorites", "comment", ) \ No newline at end of file diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 8c8cbb1f..a811c305 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -91,3 +91,319 @@ class Locations(models.Model): class Meta: managed = False db_table = 'locations' + + +class Collections(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + tag_name = models.CharField(max_length=255, blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + active = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + aasm_state = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'collections' + + + +# class CollectionEvents(MigrateMixin): +# using = 'legacy' +# +# account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) +# account_collection = models.ForeignKey(AccountCollections, models.DO_NOTHING, blank=True, null=True) +# possible_dates = models.CharField(max_length=255, blank=True, null=True) +# final_date = models.DateTimeField(blank=True, null=True) +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'collection_events' + + +# class CollectionEventAvailabilities(MigrateMixin): +# using = 'legacy' +#TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts + +# collection_event = models.ForeignKey('CollectionEvents', models.DO_NOTHING, blank=True, null=True) +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# email_address = models.ForeignKey('EmailAddresses', models.DO_NOTHING, blank=True, null=True) +# date = models.DateTimeField(blank=True, null=True) +# state = models.CharField(max_length=255, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'collection_event_availabilities' + + +class Guides(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + vintage = models.IntegerField(blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + site_id = models.IntegerField(blank=True, null=True) + inserter_field = models.CharField(max_length=255, blank=True, null=True) + items_count = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'guides' + + +class GuideAds(MigrateMixin): + using = 'legacy' + + nb_pages = models.IntegerField(blank=True, null=True) + nb_right_pages = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + guide_ad_node_id = models.IntegerField(blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'guide_ads' + + +class GuideFilters(MigrateMixin): + using = 'legacy' + + year = models.TextField(blank=True, null=True) + establishment_type = models.CharField(max_length=255, blank=True, null=True) + countries = models.TextField(blank=True, null=True) + regions = models.TextField(blank=True, null=True) + subregions = models.TextField(blank=True, null=True) + wine_regions = models.TextField(blank=True, null=True) + wine_classifications = models.TextField(blank=True, null=True) + wine_colors = models.TextField(blank=True, null=True) + wine_types = models.TextField(blank=True, null=True) + max_mark = models.FloatField(blank=True, null=True) + min_mark = models.FloatField(blank=True, null=True) + marks_only = models.IntegerField(blank=True, null=True) + locales = models.CharField(max_length=255, blank=True, null=True) + states = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + guide_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'guide_filters' + + +class GuideSections(MigrateMixin): + using = 'legacy' + + type = models.CharField(max_length=255) + key_name = models.CharField(max_length=255, blank=True, null=True) + value_name = models.CharField(max_length=255, blank=True, null=True) + right = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + +# class GuideElements(MigrateMixin): +# using = 'legacy' +# +# type = models.CharField(max_length=255) +# establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) +# review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) +# review_text = models.ForeignKey('ReviewTexts', models.DO_NOTHING, blank=True, null=True) +# wine_region = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# wine = models.ForeignKey('Products', models.DO_NOTHING, blank=True, null=True) +# color = models.CharField(max_length=255, blank=True, null=True) +# order_number = models.IntegerField(blank=True, null=True) +# guide_ad = models.ForeignKey(GuideAds, models.DO_NOTHING, blank=True, null=True) +# city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) +# section = models.ForeignKey('GuideSections', models.DO_NOTHING, blank=True, null=True) +# guide_id = models.IntegerField(blank=True, null=True) +# parent_id = models.IntegerField(blank=True, null=True) +# lft = models.IntegerField() +# rgt = models.IntegerField() +# depth = models.IntegerField() +# children_count = models.IntegerField() +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'guide_elements' + + +class Establishments(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255) + index_name = models.CharField(max_length=255, blank=True, null=True) + slug = models.CharField(unique=True, max_length=255, blank=True, null=True) + phone = models.CharField(max_length=255, blank=True, null=True) + fax = models.CharField(max_length=255, blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + location = models.ForeignKey('Locations', models.DO_NOTHING, blank=True, null=True) + unique_key = models.CharField(max_length=255, blank=True, null=True) + filemaker_id = models.IntegerField(unique=True, blank=True, null=True) + aut_mysql_id = models.IntegerField(unique=True, blank=True, null=True) + fra_encima_id = models.IntegerField(blank=True, null=True) + ca_import_id = models.IntegerField(blank=True, null=True) + ch_import_id = models.IntegerField(blank=True, null=True) + be_ezpublish_id = models.IntegerField(blank=True, null=True) + au_import_id = models.IntegerField(blank=True, null=True) + lux_import_id = models.IntegerField(blank=True, null=True) + hun_import_id = models.IntegerField(blank=True, null=True) + deu_import_id = models.IntegerField(blank=True, null=True) + win_import_id = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + state = models.CharField(max_length=255, blank=True, null=True) + manager_revised_at = models.DateTimeField(blank=True, null=True) + cover_id = models.IntegerField(blank=True, null=True) + parent_id = models.IntegerField(blank=True, null=True) + admin_updated_at = models.DateTimeField(blank=True, null=True) + admin_updated_by = models.IntegerField(blank=True, null=True) + company_id = models.IntegerField(blank=True, null=True) + production_type = models.CharField(max_length=3000, blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishments' + + +# class EstablishmentAssets(MigrateMixin): +# using = 'legacy' +# +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING) +# account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) +# menu_id = models.IntegerField(blank=True, null=True) +# type = models.CharField(max_length=64) +# scope = models.CharField(max_length=32) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# attachment_file_name = models.CharField(max_length=255, blank=True, null=True) +# attachment_content_type = models.CharField(max_length=255, blank=True, null=True) +# geometries = models.CharField(max_length=1024, blank=True, null=True) +# attachment_file_size = models.IntegerField(blank=True, null=True) +# attachment_updated_at = models.DateTimeField(blank=True, null=True) +# +# class Meta: +# managed = False +# db_table = 'establishment_assets' + +class EstablishmentBacklinks(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + partnership_name = models.CharField(max_length=255, blank=True, null=True) + partnership_icon = models.CharField(max_length=255, blank=True, null=True) + backlink_url = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + type = models.CharField(max_length=255, blank=True, null=True) + starting_date = models.DateField(blank=True, null=True) + expiry_date = models.DateField(blank=True, null=True) + price_per_month = models.DecimalField(max_digits=10, decimal_places=2, blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishment_backlinks' + + +# class EstablishmentCollections(MigrateMixin): +# using = 'legacy' +# +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# account_collection = models.ForeignKey(AccountCollections, models.DO_NOTHING, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'establishment_collections' + +class EstablishmentHolidays(MigrateMixin): + using = 'legacy' + + start_date = models.DateField(blank=True, null=True) + end_date = models.DateField(blank=True, null=True) + establishment_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'establishment_holidays' + + +class EstablishmentInfos(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + email = models.CharField(max_length=255, blank=True, null=True) + website = models.CharField(max_length=255, blank=True, null=True) + facebook = models.CharField(max_length=255, blank=True, null=True) + twitter = models.CharField(max_length=255, blank=True, null=True) + lafourchette = models.CharField(max_length=255, blank=True, null=True) + pub = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + booking_url = models.CharField(max_length=255, blank=True, null=True) + nb_slot = models.IntegerField(blank=True, null=True) + booking_enabled = models.IntegerField(blank=True, null=True) + guestonline_id = models.IntegerField(blank=True, null=True) + guestonline_auth_token = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishment_infos' + + +# class EstablishmentMerchandises(MigrateMixin): +# using = 'legacy' +# +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + + #TODO: модели Merchandises нету в гугл таблице Check Migrations + +# merchandise = models.ForeignKey('Merchandises', models.DO_NOTHING, blank=True, null=True) +# gifted = models.IntegerField(blank=True, null=True) +# quantity = models.IntegerField(blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'establishment_merchandises' + + +class Menus(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + name = models.CharField(max_length=255, blank=True, null=True) + timing = models.CharField(max_length=255, blank=True, null=True) + price = models.FloatField(blank=True, null=True) + currency = models.CharField(max_length=255, blank=True, null=True) + drinks = models.CharField(max_length=255, blank=True, null=True) + served_on_offdays = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'menus' From ea427d6451521a46081c0e65685a48380db790eb Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 3 Oct 2019 16:35:54 +0300 Subject: [PATCH 012/173] Update module --- apps/transfer/management/commands/transfer.py | 26 ++++++++++++------- apps/transfer/utils.py | 1 + 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 13cfbf90..79a0185e 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -9,20 +9,28 @@ from transfer.utils import transfer_objects class Command(BaseCommand): help = 'Transfer data between databases' + types = [ + 'dictionaries', + 'keys' + ] + def handle(self, *args, **options): + option = set(options.keys()) & set(self.types) + print(option) if options['dictionaries']: - self._transfer_dictionaries() + self._transfer_objects() def add_arguments(self, parser): - parser.add_argument( - '-d', - '--dictionaries', - action='store_true', - default=False, - help='Вывод короткого сообщения' - ) + for type in self.types: + parser.add_argument( + f'-{type[:1]}', + f'--{type}', + action='store_true', + default=False, + help=f'Transfer {type} objects' + ) - def _transfer_dictionaries(self): + def _transfer_objects(self): for app in apps.get_app_configs(): if exists(f"{app.path}/transfer.py"): card_module = SourceFileLoader("transfer", f"{app.path}/transfer.py").load_module() diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 639a2ccd..d31075bf 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -1,6 +1,7 @@ from collections import OrderedDict def transfer_objects(app, transfer_card): + print(app) for model, card in transfer_card.items(): print(model) print(f"\t{card}") From 673acb4bb9318ff2f12b2d25d4d5768efa78ccb7 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 3 Oct 2019 23:19:40 +0300 Subject: [PATCH 013/173] Add sort Move dependencies to model Add data_type to model --- apps/collection/transfer.py | 23 ++++---- apps/location/transfer.py | 4 ++ apps/transfer/management/commands/transfer.py | 36 +++++------- apps/transfer/utils.py | 56 +++++++++++++++++-- 4 files changed, 82 insertions(+), 37 deletions(-) diff --git a/apps/collection/transfer.py b/apps/collection/transfer.py index 2e7b1306..e0c091c1 100644 --- a/apps/collection/transfer.py +++ b/apps/collection/transfer.py @@ -9,21 +9,20 @@ field[1] - название поля в таблице legacy Опционально: field[2] - тип данных для преобразования """ - card = { "Collection": { - "dependencies": ("Country", ), - "fields": { - "Collections": { - # нету аналогов для полей start и end - "name": "title", - "slug": "slug", - } - }, - "relations": { - "country": "Country", + "dependencies": ("Country", ), + "fields": { + "Collections": { + # нету аналогов для полей start и end + "name": "title", + "slug": "slug", } }, + "relations": { + "country": "Country", + } + }, "Guide": { "dependencies": ("Collection", ), "fields": { @@ -39,4 +38,4 @@ card = { } } -used_apps = ("location", ) \ No newline at end of file +used_apps = ("location", ) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 9de7448f..f8fd5768 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -13,6 +13,7 @@ NOTE: среди legacy таблиц совпадение для таблицы card = { "Country": { + "data_type": "dictionaries", "dependencies": None, "fields": { "Cities": { @@ -22,6 +23,7 @@ card = { }, "Region": { + "data_type": "dictionaries", "dependencies": ("Country", "Region"), "fields": { # нету аналога для поля name @@ -36,6 +38,7 @@ card = { }, "City": { + "data_type": "dictionaries", "dependencies": ("Country", "Region"), "fields": { "Cities": { @@ -56,6 +59,7 @@ card = { "Address": { + "data_type": "dictionaries", "dependencies": ("City",), "fields": { # нету аналога для поля number diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 79a0185e..9b99ea6c 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -1,38 +1,32 @@ from django.core.management.base import BaseCommand -from django.apps import apps -from os.path import exists -from importlib.machinery import SourceFileLoader - from transfer.utils import transfer_objects class Command(BaseCommand): help = 'Transfer data between databases' - types = [ + """Типы данных для трансфера + ВНИМАНИЕ: первые буквы типов данных должны быть уникальны! + """ + DATA_TYPES = [ 'dictionaries', - 'keys' ] def handle(self, *args, **options): - option = set(options.keys()) & set(self.types) - print(option) - if options['dictionaries']: - self._transfer_objects() + """Находим включённую опцию путём пересечения множества типов данных и множества включённых опций""" + data_type = list(set(option for option in options.keys() if options[option]) & set(self.DATA_TYPES)) + if len(data_type) != 1: + print("You must set correct option!\r\nYou can get options list with \r\n\r\n\tmanage.py help transfer\r\n") + exit(1) + transfer_objects(data_type[0]) def add_arguments(self, parser): - for type in self.types: + """Добавляем опции к команде, основываясь на типах данных, определённых в DATA_TYPES""" + for option_type in self.DATA_TYPES: parser.add_argument( - f'-{type[:1]}', - f'--{type}', + f'-{option_type[:1]}', + f'--{option_type}', action='store_true', default=False, - help=f'Transfer {type} objects' + help=f'Transfer {option_type} objects' ) - - def _transfer_objects(self): - for app in apps.get_app_configs(): - if exists(f"{app.path}/transfer.py"): - card_module = SourceFileLoader("transfer", f"{app.path}/transfer.py").load_module() - transfer_objects(app, card_module.card) - diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index d31075bf..708b0151 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -1,7 +1,55 @@ +from os.path import exists +from importlib.machinery import SourceFileLoader +from django.apps import apps from collections import OrderedDict -def transfer_objects(app, transfer_card): - print(app) - for model, card in transfer_card.items(): + +def transfer_objects(data_type): + models_list = {} + + for app in apps.get_app_configs(): + if exists(f"{app.path}/transfer.py"): + card_module = SourceFileLoader("transfer", f"{app.path}/transfer.py").load_module() + if not hasattr(card_module, "card") or len(card_module.card) < 1: + continue + + for model, card in card_module.card.items(): + if "data_type" in card and data_type == card["data_type"]: + card['name'] = model + models_list[model] = card + + if len(models_list) < 1: + print(f"Models with data type {data_type} not found in structure") + exit(1) + + models_list = sort_by_dependencies(models_list) + + print(f"==========================\r\n{models_list}\r\n==========================\r\n") + + for model, card in models_list.items(): print(model) - print(f"\t{card}") + print(card) + + +def sort_by_dependencies(data): + """Сначала мы сортируем модели по зависимостям в обратном порядке, используя сортировку вставкой""" + result = [] + for model, card in data.items(): + if "dependencies" in card and isinstance(card['dependencies'], tuple): + for model_dependency in result: + if model_dependency in card['dependencies']: + result.insert(result.index(model_dependency), model) + break + + else: + result.append(model) + + """Затем мы создаём сортированный словарь из реверса получившегося результата""" + print(result) + + result = OrderedDict( + [(model, data[model]) for model in reversed(result)] + ) + + return result + From 90dd896f131294880fb32f8c6649da70f94f997f Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 4 Oct 2019 00:14:16 +0300 Subject: [PATCH 014/173] Load app models Load legacy models --- apps/transfer/utils.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 708b0151..700d3e3e 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,6 +2,7 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from collections import OrderedDict +import transfer.models as legacy_models def transfer_objects(data_type): @@ -15,7 +16,7 @@ def transfer_objects(data_type): for model, card in card_module.card.items(): if "data_type" in card and data_type == card["data_type"]: - card['name'] = model + card['app_label'] = app.label models_list[model] = card if len(models_list) < 1: @@ -24,14 +25,37 @@ def transfer_objects(data_type): models_list = sort_by_dependencies(models_list) - print(f"==========================\r\n{models_list}\r\n==========================\r\n") - for model, card in models_list.items(): - print(model) - print(card) + transfer_model(model, card) + + +def transfer_model(model, card): + print(f"=================================================\r\n" + f"MODEL: {model}\r\n" + f"=================================================") + try: + app_model = apps.get_model(app_label=card['app_label'], model_name=model) + except LookupError as e: + print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") + return + + for legacy_table, fields in card['fields'].items(): + try: + legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) + except LookupError as e: + print(f"ERROR: legacy model {model} can not be loaded: {e}") + return + + print(app_model) + + print(legacy_model) + + print(fields) def sort_by_dependencies(data): + """Сортировка моделей по зависимостям""" + """Сначала мы сортируем модели по зависимостям в обратном порядке, используя сортировку вставкой""" result = [] for model, card in data.items(): @@ -45,7 +69,6 @@ def sort_by_dependencies(data): result.append(model) """Затем мы создаём сортированный словарь из реверса получившегося результата""" - print(result) result = OrderedDict( [(model, data[model]) for model in reversed(result)] From b581ead8ae69038a5a192f003d98019699c77d96 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 4 Oct 2019 07:50:19 +0300 Subject: [PATCH 015/173] Add relation fields --- apps/location/transfer.py | 5 ++++- apps/transfer/utils.py | 46 ++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index f8fd5768..cbaeaa14 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -52,7 +52,10 @@ card = { }, }, "relations": { - "region": "Region", + "region": { + "Region", + ("region_code", ("region_id", "int")) + }, "country": "Country" } }, diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 700d3e3e..b6cca330 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -29,6 +29,30 @@ def transfer_objects(data_type): transfer_model(model, card) +def sort_by_dependencies(data): + """Сортировка моделей по зависимостям""" + + """Сначала мы сортируем модели по зависимостям в обратном порядке, используя сортировку вставкой""" + result = [] + for model, card in data.items(): + if "dependencies" in card and isinstance(card['dependencies'], tuple): + for model_dependency in result: + if model_dependency in card['dependencies']: + result.insert(result.index(model_dependency), model) + break + + else: + result.append(model) + + """Затем мы создаём сортированный словарь из реверса получившегося результата""" + + result = OrderedDict( + [(model, data[model]) for model in reversed(result)] + ) + + return result + + def transfer_model(model, card): print(f"=================================================\r\n" f"MODEL: {model}\r\n" @@ -53,26 +77,4 @@ def transfer_model(model, card): print(fields) -def sort_by_dependencies(data): - """Сортировка моделей по зависимостям""" - - """Сначала мы сортируем модели по зависимостям в обратном порядке, используя сортировку вставкой""" - result = [] - for model, card in data.items(): - if "dependencies" in card and isinstance(card['dependencies'], tuple): - for model_dependency in result: - if model_dependency in card['dependencies']: - result.insert(result.index(model_dependency), model) - break - - else: - result.append(model) - - """Затем мы создаём сортированный словарь из реверса получившегося результата""" - - result = OrderedDict( - [(model, data[model]) for model in reversed(result)] - ) - - return result From 4ce0eb284d1bbd4b8a986a5e9e30e3d033462031 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 4 Oct 2019 08:16:56 +0300 Subject: [PATCH 016/173] Update transfer structure in location --- apps/location/transfer.py | 27 +++++++++++++++++---------- apps/transfer/mixins.py | 4 +++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index cbaeaa14..9598246c 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -8,6 +8,20 @@ field[0] - название аргумента field[1] - название поля в таблице legacy Опционально: field[2] - тип данных для преобразования +Структура внешних ключей: +"legacy_table": [ + ("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type") +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) """ @@ -31,10 +45,7 @@ card = { "code": "region_code", }, }, - "relations": { - "parent_region": "Region", - "country": "Country" - } + "relations": {} }, "City": { @@ -52,11 +63,7 @@ card = { }, }, "relations": { - "region": { - "Region", - ("region_code", ("region_id", "int")) - }, - "country": "Country" + } }, @@ -72,7 +79,7 @@ card = { }, }, "relations": { - "city": "City" + } } diff --git a/apps/transfer/mixins.py b/apps/transfer/mixins.py index 88cfe091..a8e8287a 100644 --- a/apps/transfer/mixins.py +++ b/apps/transfer/mixins.py @@ -1,4 +1,5 @@ from django.db import models +from django.forms.models import model_to_dict class SecondDbManager(models.Manager): @@ -17,7 +18,8 @@ class MigrateMixin(models.Model): use_db = 'legacy' objects = SecondDbManager() - def _parse_instance_fields(self): + def _parse_instance_fields(self, fields): + model_dict = model_to_dict(self, fields) print(getattr(self, "using", "Using not found")) class Meta: From ad17a2ad7a4ddcc5eae3a3cd30008315f756d411 Mon Sep 17 00:00:00 2001 From: michail Date: Fri, 4 Oct 2019 13:18:43 +0500 Subject: [PATCH 017/173] new relations template --- apps/establishment/transfer.py | 1 + apps/location/transfer.py | 26 ++++++++++++++++++++++---- apps/transfer/models.py | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py index bf9aca39..fca4619c 100644 --- a/apps/establishment/transfer.py +++ b/apps/establishment/transfer.py @@ -33,6 +33,7 @@ card = { # "establishment_type": "EstablishmentType", # "establishment_subtypes": "EstablishmentSubType", "collections": "Collection", + # TODO: нашел schedules в legacy # "schedule": "Timetable", "award": "Award", # "tags": "MetaDataContent", diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 9598246c..64334b28 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -9,9 +9,10 @@ field[1] - название поля в таблице legacy Опционально: field[2] - тип данных для преобразования Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей "legacy_table": [ - ("legacy_key", "legacy_field"), - ("psql_table", "psql_key", "psql_field", "psql_field_type") + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) ], где: legacy_table - название модели legacy legacy_key - ForeignKey в legacy @@ -45,7 +46,15 @@ card = { "code": "region_code", }, }, - "relations": {} + "relations": { + "Cities": [( + (None, "country_code_2"), + ("Country", "country", "code", "CharField")), + ((None, "subregion_code"), #TODO: как сопоставлять parent_region из postgres с subregion_code из legacy ? + ("Region", "parent_region", "code", "CharField")) + + ] + } }, "City": { @@ -63,7 +72,12 @@ card = { }, }, "relations": { - + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] } }, @@ -79,6 +93,10 @@ card = { }, }, "relations": { + "Locations": [ #TODO: Locations ссылается внешним ключом на Cities + # (("city", "name"), + # ("City", "city", "name", "CharField")) + ] } } diff --git a/apps/transfer/models.py b/apps/transfer/models.py index a811c305..86d96532 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -407,3 +407,24 @@ class Menus(MigrateMixin): class Meta: managed = False db_table = 'menus' + + +class Schedules(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + continuous_service = models.CharField(max_length=255, blank=True, null=True) + open_august = models.CharField(max_length=255, blank=True, null=True) + lunch_start = models.TimeField(blank=True, null=True) + lunch_end = models.TimeField(blank=True, null=True) + diner_start = models.TimeField(blank=True, null=True) + diner_end = models.TimeField(blank=True, null=True) + opening_hours = models.CharField(max_length=255, blank=True, null=True) + opening_dates = models.CharField(max_length=255, blank=True, null=True) + timetable = models.CharField(max_length=10000, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'schedules' From 511312bb288ce6faa9914b90094748af341106d7 Mon Sep 17 00:00:00 2001 From: michail Date: Fri, 4 Oct 2019 16:20:04 +0500 Subject: [PATCH 018/173] added transfer.py to partner --- apps/collection/transfer.py | 6 +- apps/establishment/transfer.py | 26 +++---- apps/gallery/transfer.py | 26 +++++++ apps/location/transfer.py | 2 + apps/news/transfer.py | 58 ++++++++++++++ apps/notification/transfer.py | 46 +++++++++++ apps/partner/transfer.py | 40 ++++++++++ apps/transfer/models.py | 138 +++++++++++++++++++++++++++++++++ docker-compose.yml | 20 ++--- 9 files changed, 336 insertions(+), 26 deletions(-) create mode 100644 apps/gallery/transfer.py create mode 100644 apps/news/transfer.py create mode 100644 apps/notification/transfer.py create mode 100644 apps/partner/transfer.py diff --git a/apps/collection/transfer.py b/apps/collection/transfer.py index e0c091c1..aac4a6a2 100644 --- a/apps/collection/transfer.py +++ b/apps/collection/transfer.py @@ -20,7 +20,7 @@ card = { } }, "relations": { - "country": "Country", + # "country": "Country", } }, "Guide": { @@ -32,8 +32,8 @@ card = { } }, "relations": { - "parent": "Guide", - "collection": "Collection" + # "parent": "Guide", + # "collection": "Collection" } } } diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py index fca4619c..2305602d 100644 --- a/apps/establishment/transfer.py +++ b/apps/establishment/transfer.py @@ -29,17 +29,17 @@ card = { }, }, "relations": { - "address": "Address", - # "establishment_type": "EstablishmentType", - # "establishment_subtypes": "EstablishmentSubType", - "collections": "Collection", - # TODO: нашел schedules в legacy - # "schedule": "Timetable", - "award": "Award", - # "tags": "MetaDataContent", - "reviews": "Review", - # "comments": "Comment", - # "favorites": "Favorites" + # "address": "Address", + # # "establishment_type": "EstablishmentType", + # # "establishment_subtypes": "EstablishmentSubType", + # "collections": "Collection", + # # TODO: нашел schedules в legacy + # # "schedule": "Timetable", + # "award": "Award", + # # "tags": "MetaDataContent", + # "reviews": "Review", + # # "comments": "Comment", + # # "favorites": "Favorites" } }, @@ -51,7 +51,7 @@ card = { } }, "relations": { - "establishment": "Establishment" + # "establishment": "Establishment" } }, "ContactPhone": { @@ -61,7 +61,7 @@ card = { "phone": "phone" }, "relations": { - "establishment": "Establishment" + # "establishment": "Establishment" } } } diff --git a/apps/gallery/transfer.py b/apps/gallery/transfer.py new file mode 100644 index 00000000..50d99830 --- /dev/null +++ b/apps/gallery/transfer.py @@ -0,0 +1,26 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +""" + +card = { + "Image": { + "data_type": "objects", + "dependencies": None, + "fields": { + "MercuryImages": { + "image": "attachment_file_name" + } + } + }, + +} + +used_apps = None diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 64334b28..8cff5d55 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -102,3 +102,5 @@ card = { } } + +used_apps = None diff --git a/apps/news/transfer.py b/apps/news/transfer.py new file mode 100644 index 00000000..06ee549d --- /dev/null +++ b/apps/news/transfer.py @@ -0,0 +1,58 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +card = { + # нету аналога для NewsType + "News": { + "data_type": "objects", + "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), + "fields": { + "NewsOlds": { + "title": ("title", "TJSONField"), + + # в NewsOlds нету аналога для поля subtitle модели News, также нет аналогов для полей start, end, playlist + # "subtitle": "" + + + # Поле "description" модели News имеет тип JSONField(где ключ - это язык, а значение - новость + # на языке который указан ключом), а поле "body" NewsOlds имеет тип html-разметки + # с вставками шаблонизатора Ruby + # "description" : "body" + }, + "relations":[ + # "address": "", + # "country": "", + # "tags": "", + # "news_type": "" + # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) + ] + } + }, +} + +used_apps = ("location", "main", ) diff --git a/apps/notification/transfer.py b/apps/notification/transfer.py new file mode 100644 index 00000000..b2cb97cc --- /dev/null +++ b/apps/notification/transfer.py @@ -0,0 +1,46 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +card = { + # нету аналога для NewsType + "News": { + "data_type": "objects", + "dependencies": ("User", ), + "fields": { + "EmailAddresses": { + "email": "email", + }, + "relations": [ + # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) + # "user": "" + ] + } + }, +} + +used_apps = ("account", ) diff --git a/apps/partner/transfer.py b/apps/partner/transfer.py new file mode 100644 index 00000000..393a5a85 --- /dev/null +++ b/apps/partner/transfer.py @@ -0,0 +1,40 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "Partner": { + "data_type": "objects", + "dependencies": None, + "fields": { + "EstablishmentBacklinks": { + "url": "backlink_url", + }, + } + }, +} + +used_apps = None diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 86d96532..3bb3d90c 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -10,6 +10,87 @@ from django.contrib.gis.db import models from transfer.mixins import MigrateMixin +class Ezuser(MigrateMixin): + using = 'legacy' + + contentobject_id = models.IntegerField(primary_key=True) + email = models.CharField(max_length=150) + login = models.CharField(max_length=150) + password_hash = models.CharField(max_length=50, blank=True, null=True) + password_hash_type = models.IntegerField() + facebook_id = models.BigIntegerField() + #TODO: в legacy нету таблицы 'CadLevel' + + # level = models.ForeignKey('CadLevel', models.DO_NOTHING) + points = models.IntegerField() + publish_fb_activity = models.IntegerField() + + class Meta: + managed = False + db_table = 'ezuser' + + +class Accounts(MigrateMixin): + using = 'legacy' + + uuid = models.CharField(max_length=24) + nickname = models.CharField(max_length=128, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + city = models.CharField(max_length=32, blank=True, null=True) + role = models.CharField(max_length=16, blank=True, null=True) + consent_purpose = models.CharField(max_length=255, blank=True, null=True) + consent_at = models.DateTimeField(blank=True, null=True) + last_seen_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + email = models.CharField(unique=True, max_length=255) + is_admin = models.IntegerField(blank=True, null=True) + ezuser_id = models.IntegerField(blank=True, null=True) + ez_user_id = models.IntegerField(blank=True, null=True) + encrypted_password = models.CharField(max_length=255) + reset_password_token = models.CharField(unique=True, max_length=255, blank=True, null=True) + reset_password_sent_at = models.DateTimeField(blank=True, null=True) + remember_created_at = models.DateTimeField(blank=True, null=True) + sign_in_count = models.IntegerField() + current_sign_in_at = models.DateTimeField(blank=True, null=True) + last_sign_in_at = models.DateTimeField(blank=True, null=True) + current_sign_in_ip = models.CharField(max_length=255, blank=True, null=True) + last_sign_in_ip = models.CharField(max_length=255, blank=True, null=True) + confirmation_token = models.CharField(max_length=255, blank=True, null=True) + confirmed_at = models.DateTimeField(blank=True, null=True) + confirmation_sent_at = models.DateTimeField(blank=True, null=True) + unconfirmed_email = models.CharField(max_length=255, blank=True, null=True) + webpush_subscription = models.CharField(max_length=5000, blank=True, null=True) + + class Meta: + managed = False + db_table = 'accounts' + + +class Profiles(MigrateMixin): + using = 'legacy' + + firstname = models.CharField(max_length=255, blank=True, null=True) + lastname = models.CharField(max_length=255, blank=True, null=True) + gender = models.CharField(max_length=255, blank=True, null=True) + dob = models.DateField(blank=True, null=True) + email = models.CharField(max_length=255, blank=True, null=True) + phone = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + unique_key = models.CharField(max_length=255, blank=True, null=True) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + requester_id = models.IntegerField(blank=True, null=True) + available_for_events = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'profiles' + + class Cities(MigrateMixin): using = 'legacy' @@ -428,3 +509,60 @@ class Schedules(MigrateMixin): class Meta: managed = False db_table = 'schedules' + + +class MercuryImages(MigrateMixin): + using = 'legacy' + + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'mercury_images' + + +class NewsOlds(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + body = models.TextField(blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + template = models.CharField(max_length=255, blank=True, null=True) + account = models.ForeignKey("Accounts", models.DO_NOTHING, blank=True, null=True) + # site = models.ForeignKey('Sites', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + image_file_name = models.CharField(max_length=255, blank=True, null=True) + image_content_type = models.CharField(max_length=255, blank=True, null=True) + image_file_size = models.IntegerField(blank=True, null=True) + image_updated_at = models.DateTimeField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'news_olds' + + +class EmailAddresses(MigrateMixin): + using = 'legacy' + + account = models.ForeignKey("Accounts", models.DO_NOTHING, blank=True, null=True) + partner_notification = models.IntegerField(blank=True, null=True) + ip = models.CharField(max_length=255, blank=True, null=True) + country_code = models.CharField(max_length=3, blank=True, null=True) + city = models.CharField(max_length=255, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + email = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'email_addresses' + diff --git a/docker-compose.yml b/docker-compose.yml index 3fb05f98..2ccd6de3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,8 @@ services: - POSTGRES_DB=postgres ports: - "5436:5432" - networks: - - db-net +# networks: +# - db-net volumes: - gm-db:/var/lib/postgresql/data/ elasticsearch: @@ -28,8 +28,8 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - discovery.type=single-node - xpack.security.enabled=false - networks: - - app-net +# networks: +# - app-net # RabbitMQ rabbitmq: image: rabbitmq:latest @@ -83,18 +83,18 @@ services: - worker - worker_beat - elasticsearch - networks: - - app-net - - db-net +# networks: +# - app-net +# - db-net volumes: - .:/code - gm-media:/media-data ports: - "8000:8000" -networks: - app-net: - db-net: +#networks: +# app-net: +# db-net: volumes: gm-db: From 4e96d09513d4543c866d01186d2bb6b3e7b85840 Mon Sep 17 00:00:00 2001 From: michail Date: Mon, 7 Oct 2019 14:34:24 +0500 Subject: [PATCH 019/173] added transfer.py to account, comment and review --- apps/account/transfer.py | 49 ++++++++++++++++++++++++++++++ apps/comment/transfer.py | 47 +++++++++++++++++++++++++++++ apps/review/transfer.py | 58 ++++++++++++++++++++++++++++++++++++ apps/transfer/models.py | 64 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 apps/account/transfer.py create mode 100644 apps/comment/transfer.py create mode 100644 apps/review/transfer.py diff --git a/apps/account/transfer.py b/apps/account/transfer.py new file mode 100644 index 00000000..dc02c49c --- /dev/null +++ b/apps/account/transfer.py @@ -0,0 +1,49 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + +""" + +card = { + "User": { + "data_type": "objects", + "dependencies": None, + "fields": { + "Accounts": { + "username": "nickname", + "email": "email", + "email_confirmed": ("confirmed_at", "BooleanField") + }, + # Profiles ссылается внешним ключом(account) на Accounts + "Profiles": { + "first_name": "firstname", + "last_name": "lastname" + } + } + } + +} + +used_apps = None + + diff --git a/apps/comment/transfer.py b/apps/comment/transfer.py new file mode 100644 index 00000000..ef84bfe7 --- /dev/null +++ b/apps/comment/transfer.py @@ -0,0 +1,47 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + +""" + +card = { + "Comment": { + "data_type": "objects", + "dependencies": ("User",), + "fields": { + "text": "comment", + "mark": "mark" + # как работать с GenericForeignKey - ? + # "content_object" : "" + }, + "relations": { + # нужен ответ по вопросу Как поступать со сбором данных + # "Accounts": [ + # (("account_id", "account_id"), + # ("User", "id", "user", "PositiveIntegerField")) + # ] + } + } +} + +used_apps = ("account", ) \ No newline at end of file diff --git a/apps/review/transfer.py b/apps/review/transfer.py new file mode 100644 index 00000000..109d88be --- /dev/null +++ b/apps/review/transfer.py @@ -0,0 +1,58 @@ +""" +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "Review": { + "data_type": "objects", + "dependencies": ("User",), + "fields": { + "Reviews": { + "published_at": "published_at", + "vintage": "vintage", + # как работать с GenericForeignKey - ? + # "content_object": "" + }, + "ReviewTexts": { + # полу text в модели Review имеет тип TJSONField, а поле text в модели ReviewTexts имеет тип TextField + # при их сопоставлении использовать поле locale модели ReviewTexts + "text": "text" + } + }, + "relations": { + # вопрос с внешним ключом language + + # нужен ответ по вопросу как поступать со сбором данных + # "Accounts": [ + # (("account_id", "account_id"), + # ("User", "id", "reviewer", "PositiveIntegerField")) + # ] + } + } + +} + +used_apps = ("account", ) \ No newline at end of file diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 3bb3d90c..59f827c6 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -566,3 +566,67 @@ class EmailAddresses(MigrateMixin): managed = False db_table = 'email_addresses' + +class Reviews(MigrateMixin): + using = 'legacy' + + vintage = models.PositiveIntegerField() + mark = models.FloatField(blank=True, null=True) + favorite = models.IntegerField(blank=True, null=True) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + visited_at = models.DateField(blank=True, null=True) + created_at = models.DateTimeField() + published_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField() + aasm_state = models.CharField(max_length=255, blank=True, null=True) + reviewer_id = models.IntegerField() + priority = models.IntegerField(blank=True, null=True) + #TODO: модель Products в postgres закомментирована + product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) + received_at = models.DateTimeField(blank=True, null=True) + reviewer_name = models.CharField(max_length=255, blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + locked = models.IntegerField(blank=True, null=True) + temporary = models.IntegerField(blank=True, null=True) + last_state_change_at = models.DateTimeField(blank=True, null=True) + editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + + class Meta: + managed = False + db_table = 'reviews' + + +class ReviewTexts(MigrateMixin): + using = 'legacy' + + review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + text = models.TextField(blank=True, null=True) + updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'review_texts' + +class Comments(MigrateMixin): + using = 'legacy' + + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + establishment = models.ForeignKey('Establishments', models.DO_NOTHING) + parent_id = models.IntegerField(blank=True, null=True) + main_parent_id = models.IntegerField(blank=True, null=True) + comment = models.TextField(blank=True, null=True) + mark = models.DecimalField(max_digits=4, decimal_places=2, blank=True, null=True) + locale = models.CharField(max_length=5) + ip = models.CharField(max_length=40) + state = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + date = models.DateField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'comments' \ No newline at end of file From ab768311f02b03f9d09b47e3b5864b56eaa37dc1 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 7 Oct 2019 22:23:01 +0300 Subject: [PATCH 020/173] Update tables structure --- apps/location/transfer.py | 44 ++++++++++++++++++-- apps/transfer/models.py | 84 +++++++++++++++++++-------------------- apps/transfer/utils.py | 11 +++-- 3 files changed, 88 insertions(+), 51 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 8cff5d55..8d94486b 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -1,4 +1,37 @@ """ + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + Структура fields: key - поле в таблице postgres value - поле или группа полей в таблице legacy @@ -67,9 +100,14 @@ card = { "postal_code": "zip_code", "is_island": ("is_island", "Boolean") }, - "CityNames": { - "name": "name", - }, + "relations": { + "CityNames": { + "key": "city", + "fields": { + "name": "name", + } + } + } }, "relations": { "Cities": [( diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 59f827c6..024c9596 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -567,49 +567,49 @@ class EmailAddresses(MigrateMixin): db_table = 'email_addresses' -class Reviews(MigrateMixin): - using = 'legacy' +# class Reviews(MigrateMixin): +# using = 'legacy' +# +# vintage = models.PositiveIntegerField() +# mark = models.FloatField(blank=True, null=True) +# favorite = models.IntegerField(blank=True, null=True) +# account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) +# establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) +# visited_at = models.DateField(blank=True, null=True) +# created_at = models.DateTimeField() +# published_at = models.DateTimeField(blank=True, null=True) +# updated_at = models.DateTimeField() +# aasm_state = models.CharField(max_length=255, blank=True, null=True) +# reviewer_id = models.IntegerField() +# priority = models.IntegerField(blank=True, null=True) +# #TODO: модель Products в postgres закомментирована +# product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) +# received_at = models.DateTimeField(blank=True, null=True) +# reviewer_name = models.CharField(max_length=255, blank=True, null=True) +# type = models.CharField(max_length=255, blank=True, null=True) +# locked = models.IntegerField(blank=True, null=True) +# temporary = models.IntegerField(blank=True, null=True) +# last_state_change_at = models.DateTimeField(blank=True, null=True) +# editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) +# +# class Meta: +# managed = False +# db_table = 'reviews' +# - vintage = models.PositiveIntegerField() - mark = models.FloatField(blank=True, null=True) - favorite = models.IntegerField(blank=True, null=True) - account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) - establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) - visited_at = models.DateField(blank=True, null=True) - created_at = models.DateTimeField() - published_at = models.DateTimeField(blank=True, null=True) - updated_at = models.DateTimeField() - aasm_state = models.CharField(max_length=255, blank=True, null=True) - reviewer_id = models.IntegerField() - priority = models.IntegerField(blank=True, null=True) - #TODO: модель Products в postgres закомментирована - product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) - received_at = models.DateTimeField(blank=True, null=True) - reviewer_name = models.CharField(max_length=255, blank=True, null=True) - type = models.CharField(max_length=255, blank=True, null=True) - locked = models.IntegerField(blank=True, null=True) - temporary = models.IntegerField(blank=True, null=True) - last_state_change_at = models.DateTimeField(blank=True, null=True) - editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) - - class Meta: - managed = False - db_table = 'reviews' - - -class ReviewTexts(MigrateMixin): - using = 'legacy' - - review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) - locale = models.CharField(max_length=5, blank=True, null=True) - text = models.TextField(blank=True, null=True) - updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) - created_at = models.DateTimeField() - updated_at = models.DateTimeField() - - class Meta: - managed = False - db_table = 'review_texts' +# class ReviewTexts(MigrateMixin): +# using = 'legacy' +# +# review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) +# locale = models.CharField(max_length=5, blank=True, null=True) +# text = models.TextField(blank=True, null=True) +# updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'review_texts' class Comments(MigrateMixin): using = 'legacy' diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index b6cca330..8aa4be63 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -63,6 +63,8 @@ def transfer_model(model, card): print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") return + querysets = {} + for legacy_table, fields in card['fields'].items(): try: legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) @@ -70,11 +72,8 @@ def transfer_model(model, card): print(f"ERROR: legacy model {model} can not be loaded: {e}") return - print(app_model) - - print(legacy_model) - - print(fields) - + print(f"APP MODEL: {app_model}") + print(f"LEGACY MODEL: {legacy_model}") + print(f"FIELDS: {fields}") From bc7d4ef5a55b18575b57522da195cc1e4ee2cbb8 Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 8 Oct 2019 12:24:57 +0500 Subject: [PATCH 021/173] comment location/transfer.py --- apps/location/transfer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 8d94486b..11a93dd0 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -11,6 +11,7 @@ }, "relations": список зависимостей legacy-таблицы, пример: { + # имеет внешний ключ на "название legacy таблицы" из "fields" "название legacy таблицы": { "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля в дочерней таблице и названия поля в родительской таблице в ином случае From 03b22d79dba654178f6350a1f00f0c3855ef349c Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 8 Oct 2019 14:01:40 +0500 Subject: [PATCH 022/173] edited Address in location/transfer.py --- apps/location/transfer.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 11a93dd0..d44251b0 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -84,8 +84,8 @@ card = { "Cities": [( (None, "country_code_2"), ("Country", "country", "code", "CharField")), - ((None, "subregion_code"), #TODO: как сопоставлять parent_region из postgres с subregion_code из legacy ? - ("Region", "parent_region", "code", "CharField")) + # ((None, "subregion_code"), #TODO: как сопоставлять parent_region из postgres с subregion_code из legacy ? + # ("Region", "parent_region", "code", "CharField")) ] } @@ -132,14 +132,13 @@ card = { }, }, "relations": { - "Locations": [ #TODO: Locations ссылается внешним ключом на Cities - # (("city", "name"), - # ("City", "city", "name", "CharField")) + "Cities": [ #TODO: Locations ссылается внешним ключом на Cities + (("city_id", "id"), + ("City", "city", "id", "PositiveIntegerField")) ] - } } - } + used_apps = None From e8c48e99c65079ac3e607e9838711ad6830dacbe Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 8 Oct 2019 22:02:08 +0300 Subject: [PATCH 023/173] Add key to transfer Add queryset to transfer Add models and fields get to transfer --- apps/location/transfer.py | 2 +- apps/transfer/utils.py | 51 +++++++++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index d44251b0..3c9900f4 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -99,7 +99,7 @@ card = { "coordinates": (("lat", "latitude"), ("long", "longitude")), "code": "country_code_2", "postal_code": "zip_code", - "is_island": ("is_island", "Boolean") + "is_island": ("is_island", "is_island", "Boolean") }, "relations": { "CityNames": { diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 8aa4be63..bf1c7395 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -63,17 +63,48 @@ def transfer_model(model, card): print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") return - querysets = {} + card_fields = card['fields'] + relations = card_fields['relations'] if 'relations' in card_fields else None - for legacy_table, fields in card['fields'].items(): - try: - legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) - except LookupError as e: - print(f"ERROR: legacy model {model} can not be loaded: {e}") - return + if relations: + del(card_fields['relations']) - print(f"APP MODEL: {app_model}") + if len(card_fields) > 1: + print("You must set one table to transfer") + return - print(f"LEGACY MODEL: {legacy_model}") + legacy_table = list(card_fields.keys())[0] + fields = list(card_fields.values())[0] - print(f"FIELDS: {fields}") + try: + legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) + except LookupError as e: + print(f"ERROR: legacy model {model} can not be loaded: {e}") + return + + legacy_fields = [] + + for legacy_field in fields.values(): + if isinstance(legacy_field, str): + legacy_fields.append(legacy_field) + + elif isinstance(legacy_field, tuple): + """Группа полей""" + if isinstance(legacy_field[0], tuple): + for legacy_field_group in legacy_field: + legacy_fields.append(legacy_field_group[1]) + else: + legacy_fields.append(legacy_field[1]) + + if model != "City": + return + + print(legacy_fields) + + queryset = legacy_model.objects.only(*legacy_fields) + print(queryset) + + if relations: + for relation_table, relation_data in relations.items(): + print(relation_table) + print(relation_data) From 4cc9fabd7385112e7ae7566e3d72e9e64faf4d27 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 9 Oct 2019 17:43:48 +0500 Subject: [PATCH 024/173] edited transfer.py for notification, account, establishment --- apps/account/transfer.py | 12 ++++-- apps/establishment/transfer.py | 78 ++++++++++++++++++++++++++++------ apps/notification/transfer.py | 14 +++--- 3 files changed, 80 insertions(+), 24 deletions(-) diff --git a/apps/account/transfer.py b/apps/account/transfer.py index dc02c49c..964705e5 100644 --- a/apps/account/transfer.py +++ b/apps/account/transfer.py @@ -34,10 +34,14 @@ card = { "email": "email", "email_confirmed": ("confirmed_at", "BooleanField") }, - # Profiles ссылается внешним ключом(account) на Accounts - "Profiles": { - "first_name": "firstname", - "last_name": "lastname" + "relations": { + "Profiles": { + "key": "account", + "fields": { + "first_name": "firstname", + "last_name": "lastname" + } + } } } } diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py index 2305602d..7e9fe57a 100644 --- a/apps/establishment/transfer.py +++ b/apps/establishment/transfer.py @@ -12,25 +12,51 @@ field[1] - название поля в таблице legacy card = { + "EstablishmentType": { + "data_type": "dictionaries", + "dependencies": None, + "fields": { + "Establishments": { + # значения для поля "name" берутся из поля "type" legacy модели "establishments" + # https://jira.spider.ru/browse/GM-199 + "name": ("name", "TJSONField") + } + } + }, "Establishment": { - "dependencies": ("Address", "Collection", ), + "data_type": "dictionaries", + "dependencies": ("Address", "Collection", "EstablishmentType"), # нету аналогов для establishment_type, establishment_subtypes, schedule, comments, tags "fields": { "Establishments": { "name": "name", - "slug": "slug" + "slug": "slug", + "is_publish": ("state", "BooleanField") }, - "EstablishmentInfos": { - "website": "website", - "facebook": "facebook", - "twitter": "twitter", - "lafourchette": "lafourchette", - "booking": "booking_url" + "relations": { + "EstablishmentInfos": { + "key": "establishment", + "fields": { + "website": "website", + "facebook": "facebook", + "twitter": "twitter", + "lafourchette": "lafourchette", + "booking": "booking_url" + } + }, }, }, + # как работать с GenericRelation - ? + # как работать с ManyToManyField - ? "EstablishmentSubType", "schedule" "relations": { - # "address": "Address", - # # "establishment_type": "EstablishmentType", + "Locations": [ + (("location_id", "id"), + ("Address", "address", "id", "PositiveIntegerField")), + ], + "Establishment": [#TODO правильно ли заполнена связь с EstablishmentType - ? + (("type", "type"), + ("EstablishmentType", "establishment_type", "id", "PositiveIntegerField")) + ] # # "establishment_subtypes": "EstablishmentSubType", # "collections": "Collection", # # TODO: нашел schedules в legacy @@ -39,11 +65,12 @@ card = { # # "tags": "MetaDataContent", # "reviews": "Review", # # "comments": "Comment", - # # "favorites": "Favorites" + # # "favorites": "Favorites" # В legacy этой таблицы не было } }, "Menu": { + "data_type": "objects", "dependencies": ("Establishment", ), "fields": { "Menus": { @@ -51,19 +78,42 @@ card = { } }, "relations": { - # "establishment": "Establishment" + "Establishments": [ + (("establishment_id", "id"), + ("Establishment", "establishment", "id", "PositiveIntegerField")) + ] } }, "ContactPhone": { + "data_type": "objects", "dependencies": ("Establishment", ), "fields": { "Establishments": { "phone": "phone" }, - "relations": { - # "establishment": "Establishment" + "relations": { #TODO правильно ли заполнена связь с Establishment - ? + "Establishments": [ + (("id", "id"), + ("Establishment", "establishment", "id", "PositiveIntegerField")) + ] } } + }, + + "ContactEmail": { + "data_type": "objects", + "dependencies": ("Establishment", ), + "fields": { + "EstablishmentInfos": { + "email": "email" + }, + }, + "relations": { + "Establishments": [ + (("establishment_id", "id"), + ("Establishment", "establishment", "id", "PositiveIntegerField")) + ] + } } } diff --git a/apps/notification/transfer.py b/apps/notification/transfer.py index b2cb97cc..1feb4385 100644 --- a/apps/notification/transfer.py +++ b/apps/notification/transfer.py @@ -28,17 +28,19 @@ NOTE: среди legacy таблиц совпадение для таблицы card = { # нету аналога для NewsType - "News": { + "Subscriber": { "data_type": "objects", - "dependencies": ("User", ), + # "dependencies": ("User", ), + "dependencies": None, "fields": { "EmailAddresses": { "email": "email", + "state": ("state", "PositiveIntegerField") # из legacy брать только те записи у которых state=usable }, - "relations": [ - # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) - # "user": "" - ] + # "relations": [ + # # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) + # # "user": "" + # ] } }, } From b2b8798c6c6f9aca119ef4cf008534b5ffda1ae7 Mon Sep 17 00:00:00 2001 From: michail Date: Thu, 10 Oct 2019 17:46:52 +0500 Subject: [PATCH 025/173] Edited transfer.py for news, partner, reviews, comment. Added transfer.py for recipe. --- apps/comment/transfer.py | 20 +++++----- apps/news/transfer.py | 43 ++++++++++++++++---- apps/partner/transfer.py | 1 + apps/recipe/transfer.py | 86 ++++++++++++++++++++++++++++++++++++++++ apps/review/transfer.py | 36 ++++++++++------- 5 files changed, 154 insertions(+), 32 deletions(-) create mode 100644 apps/recipe/transfer.py diff --git a/apps/comment/transfer.py b/apps/comment/transfer.py index ef84bfe7..51eeed38 100644 --- a/apps/comment/transfer.py +++ b/apps/comment/transfer.py @@ -25,21 +25,23 @@ psql_field_type - тип уникального поля в postgresql """ card = { + # как работать с GenericForeignKey(content_type) - ? "Comment": { "data_type": "objects", "dependencies": ("User",), "fields": { - "text": "comment", - "mark": "mark" - # как работать с GenericForeignKey - ? - # "content_object" : "" + "Comments": { + "text": "comment", + "mark": ("mark", "PositiveIntegerField") + # как работать с GenericForeignKey - ? + # "content_object" : "" + }, }, "relations": { - # нужен ответ по вопросу Как поступать со сбором данных - # "Accounts": [ - # (("account_id", "account_id"), - # ("User", "id", "user", "PositiveIntegerField")) - # ] + "Accounts": [ + (("account", None), + ("User", "user", None, None)) + ] } } } diff --git a/apps/news/transfer.py b/apps/news/transfer.py index 06ee549d..4f28d5a4 100644 --- a/apps/news/transfer.py +++ b/apps/news/transfer.py @@ -28,12 +28,29 @@ NOTE: среди legacy таблиц совпадение для таблицы card = { # нету аналога для NewsType + "NewsType": { + "data_type": "dictionaries", + "dependencies": None, + "fields": { + "Pages": { + # будет только один тип новости "News" + # значения для поля "name" берутся из поля "type" legacy модели "Pages", притом type="News" + # Mysql - select distinct(type) from pages; + "name": "name" + } + } + }, + "News": { "data_type": "objects", "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), "fields": { - "NewsOlds": { - "title": ("title", "TJSONField"), + # нету аналогов для start, end, playlist + "Pages": { + "state": ("state", "PositiveSmallIntegerField"), + "template": ("template", "PositiveIntegerField"), + "image_url": ("attachment_file_name", "URLField"), + "preview_image_url": ("attachment_file_name", "URLField"), # в NewsOlds нету аналога для поля subtitle модели News, также нет аналогов для полей start, end, playlist # "subtitle": "" @@ -44,12 +61,22 @@ card = { # с вставками шаблонизатора Ruby # "description" : "body" }, - "relations":[ - # "address": "", - # "country": "", - # "tags": "", - # "news_type": "" - # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) + "relations": { + "PageTexts": { + "key": "page_id", + "fields": { + "title": ("title", "TJSONField"), + "description": ("body", "TJSONField"), + "slug": ("slug", "SlugField") + } + } + }, + }, + # нету аналога для поля tags + "relations": { + "Pages": [ + ((None, "type"), + ("NewsType", "news_type", "name", "CharField")) ] } }, diff --git a/apps/partner/transfer.py b/apps/partner/transfer.py index 393a5a85..f89c7970 100644 --- a/apps/partner/transfer.py +++ b/apps/partner/transfer.py @@ -30,6 +30,7 @@ card = { "data_type": "objects", "dependencies": None, "fields": { + # из EstablishmentBacklinks выбирать только те у которых type="Partner" "EstablishmentBacklinks": { "url": "backlink_url", }, diff --git a/apps/recipe/transfer.py b/apps/recipe/transfer.py new file mode 100644 index 00000000..f631980d --- /dev/null +++ b/apps/recipe/transfer.py @@ -0,0 +1,86 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) +""" + +# значения берутся из legacy модели "Pages", притом type="Recipe" +card = { + "Recipe": { + "data_type": "objects", + "dependencies": None, + "fields": { + "Pages": { + "state": ("state", "PositiveSmallIntegerField") + }, + "relations": { + "PageTexts": { + "key": "page_id", + "fields": { + "title": ("title", "TJSONField"), + "description": ("body", "TJSONField") + } + } + } + } + } +} + + +used_apps = None diff --git a/apps/review/transfer.py b/apps/review/transfer.py index 109d88be..63f6f925 100644 --- a/apps/review/transfer.py +++ b/apps/review/transfer.py @@ -26,33 +26,39 @@ psql_field_type - тип уникального поля в postgresql """ card = { + # как работать с GenericForeignKey(content_type) - ? + # как работать с ForeignKey на самого себя(self), поле "child" + # вопрос с внешним ключом language на таблицу Language "Review": { "data_type": "objects", - "dependencies": ("User",), + "dependencies": ("User", "Language", "Review"), "fields": { "Reviews": { "published_at": "published_at", "vintage": "vintage", - # как работать с GenericForeignKey - ? + "status": ("aasm_state", "PositiveSmallIntegerField") + # "content_object": "" }, - "ReviewTexts": { - # полу text в модели Review имеет тип TJSONField, а поле text в модели ReviewTexts имеет тип TextField - # при их сопоставлении использовать поле locale модели ReviewTexts - "text": "text" - } + "relations": { + "ReviewTexts": { + "key": "review", + "fields": { + # полу text в модели Review имеет тип TJSONField, а поле text в модели ReviewTexts имеет тип TextField + # при их сопоставлении использовать поле locale модели ReviewTexts + "text": ("text", "TJSONField") + } + } + }, }, "relations": { - # вопрос с внешним ключом language - - # нужен ответ по вопросу как поступать со сбором данных - # "Accounts": [ - # (("account_id", "account_id"), - # ("User", "id", "reviewer", "PositiveIntegerField")) - # ] + "Accounts": [ + (("account", None), + ("User", "reviewer", None, None)) + ] } } } -used_apps = ("account", ) \ No newline at end of file +used_apps = ("account", "translation") \ No newline at end of file From 839491c7401ecc706f022660b2a15d255cae1223 Mon Sep 17 00:00:00 2001 From: michail Date: Thu, 10 Oct 2019 17:55:28 +0500 Subject: [PATCH 026/173] added Pages, PageTexts, PageCounters and PageMetadata models into transfer/models.py --- apps/transfer/models.py | 75 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 024c9596..2789427e 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -629,4 +629,77 @@ class Comments(MigrateMixin): class Meta: managed = False - db_table = 'comments' \ No newline at end of file + db_table = 'comments' + + +class Pages(MigrateMixin): + using = 'legacy' + + root_title = models.CharField(max_length=255, blank=True, null=True) + site_id = models.IntegerField() + account_id = models.IntegerField(blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + template = models.CharField(max_length=255, blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + scheduled_at = models.DateTimeField(blank=True, null=True) + created_at = models.DateTimeField() + published_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField() + type = models.CharField(max_length=255, blank=True, null=True) + is_main = models.IntegerField(blank=True, null=True) + related_pages_id = models.TextField(blank=True, null=True) + config = models.CharField(max_length=1000, blank=True, null=True) + + class Meta: + managed = False + db_table = 'pages' + + +class PageTexts(MigrateMixin): + using = 'legacy' + + title = models.CharField(max_length=255, blank=True, null=True) + slug = models.CharField(max_length=255, blank=True, null=True) + body = models.TextField(blank=True, null=True) + locale = models.CharField(max_length=255, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + page_id = models.IntegerField() + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + summary = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'page_texts' + + +class PageCounters(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + count = models.IntegerField(blank=True, null=True) + page = models.ForeignKey('Pages', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'page_counters' + + +class PageMetadata(MigrateMixin): + using = 'legacy' + + key = models.CharField(max_length=255, blank=True, null=True) + value = models.CharField(max_length=255, blank=True, null=True) + page_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'page_metadata' From f5c16fba9dc69f099848b22c0dddc48b766d1408 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 11 Oct 2019 10:45:49 +0300 Subject: [PATCH 027/173] Tmp fix --- apps/location/transfer.py | 6 +++ apps/transfer/utils.py | 85 +++++++++++++++++++++++++++++++-------- project/settings/base.py | 2 +- 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 3c9900f4..5bdc6303 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -107,6 +107,12 @@ card = { "fields": { "name": "name", } + }, + "CityPhotos": { + "key": ("city_id", "id"), + "fields": { + "coordinates": "geometries" + } } } }, diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index bf1c7395..55783e9c 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,8 +2,12 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from collections import OrderedDict +from django.db.models import OuterRef, Subquery + import transfer.models as legacy_models +# from pprint import pprint as print + def transfer_objects(data_type): models_list = {} @@ -25,8 +29,10 @@ def transfer_objects(data_type): models_list = sort_by_dependencies(models_list) + legacy_objects = OrderedDict() + for model, card in models_list.items(): - transfer_model(model, card) + legacy_objects[model] = get_model_data(model, card) def sort_by_dependencies(data): @@ -53,15 +59,10 @@ def sort_by_dependencies(data): return result -def transfer_model(model, card): - print(f"=================================================\r\n" - f"MODEL: {model}\r\n" - f"=================================================") - try: - app_model = apps.get_model(app_label=card['app_label'], model_name=model) - except LookupError as e: - print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") - return +def get_model_data(model, card): + # print(f"=================================================\r\n" + # f"MODEL: {model}\r\n" + # f"=================================================") card_fields = card['fields'] relations = card_fields['relations'] if 'relations' in card_fields else None @@ -89,22 +90,72 @@ def transfer_model(model, card): legacy_fields.append(legacy_field) elif isinstance(legacy_field, tuple): - """Группа полей""" if isinstance(legacy_field[0], tuple): + """Группа полей""" for legacy_field_group in legacy_field: legacy_fields.append(legacy_field_group[1]) else: + """Одиночное поле""" legacy_fields.append(legacy_field[1]) + #TODO: убрать после совместимости if model != "City": return - print(legacy_fields) - queryset = legacy_model.objects.only(*legacy_fields) - print(queryset) + """Проверяем наличие внешних связей""" if relations: - for relation_table, relation_data in relations.items(): - print(relation_table) - print(relation_data) + queryset = get_relation_data(relations, queryset, fields) + + return get_transfer_data(model, card, queryset) + + +def get_relation_data(relations, queryset, fields): + + # related_legacy_models = [x.related_model._meta.model.__name__ for x in legacy_model._meta.related_objects] + + """Получаем названия внешних зависимостей""" + for relation_table, relation_data in relations.items(): + """Если объекты связаны через ForeignKey, то ничего не надо делать - все аннотации уже подключены""" + + if isinstance(relation_data['key'], tuple): + """Вытаскиваем relation по id""" + try: + child_legacy_model = apps.get_model(app_label="transfer", model_name=relation_table) + except LookupError as e: + raise LookupError(f"ERROR: legacy model {relation_table} can not be loaded: {e}") + + child_table_queryset = child_legacy_model.objects.filter( + city_id=OuterRef('pk') + ).only(*relation_data['fields'].values()) + + queryset = queryset.annotate(**{ + f"{relation_table.lower()}": + Subquery(child_table_queryset) + }) + + return queryset + + +def get_transfer_data(model, card, queryset): + from pprint import pprint as print + + try: + app_model = apps.get_model(app_label=card['app_label'], model_name=model) + except LookupError as e: + print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") + return + + if "fields" not in card or not isinstance(card['fields'], dict) or len(card['fields']) != 1: + raise ValueError("You must set correct fields to transfer") + + for app_field, legacy_field_data in card['fields'].items(): + print(app_field) + print(legacy_field_data) + + print("*************************************************************************") + # print(app_model) + # print("*************************************************************************") + # print(queryset) + return queryset diff --git a/project/settings/base.py b/project/settings/base.py index d717579d..509b9b14 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -152,7 +152,7 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.19.0.1', + 'HOST': '172.23.0.1', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From 7bcd2edab60516cc240904855458ef0a1ace4357 Mon Sep 17 00:00:00 2001 From: michail Date: Fri, 11 Oct 2019 15:41:09 +0500 Subject: [PATCH 028/173] Added transfer.py into timetable. Edited transfer.py for collection and location. --- apps/collection/transfer.py | 11 +++++- apps/location/transfer.py | 5 +++ apps/timetable/transfer.py | 78 +++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 apps/timetable/transfer.py diff --git a/apps/collection/transfer.py b/apps/collection/transfer.py index aac4a6a2..0d276030 100644 --- a/apps/collection/transfer.py +++ b/apps/collection/transfer.py @@ -11,12 +11,16 @@ field[1] - название поля в таблице legacy """ card = { "Collection": { + "data_type": "objects", "dependencies": ("Country", ), "fields": { "Collections": { - # нету аналогов для полей start и end + # нету аналогов для полей description, start и end "name": "title", "slug": "slug", + "block_size": ("geometries", "JSONField"), + "is_publish": ("active", "BooleanField"), + "image_url": ("attachment_file_name", "URLField") } }, "relations": { @@ -24,7 +28,9 @@ card = { } }, "Guide": { - "dependencies": ("Collection", ), + # как работать с ForeignKey на самого себя(self), поле "parent" + "data_type": "objects", + "dependencies": ("Collection", "self"), "fields": { "Guides": { # нету аналогов для полей start и end @@ -32,6 +38,7 @@ card = { } }, "relations": { + # аналалог для поля "collection" не найдено # "parent": "Guide", # "collection": "Collection" } diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 3c9900f4..517db183 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -72,6 +72,11 @@ card = { }, "Region": { + # subregion_code -> code + # region_code -> parent_region + # MySQL: select distinct(country_code_2) from cities where subregion_code is not NULL; + # только у пяти стран есть понятие "subregion_code"(не NULL): fr, be, ma, aa, gr (country_code_2) + # Возможно получиться обойтись без изменений модели Region "data_type": "dictionaries", "dependencies": ("Country", "Region"), "fields": { diff --git a/apps/timetable/transfer.py b/apps/timetable/transfer.py new file mode 100644 index 00000000..915b9ad5 --- /dev/null +++ b/apps/timetable/transfer.py @@ -0,0 +1,78 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "Timetable": { + "data_type": "objects", + "dependencies": None, + "fields": { + "Schedules": { + # нет аналогов для weekday, opening_at, closed_at + "lunch_start": "lunch_start", + "lunch_end": "lunch_end", + "diner_start": "diner_start", + "diner_end": "diner_end" + } + } + } +} + +used_apps = None \ No newline at end of file From 8ca27c7f87f6c5fca59994b724e21c4269fe076a Mon Sep 17 00:00:00 2001 From: michail Date: Mon, 14 Oct 2019 14:30:13 +0500 Subject: [PATCH 029/173] Added transfer.py into main app. Added legacy models into transfer/models.py --- apps/main/transfer.py | 146 ++++++++++++++++++++++++++++++++++++++++ apps/transfer/models.py | 78 +++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 apps/main/transfer.py diff --git a/apps/main/transfer.py b/apps/main/transfer.py new file mode 100644 index 00000000..0d22ef72 --- /dev/null +++ b/apps/main/transfer.py @@ -0,0 +1,146 @@ +""" + +Структура записи в card: +Название таблицы в postgresql: { + "data_type": "тип данных в таблице (словарь, объект, дочерний объект и так далее)", + "dependencies": кортеж с зависимостями от других таблиц в postgresql, + "fields": список полей для таблицы postgresql, пример: + { + "название legacy таблицы": { + список полей для переноса, пример структуры описан далее + }, + "relations": список зависимостей legacy-таблицы, пример: + { + # имеет внешний ключ на "название legacy таблицы" из "fields" + "название legacy таблицы": { + "key": ключ для связи. Строка, если тип поля в legacy таблице - ForeignKey, или кортеж из названия поля + в дочерней таблице и названия поля в родительской таблице в ином случае + "fields": { + список полей для переноса, пример структуры описан далее + } + } + } + }, + "relations": список внешних ключей таблицы postgresql, пример структуры описан далее + { + "Cities": [( + (None, "region_code"), + ("Region", "region", "code", "CharField")), + ((None, "country_code_2"), + ("Country", "country", "code", "CharField")) + ] + } +}, + + +Структура fields: +key - поле в таблице postgres +value - поле или группа полей в таблице legacy + +В случае передачи группы полей каждое поле представляет собой кортеж, где: +field[0] - название аргумента +field[1] - название поля в таблице legacy +Опционально: field[2] - тип данных для преобразования + +Структура внешних ключей: +"legacy_table" - спикок кортежей для сопоставления полей +"legacy_table": [ + (("legacy_key", "legacy_field"), + ("psql_table", "psql_key", "psql_field", "psql_field_type")) +], где: +legacy_table - название модели legacy +legacy_key - ForeignKey в legacy +legacy_field - уникальное поле в модели legacy для сопоставления с postgresql +psql_table - название модели psql +psql_key - ForeignKey в postgresql +psql_field - уникальное поле в модели postgresql для сопоставления с legacy +psql_field_type - тип уникального поля в postgresql + + +""" + +card = { + "SiteSettings": { + "data_type": "objects", + "dependencies": ("Country",), + "fields": { + "SiteSettings": { + "subdomain": "country_code_2", + "pinterest_page_url": "pinterest_page_url", + "twitter_page_url": "twitter_page_url", + "facebook_page_url": "facebook_page_url", + "instagram_page_url": "instagram_page_url", + "contact_email": "contact_email", + "config": ("config", "JSONField") #TODO в качесте ключа использовать country_code_2 из legacy - ? + }, + # "relations": { + # # Как работать c отношение OneToOneField + # } + } + + }, + "Feature": { + "data_type": "objects", + "dependencies": ("SiteSettings",), + "fields": { + "Features": { + "slug": "slug" + } + }, + # поле "site_settings" ManyToManyField имеет through='SiteFeature' в postgres + # "relations": { # как работать с ManyToManyField + # + # } + }, + "SiteFeature": { + "data_type": "objects", + "dependencies": ("SiteSettings", "Feature"), + "fields": { + "SiteFeatures": { + "published": ("state", "BooleanField") + } + }, + "relations": { + "Sites": [( + ("site", None), + ("SiteSettings", "site_settings", None, None)) + ], + "Features": [( + ("feature", None), + ("Feature", "feature", None, None)) + ] + } + }, + "AwardType": { + "data_type": "objects", + "dependencies": ("Country",), + "fields": { + "AwardTypes": { + "name": "title" + } + }, + # вопрос с ForeignKey на Country + # "relations": { + # + # } + }, + "Award": { + "data_type": "objects", + "dependencies": ("AwardType", "ContentType"), + "fields": { + "Awards": { + "title": ("title", "TJSONField"), + "vintage_year": "year", + } + }, + #TODO вопрос с content_type + "relations": { + "AwardTypes": [( + ("award_type", None), + ("AwardType", "award_type", None, None)) + ] + } + } +} + +used_apps = ("locations", ) \ No newline at end of file diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 2789427e..673966d6 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -10,6 +10,84 @@ from django.contrib.gis.db import models from transfer.mixins import MigrateMixin +class Sites(MigrateMixin): + using = 'legacy' + + country_code_2 = models.CharField(max_length=255, blank=True, null=True) + pinterest_page_url = models.CharField(max_length=255, blank=True, null=True) + twitter_page_url = models.CharField(max_length=255, blank=True, null=True) + facebook_page_url = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + contact_email = models.CharField(max_length=255, blank=True, null=True) + config = models.CharField(max_length=3000, blank=True, null=True) + released = models.IntegerField(blank=True, null=True) + instagram_page_url = models.CharField(max_length=255, blank=True, null=True) + ad_config = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'sites' + + +class Features(MigrateMixin): + using = 'legacy' + + slug = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'features' + + +class SiteFeatures(MigrateMixin): + using = 'legacy' + + site = models.ForeignKey('Sites', models.DO_NOTHING, blank=True, null=True) + feature = models.ForeignKey(Features, models.DO_NOTHING, blank=True, null=True) + state = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'site_features' + + +class AwardTypes(MigrateMixin): + using = 'legacy' + + site = models.ForeignKey('Sites', models.DO_NOTHING, blank=True, null=True) + title = models.CharField(max_length=255, blank=True, null=True) + region = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'award_types' + + +class Awards(MigrateMixin): + using = 'legacy' + + profile_id = models.IntegerField(blank=True, null=True) + award_type = models.ForeignKey(AwardTypes, models.DO_NOTHING, blank=True, null=True) + award = models.CharField(max_length=255, blank=True, null=True) + title = models.CharField(max_length=255, blank=True, null=True) + region_code = models.CharField(max_length=255, blank=True, null=True) + year = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + state = models.CharField(max_length=255, blank=True, null=True) + + class Meta: + managed = False + db_table = 'awards' + + class Ezuser(MigrateMixin): using = 'legacy' From a22383224153e494d3d1e752566988843ff56867 Mon Sep 17 00:00:00 2001 From: michail Date: Mon, 14 Oct 2019 15:48:26 +0500 Subject: [PATCH 030/173] Relational integers to ForeignKey --- apps/establishment/transfer.py | 12 ++++++------ apps/location/transfer.py | 6 +++--- apps/news/transfer.py | 2 +- apps/recipe/transfer.py | 2 +- apps/transfer/models.py | 7 +++++-- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py index 7e9fe57a..6e3e49aa 100644 --- a/apps/establishment/transfer.py +++ b/apps/establishment/transfer.py @@ -50,8 +50,8 @@ card = { # как работать с ManyToManyField - ? "EstablishmentSubType", "schedule" "relations": { "Locations": [ - (("location_id", "id"), - ("Address", "address", "id", "PositiveIntegerField")), + (("location", None), + ("Address", "address", None, None)), ], "Establishment": [#TODO правильно ли заполнена связь с EstablishmentType - ? (("type", "type"), @@ -79,8 +79,8 @@ card = { }, "relations": { "Establishments": [ - (("establishment_id", "id"), - ("Establishment", "establishment", "id", "PositiveIntegerField")) + (("establishment", None), + ("Establishment", "establishment", None, None)) ] } }, @@ -110,8 +110,8 @@ card = { }, "relations": { "Establishments": [ - (("establishment_id", "id"), - ("Establishment", "establishment", "id", "PositiveIntegerField")) + (("establishment", None), + ("Establishment", "establishment", None, None)) ] } } diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 4ae132c8..7528b45c 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -114,7 +114,7 @@ card = { } }, "CityPhotos": { - "key": ("city_id", "id"), + "key": "city", "fields": { "coordinates": "geometries" } @@ -144,8 +144,8 @@ card = { }, "relations": { "Cities": [ #TODO: Locations ссылается внешним ключом на Cities - (("city_id", "id"), - ("City", "city", "id", "PositiveIntegerField")) + (("city", None), + ("City", "city", None, None)) ] } } diff --git a/apps/news/transfer.py b/apps/news/transfer.py index 4f28d5a4..aebee449 100644 --- a/apps/news/transfer.py +++ b/apps/news/transfer.py @@ -63,7 +63,7 @@ card = { }, "relations": { "PageTexts": { - "key": "page_id", + "key": "page", "fields": { "title": ("title", "TJSONField"), "description": ("body", "TJSONField"), diff --git a/apps/recipe/transfer.py b/apps/recipe/transfer.py index f631980d..081ca51c 100644 --- a/apps/recipe/transfer.py +++ b/apps/recipe/transfer.py @@ -71,7 +71,7 @@ card = { }, "relations": { "PageTexts": { - "key": "page_id", + "key": "page", "fields": { "title": ("title", "TJSONField"), "description": ("body", "TJSONField") diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 673966d6..caab0321 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -10,6 +10,8 @@ from django.contrib.gis.db import models from transfer.mixins import MigrateMixin +# models.ForeignKey(ForeignModel, models.DO_NOTHING, blank=True, null=True) + class Sites(MigrateMixin): using = 'legacy' @@ -215,7 +217,8 @@ class CityNames(MigrateMixin): class CityPhotos(MigrateMixin): using = 'legacy' - city_id = models.IntegerField(blank=True, null=True) + # city_id = models.IntegerField(blank=True, null=True) + city = models.ForeignKey(Cities, models.DO_NOTHING, blank=True, null=True) attachment_file_name = models.CharField(max_length=255, blank=True, null=True) attachment_content_type = models.CharField(max_length=255, blank=True, null=True) geometries = models.CharField(max_length=1024, blank=True, null=True) @@ -745,7 +748,7 @@ class PageTexts(MigrateMixin): body = models.TextField(blank=True, null=True) locale = models.CharField(max_length=255, blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) - page_id = models.IntegerField() + page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() summary = models.TextField(blank=True, null=True) From dc12a184a1d597ffa35dd40d0e69a061e5a42563 Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 15 Oct 2019 12:46:06 +0500 Subject: [PATCH 031/173] the fix types of news fields --- apps/news/transfer.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/apps/news/transfer.py b/apps/news/transfer.py index aebee449..68c22dab 100644 --- a/apps/news/transfer.py +++ b/apps/news/transfer.py @@ -26,6 +26,12 @@ psql_field_type - тип уникального поля в postgresql NOTE: среди legacy таблиц совпадение для таблицы Address не найдено (Возможно для Address подходит Locations в legacy) """ +# "from django.db import models" + +# "models.PositiveIntegerField" + +# "django.db.models." + card = { # нету аналога для NewsType "NewsType": { @@ -47,10 +53,10 @@ card = { "fields": { # нету аналогов для start, end, playlist "Pages": { - "state": ("state", "PositiveSmallIntegerField"), - "template": ("template", "PositiveIntegerField"), - "image_url": ("attachment_file_name", "URLField"), - "preview_image_url": ("attachment_file_name", "URLField"), + "state": ("state", "django.db.models.PositiveSmallIntegerField"), + "template": ("template", "django.db.models.PositiveIntegerField"), + "image_url": ("attachment_file_name", "django.db.models.URLField"), + "preview_image_url": ("attachment_file_name", "django.db.models.URLField"), # в NewsOlds нету аналога для поля subtitle модели News, также нет аналогов для полей start, end, playlist # "subtitle": "" @@ -65,9 +71,9 @@ card = { "PageTexts": { "key": "page", "fields": { - "title": ("title", "TJSONField"), - "description": ("body", "TJSONField"), - "slug": ("slug", "SlugField") + "title": ("title", "django.db.models.JSONField"), + "description": ("body", "django.db.models.JSONField"), + "slug": ("slug", "django.db.models.SlugField") } } }, @@ -76,7 +82,7 @@ card = { "relations": { "Pages": [ ((None, "type"), - ("NewsType", "news_type", "name", "CharField")) + ("NewsType", "news_type", "name", "django.db.models.CharField")) ] } }, From 54cfccbcbeb3a0c63927218711aec548d084c7c3 Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 15 Oct 2019 13:23:31 +0500 Subject: [PATCH 032/173] the fix of types fields in all apps --- apps/account/transfer.py | 2 +- apps/collection/transfer.py | 6 +++--- apps/comment/transfer.py | 2 +- apps/establishment/transfer.py | 12 ++++++------ apps/location/transfer.py | 8 ++++---- apps/main/transfer.py | 8 ++++---- apps/notification/transfer.py | 2 +- apps/recipe/transfer.py | 6 +++--- apps/review/transfer.py | 4 ++-- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/apps/account/transfer.py b/apps/account/transfer.py index 964705e5..54db107b 100644 --- a/apps/account/transfer.py +++ b/apps/account/transfer.py @@ -32,7 +32,7 @@ card = { "Accounts": { "username": "nickname", "email": "email", - "email_confirmed": ("confirmed_at", "BooleanField") + "email_confirmed": ("confirmed_at", "django.db.models.BooleanField") }, "relations": { "Profiles": { diff --git a/apps/collection/transfer.py b/apps/collection/transfer.py index 0d276030..6c18c9ae 100644 --- a/apps/collection/transfer.py +++ b/apps/collection/transfer.py @@ -18,9 +18,9 @@ card = { # нету аналогов для полей description, start и end "name": "title", "slug": "slug", - "block_size": ("geometries", "JSONField"), - "is_publish": ("active", "BooleanField"), - "image_url": ("attachment_file_name", "URLField") + "block_size": ("geometries", "django.db.models.JSONField"), + "is_publish": ("active", "django.db.models.BooleanField"), + "image_url": ("attachment_file_name", "django.db.models.URLField") } }, "relations": { diff --git a/apps/comment/transfer.py b/apps/comment/transfer.py index 51eeed38..64c0abab 100644 --- a/apps/comment/transfer.py +++ b/apps/comment/transfer.py @@ -32,7 +32,7 @@ card = { "fields": { "Comments": { "text": "comment", - "mark": ("mark", "PositiveIntegerField") + "mark": ("mark", "django.db.models.PositiveIntegerField") # как работать с GenericForeignKey - ? # "content_object" : "" }, diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py index 6e3e49aa..604cc933 100644 --- a/apps/establishment/transfer.py +++ b/apps/establishment/transfer.py @@ -19,7 +19,7 @@ card = { "Establishments": { # значения для поля "name" берутся из поля "type" legacy модели "establishments" # https://jira.spider.ru/browse/GM-199 - "name": ("name", "TJSONField") + "name": ("name", "django.db.models.TJSONField") } } }, @@ -31,7 +31,7 @@ card = { "Establishments": { "name": "name", "slug": "slug", - "is_publish": ("state", "BooleanField") + "is_publish": ("state", "django.db.models.BooleanField") }, "relations": { "EstablishmentInfos": { @@ -53,9 +53,9 @@ card = { (("location", None), ("Address", "address", None, None)), ], - "Establishment": [#TODO правильно ли заполнена связь с EstablishmentType - ? + "Establishments": [#TODO правильно ли заполнена связь с EstablishmentType - ? (("type", "type"), - ("EstablishmentType", "establishment_type", "id", "PositiveIntegerField")) + ("EstablishmentType", "establishment_type", "id", "django.db.models.PositiveIntegerField")) ] # # "establishment_subtypes": "EstablishmentSubType", # "collections": "Collection", @@ -74,7 +74,7 @@ card = { "dependencies": ("Establishment", ), "fields": { "Menus": { - "category": ("name", "TJSONField") + "category": ("name", "django.db.models.TJSONField") } }, "relations": { @@ -94,7 +94,7 @@ card = { "relations": { #TODO правильно ли заполнена связь с Establishment - ? "Establishments": [ (("id", "id"), - ("Establishment", "establishment", "id", "PositiveIntegerField")) + ("Establishment", "establishment", "id", "django.db.models.PositiveIntegerField")) ] } } diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 7528b45c..74099303 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -88,7 +88,7 @@ card = { "relations": { "Cities": [( (None, "country_code_2"), - ("Country", "country", "code", "CharField")), + ("Country", "country", "code", "django.db.models.CharField")), # ((None, "subregion_code"), #TODO: как сопоставлять parent_region из postgres с subregion_code из legacy ? # ("Region", "parent_region", "code", "CharField")) @@ -104,7 +104,7 @@ card = { "coordinates": (("lat", "latitude"), ("long", "longitude")), "code": "country_code_2", "postal_code": "zip_code", - "is_island": ("is_island", "is_island", "Boolean") + "is_island": ("is_island", "is_island", "django.db.models.Boolean") }, "relations": { "CityNames": { @@ -124,9 +124,9 @@ card = { "relations": { "Cities": [( (None, "region_code"), - ("Region", "region", "code", "CharField")), + ("Region", "region", "code", "django.db.models.CharField")), ((None, "country_code_2"), - ("Country", "country", "code", "CharField")) + ("Country", "country", "code", "django.db.models.CharField")) ] } }, diff --git a/apps/main/transfer.py b/apps/main/transfer.py index 0d22ef72..5d9caafe 100644 --- a/apps/main/transfer.py +++ b/apps/main/transfer.py @@ -64,14 +64,14 @@ card = { "data_type": "objects", "dependencies": ("Country",), "fields": { - "SiteSettings": { + "Sites": { "subdomain": "country_code_2", "pinterest_page_url": "pinterest_page_url", "twitter_page_url": "twitter_page_url", "facebook_page_url": "facebook_page_url", "instagram_page_url": "instagram_page_url", "contact_email": "contact_email", - "config": ("config", "JSONField") #TODO в качесте ключа использовать country_code_2 из legacy - ? + "config": ("config", "django.db.models.JSONField") #TODO в качесте ключа использовать country_code_2 из legacy - ? }, # "relations": { # # Как работать c отношение OneToOneField @@ -97,7 +97,7 @@ card = { "dependencies": ("SiteSettings", "Feature"), "fields": { "SiteFeatures": { - "published": ("state", "BooleanField") + "published": ("state", "django.db.models.BooleanField") } }, "relations": { @@ -129,7 +129,7 @@ card = { "dependencies": ("AwardType", "ContentType"), "fields": { "Awards": { - "title": ("title", "TJSONField"), + "title": ("title", "django.db.models.JSONField"), "vintage_year": "year", } }, diff --git a/apps/notification/transfer.py b/apps/notification/transfer.py index 1feb4385..9fa47a20 100644 --- a/apps/notification/transfer.py +++ b/apps/notification/transfer.py @@ -35,7 +35,7 @@ card = { "fields": { "EmailAddresses": { "email": "email", - "state": ("state", "PositiveIntegerField") # из legacy брать только те записи у которых state=usable + "state": ("state", "django.db.models.PositiveIntegerField") # из legacy брать только те записи у которых state=usable }, # "relations": [ # # отложено до выяснения Уточнения и вопросы по мигратору(Как поступать со сбором данных) diff --git a/apps/recipe/transfer.py b/apps/recipe/transfer.py index 081ca51c..7b0d4bf8 100644 --- a/apps/recipe/transfer.py +++ b/apps/recipe/transfer.py @@ -67,14 +67,14 @@ card = { "dependencies": None, "fields": { "Pages": { - "state": ("state", "PositiveSmallIntegerField") + "state": ("state", "django.db.models.PositiveSmallIntegerField") }, "relations": { "PageTexts": { "key": "page", "fields": { - "title": ("title", "TJSONField"), - "description": ("body", "TJSONField") + "title": ("title", "django.db.models.JSONField"), + "description": ("body", "django.db.models.JSONField") } } } diff --git a/apps/review/transfer.py b/apps/review/transfer.py index 63f6f925..fa56c4b2 100644 --- a/apps/review/transfer.py +++ b/apps/review/transfer.py @@ -36,7 +36,7 @@ card = { "Reviews": { "published_at": "published_at", "vintage": "vintage", - "status": ("aasm_state", "PositiveSmallIntegerField") + "status": ("aasm_state", "django.db.models.PositiveSmallIntegerField") # "content_object": "" }, @@ -46,7 +46,7 @@ card = { "fields": { # полу text в модели Review имеет тип TJSONField, а поле text в модели ReviewTexts имеет тип TextField # при их сопоставлении использовать поле locale модели ReviewTexts - "text": ("text", "TJSONField") + "text": ("text", "django.db.models.JSONField") } } }, From 8cae39cbb61926fa221d3650e9313995e6dd9ce0 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 14 Oct 2019 16:30:26 +0300 Subject: [PATCH 033/173] Tmp fix --- apps/transfer/utils.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 55783e9c..32cb4bae 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,7 +2,7 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from collections import OrderedDict -from django.db.models import OuterRef, Subquery +from django.db.models import OuterRef, Subquery, FilteredRelation, Q import transfer.models as legacy_models @@ -118,7 +118,6 @@ def get_relation_data(relations, queryset, fields): """Получаем названия внешних зависимостей""" for relation_table, relation_data in relations.items(): """Если объекты связаны через ForeignKey, то ничего не надо делать - все аннотации уже подключены""" - if isinstance(relation_data['key'], tuple): """Вытаскиваем relation по id""" try: @@ -132,7 +131,9 @@ def get_relation_data(relations, queryset, fields): queryset = queryset.annotate(**{ f"{relation_table.lower()}": - Subquery(child_table_queryset) + Subquery( + child_table_queryset + ) }) return queryset @@ -150,12 +151,17 @@ def get_transfer_data(model, card, queryset): if "fields" not in card or not isinstance(card['fields'], dict) or len(card['fields']) != 1: raise ValueError("You must set correct fields to transfer") - for app_field, legacy_field_data in card['fields'].items(): - print(app_field) - print(legacy_field_data) + # for app_field, legacy_field_data in card['fields'].items(): + # print(app_field) + # print(legacy_field_data) - print("*************************************************************************") + # print("*************************************************************************") # print(app_model) # print("*************************************************************************") # print(queryset) + # for modellt in queryset: + # print(dir(modellt)) + # if hasattr(modellt, "cityphotos_set"): + # print(modellt.cityphotos_set.all()) + # break return queryset From 47a4d28cf02d5814a1fde3778c81f4b5f3e429b6 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 15 Oct 2019 10:38:51 +0300 Subject: [PATCH 034/173] Add fields map for simple field Add fields map for tuple field Refactoring code --- apps/news/transfer.py | 9 +- apps/transfer/management/commands/transfer.py | 1 + apps/transfer/utils.py | 245 +++++++++--------- 3 files changed, 133 insertions(+), 122 deletions(-) diff --git a/apps/news/transfer.py b/apps/news/transfer.py index 68c22dab..6a904ad9 100644 --- a/apps/news/transfer.py +++ b/apps/news/transfer.py @@ -35,21 +35,22 @@ NOTE: среди legacy таблиц совпадение для таблицы card = { # нету аналога для NewsType "NewsType": { - "data_type": "dictionaries", + "data_type": "news", "dependencies": None, "fields": { "Pages": { # будет только один тип новости "News" # значения для поля "name" берутся из поля "type" legacy модели "Pages", притом type="News" # Mysql - select distinct(type) from pages; - "name": "name" + "name": "type" } } }, "News": { - "data_type": "objects", - "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), + "data_type": "news", + # "dependencies": ("NewsType", "MetaDataContent", "Country", "Address"), + "dependencies": ("NewsType", ), "fields": { # нету аналогов для start, end, playlist "Pages": { diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 9b99ea6c..8018e38f 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -10,6 +10,7 @@ class Command(BaseCommand): """ DATA_TYPES = [ 'dictionaries', + 'news' ] def handle(self, *args, **options): diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 32cb4bae..cd5a3f19 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,12 +2,7 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from collections import OrderedDict -from django.db.models import OuterRef, Subquery, FilteredRelation, Q - -import transfer.models as legacy_models - -# from pprint import pprint as print - +from pprint import pprint def transfer_objects(data_type): models_list = {} @@ -29,12 +24,134 @@ def transfer_objects(data_type): models_list = sort_by_dependencies(models_list) - legacy_objects = OrderedDict() - for model, card in models_list.items(): - legacy_objects[model] = get_model_data(model, card) + transfer_data(model, card) +# TRANSFER DATA FUNCTION +def transfer_data(model, card): + legacy_model_queryset = get_legacy_data(card) + fields_map = get_fields_map(card, legacy_model_queryset) + + + + + + + + + + + + + + + + + + + + +# ************************** UTILITIES ****************************** + +# Get map of fields +def get_fields_map(card, queryset): + relations = card['fields']['relations'] if "relations" in card['fields'] else None + if relations: + del (card['fields']['relations']) + + fields_list = list(card['fields'].values())[0] + + app_queryset_list = [] + + for legacy_object in queryset.iterator(): + app_queryset_dict = {} + for app_field, legacy_field in fields_list.items(): + if isinstance(legacy_field, tuple): + if isinstance(legacy_field[0], tuple): + # Группа полей с аргументами + pass + else: + print(app_field) + print(legacy_field) + else: + try: + app_queryset_dict[app_field] = getattr(legacy_object, legacy_field) + except Exception as e: + print(f"Attribute {legacy_field} not found in {legacy_object}: {e}") + return + + app_queryset_list.append(app_queryset_dict) + + # pprint(card) + # print("*************************************") + # pprint(fields_list) + # print("*************************************") + # pprint(relations) + # print("*************************************") + # pprint(len(queryset)) + print("==============================") + + return card + + +# Get legacy model data +def get_legacy_data(card): + if "fields" not in card: + print("You must set fields list") + return + card_fields = card['fields'] + relations = card_fields['relations'] if 'relations' in card_fields else None + + if relations: + del (card_fields['relations']) + + if len(card_fields) > 1: + print("You must set one table to transfer") + return + + try: + legacy_table = list(card_fields.keys())[0] + except Exception as e: + print(f"Cannot get legacy table name from {card_fields.keys()}") + return + try: + fields = list(card_fields.values())[0] + except Exception as e: + print(f"Cannot get legacy fields from {card_fields.values()}") + return + + try: + legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) + except LookupError as e: + print(f"ERROR: legacy model {legacy_table} can not be loaded: {e}") + return + + legacy_fields = [] + + for legacy_field in fields.values(): + if isinstance(legacy_field, str): + legacy_fields.append(legacy_field) + + elif isinstance(legacy_field, tuple): + if isinstance(legacy_field[0], tuple): + """Группа полей""" + for legacy_field_group in legacy_field: + legacy_fields.append(legacy_field_group[1]) + else: + """Одиночное поле""" + legacy_fields.append(legacy_field[0]) + + queryset = legacy_model.objects.only(*legacy_fields) + + # Возвращаем зависимости на место + if relations: + card['fields']['relations'] = relations + + return queryset + + +# Models sort def sort_by_dependencies(data): """Сортировка моделей по зависимостям""" @@ -56,112 +173,4 @@ def sort_by_dependencies(data): [(model, data[model]) for model in reversed(result)] ) - return result - - -def get_model_data(model, card): - # print(f"=================================================\r\n" - # f"MODEL: {model}\r\n" - # f"=================================================") - - card_fields = card['fields'] - relations = card_fields['relations'] if 'relations' in card_fields else None - - if relations: - del(card_fields['relations']) - - if len(card_fields) > 1: - print("You must set one table to transfer") - return - - legacy_table = list(card_fields.keys())[0] - fields = list(card_fields.values())[0] - - try: - legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) - except LookupError as e: - print(f"ERROR: legacy model {model} can not be loaded: {e}") - return - - legacy_fields = [] - - for legacy_field in fields.values(): - if isinstance(legacy_field, str): - legacy_fields.append(legacy_field) - - elif isinstance(legacy_field, tuple): - if isinstance(legacy_field[0], tuple): - """Группа полей""" - for legacy_field_group in legacy_field: - legacy_fields.append(legacy_field_group[1]) - else: - """Одиночное поле""" - legacy_fields.append(legacy_field[1]) - - #TODO: убрать после совместимости - if model != "City": - return - - queryset = legacy_model.objects.only(*legacy_fields) - - """Проверяем наличие внешних связей""" - if relations: - queryset = get_relation_data(relations, queryset, fields) - - return get_transfer_data(model, card, queryset) - - -def get_relation_data(relations, queryset, fields): - - # related_legacy_models = [x.related_model._meta.model.__name__ for x in legacy_model._meta.related_objects] - - """Получаем названия внешних зависимостей""" - for relation_table, relation_data in relations.items(): - """Если объекты связаны через ForeignKey, то ничего не надо делать - все аннотации уже подключены""" - if isinstance(relation_data['key'], tuple): - """Вытаскиваем relation по id""" - try: - child_legacy_model = apps.get_model(app_label="transfer", model_name=relation_table) - except LookupError as e: - raise LookupError(f"ERROR: legacy model {relation_table} can not be loaded: {e}") - - child_table_queryset = child_legacy_model.objects.filter( - city_id=OuterRef('pk') - ).only(*relation_data['fields'].values()) - - queryset = queryset.annotate(**{ - f"{relation_table.lower()}": - Subquery( - child_table_queryset - ) - }) - - return queryset - - -def get_transfer_data(model, card, queryset): - from pprint import pprint as print - - try: - app_model = apps.get_model(app_label=card['app_label'], model_name=model) - except LookupError as e: - print(f"ERROR: model {model} from {card['app_label']} can not be loaded: {e}") - return - - if "fields" not in card or not isinstance(card['fields'], dict) or len(card['fields']) != 1: - raise ValueError("You must set correct fields to transfer") - - # for app_field, legacy_field_data in card['fields'].items(): - # print(app_field) - # print(legacy_field_data) - - # print("*************************************************************************") - # print(app_model) - # print("*************************************************************************") - # print(queryset) - # for modellt in queryset: - # print(dir(modellt)) - # if hasattr(modellt, "cityphotos_set"): - # print(modellt.cityphotos_set.all()) - # break - return queryset + return result \ No newline at end of file From 4d908d0bbb6ca4c397cf90ebbaf1d2c94823d663 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 15 Oct 2019 12:02:17 +0300 Subject: [PATCH 035/173] Add field converter --- apps/transfer/utils.py | 58 ++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index cd5a3f19..97c58a3a 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -2,8 +2,10 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps from collections import OrderedDict +import importlib from pprint import pprint + def transfer_objects(data_type): models_list = {} @@ -67,19 +69,9 @@ def get_fields_map(card, queryset): for legacy_object in queryset.iterator(): app_queryset_dict = {} for app_field, legacy_field in fields_list.items(): - if isinstance(legacy_field, tuple): - if isinstance(legacy_field[0], tuple): - # Группа полей с аргументами - pass - else: - print(app_field) - print(legacy_field) - else: - try: - app_queryset_dict[app_field] = getattr(legacy_object, legacy_field) - except Exception as e: - print(f"Attribute {legacy_field} not found in {legacy_object}: {e}") - return + app_value = convert_field_from_legacy_to_app(legacy_object, legacy_field) + if app_value is not None: + app_queryset_dict[app_field] = app_value app_queryset_list.append(app_queryset_dict) @@ -92,7 +84,45 @@ def get_fields_map(card, queryset): # pprint(len(queryset)) print("==============================") - return card + return app_queryset_list + + +# Convert field +def convert_field_from_legacy_to_app(legacy_object, legacy_field): + result = None + if isinstance(legacy_field, tuple): + if isinstance(legacy_field[0], tuple): + # legacy_attributes = {} + # for legacy_attribute in legacy_field: + print(legacy_field) + else: + if hasattr(legacy_object, legacy_field[0]): + try: + legacy_data = getattr(legacy_object, legacy_field[0]) + except Exception as e: + raise AttributeError(f"Attribute {legacy_field} not found in {legacy_object}: {e}") + + field_module = legacy_field[-1].split(".") + try: + field_type = getattr(importlib.import_module(".".join(field_module[:-1])), field_module[-1]) + except Exception as e: + raise ValueError(f"Cannot set new field type for field {legacy_field}: {e}") + + if len(legacy_field) == 3: + result = field_type(**{legacy_field[1]: legacy_data}) + elif len(legacy_field) == 2: + result = field_type(legacy_data) + + else: + print(f"Attribute {legacy_field} not found in {legacy_object.__dict__}") + + else: + if hasattr(legacy_object, legacy_field): + result = getattr(legacy_object, legacy_field) + else: + print(f"Attribute {legacy_field} not found in {legacy_object.__dict__}") + + return result # Get legacy model data From 09865db08ad934c917db5ac0f55d7967ef37a1d3 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 15 Oct 2019 12:05:30 +0300 Subject: [PATCH 036/173] Tmp commit to debug --- apps/transfer/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 97c58a3a..33f096e3 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -33,6 +33,8 @@ def transfer_objects(data_type): # TRANSFER DATA FUNCTION def transfer_data(model, card): legacy_model_queryset = get_legacy_data(card) + print("==============================================================================") + print(model) fields_map = get_fields_map(card, legacy_model_queryset) @@ -65,6 +67,9 @@ def get_fields_map(card, queryset): fields_list = list(card['fields'].values())[0] app_queryset_list = [] + print("==============================================================================") + pprint(card) + print("==============================================================================") for legacy_object in queryset.iterator(): app_queryset_dict = {} @@ -72,7 +77,8 @@ def get_fields_map(card, queryset): app_value = convert_field_from_legacy_to_app(legacy_object, legacy_field) if app_value is not None: app_queryset_dict[app_field] = app_value - + print("==============================") + pprint(app_queryset_dict) app_queryset_list.append(app_queryset_dict) # pprint(card) @@ -82,7 +88,7 @@ def get_fields_map(card, queryset): # pprint(relations) # print("*************************************") # pprint(len(queryset)) - print("==============================") + return app_queryset_list @@ -172,7 +178,7 @@ def get_legacy_data(card): """Одиночное поле""" legacy_fields.append(legacy_field[0]) - queryset = legacy_model.objects.only(*legacy_fields) + queryset = legacy_model.objects.only(*legacy_fields).limit(50) #TODO: remove after debug # Возвращаем зависимости на место if relations: From ba1ce674444119b467f6664f3ca199c6577cbdc2 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 15 Oct 2019 12:07:37 +0300 Subject: [PATCH 037/173] Fix limit --- apps/transfer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 33f096e3..dba43b17 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -178,7 +178,7 @@ def get_legacy_data(card): """Одиночное поле""" legacy_fields.append(legacy_field[0]) - queryset = legacy_model.objects.only(*legacy_fields).limit(50) #TODO: remove after debug + queryset = legacy_model.objects.only(*legacy_fields)[:50]# TODO: remove after debug # Возвращаем зависимости на место if relations: From e1b031caf3fb425bc7bc461d49290654e6f22672 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 16 Oct 2019 07:05:05 +0300 Subject: [PATCH 038/173] Tmp commit --- apps/transfer/utils.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index dba43b17..818a8220 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -67,9 +67,9 @@ def get_fields_map(card, queryset): fields_list = list(card['fields'].values())[0] app_queryset_list = [] - print("==============================================================================") - pprint(card) - print("==============================================================================") + # print("==============================================================================") + # pprint(card) + # print("==============================================================================") for legacy_object in queryset.iterator(): app_queryset_dict = {} @@ -77,19 +77,18 @@ def get_fields_map(card, queryset): app_value = convert_field_from_legacy_to_app(legacy_object, legacy_field) if app_value is not None: app_queryset_dict[app_field] = app_value + + if relations is not None: + for relation_table, relation_data in relations.items(): + # print(f"{relation_table.lower()}_set") + print(legacy_object.pagetext) + # relation_object = getattr(legacy_object, f"{relation_table.lower()}_set") + + # print(relation_object) + print("==============================") - pprint(app_queryset_dict) app_queryset_list.append(app_queryset_dict) - # pprint(card) - # print("*************************************") - # pprint(fields_list) - # print("*************************************") - # pprint(relations) - # print("*************************************") - # pprint(len(queryset)) - - return app_queryset_list @@ -178,15 +177,23 @@ def get_legacy_data(card): """Одиночное поле""" legacy_fields.append(legacy_field[0]) - queryset = legacy_model.objects.only(*legacy_fields)[:50]# TODO: remove after debug + queryset = legacy_model.objects.all()[:50]# TODO: remove after debug # Возвращаем зависимости на место - if relations: + if relations is not None: + queryset = add_legacy_relation_data(queryset, relations) card['fields']['relations'] = relations return queryset +# Add relation data to queryset +def add_legacy_relation_data(queryset, relations): + for relation_table in relations.keys(): + queryset = queryset.prefetch_related(relation_table.lower()) + return queryset + + # Models sort def sort_by_dependencies(data): """Сортировка моделей по зависимостям""" From 8965cf0460e9ea83bf1740507ac8b27065fcc9aa Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 20 Oct 2019 17:59:18 +0300 Subject: [PATCH 039/173] Add news serializer --- apps/news/transfer_data.py | 28 ++++++++++++++++ apps/transfer/models.py | 20 ++++++------ apps/transfer/serializers/__init__.py | 0 apps/transfer/serializers/news.py | 37 +++++++++++++++++++++ apps/transfer/tests.py | 15 +++++++-- apps/transfer/utils.py | 46 +++++++++++++++++++-------- 6 files changed, 120 insertions(+), 26 deletions(-) create mode 100644 apps/news/transfer_data.py create mode 100644 apps/transfer/serializers/__init__.py create mode 100644 apps/transfer/serializers/news.py diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py new file mode 100644 index 00000000..ab02ca7f --- /dev/null +++ b/apps/news/transfer_data.py @@ -0,0 +1,28 @@ +from transfer.serializers.news import NewsSerializer +from transfer.models import PageTexts +from news.models import NewsType +from django.db.models import Value, IntegerField, F +from pprint import pprint + + +def transfer_news(): + news_type, _ = NewsType.objects.get_or_create(name="News") + print(news_type) + # + queryset = PageTexts.objects.all().annotate(news_type=Value(news_type.id, output_field=IntegerField())) + queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) + queryset = queryset.annotate(template=F('page__template')) + pprint(vars(queryset.first())) + print("===================================================================================") + # + serialized_data = NewsSerializer(queryset, many=True) + # data = serialized_data.data + # pprint(serialized_data.data) + for serialized_obj in serialized_data.data: + pprint(serialized_obj) + break + + +data_types = { + "news": [transfer_news] +} diff --git a/apps/transfer/models.py b/apps/transfer/models.py index caab0321..346ef6aa 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -369,16 +369,16 @@ class GuideFilters(MigrateMixin): managed = False db_table = 'guide_filters' - -class GuideSections(MigrateMixin): - using = 'legacy' - - type = models.CharField(max_length=255) - key_name = models.CharField(max_length=255, blank=True, null=True) - value_name = models.CharField(max_length=255, blank=True, null=True) - right = models.IntegerField(blank=True, null=True) - created_at = models.DateTimeField() - updated_at = models.DateTimeField() +# +# class GuideSections(MigrateMixin): +# using = 'legacy' +# +# type = models.CharField(max_length=255) +# key_name = models.CharField(max_length=255, blank=True, null=True) +# value_name = models.CharField(max_length=255, blank=True, null=True) +# right = models.IntegerField(blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() # class GuideElements(MigrateMixin): diff --git a/apps/transfer/serializers/__init__.py b/apps/transfer/serializers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py new file mode 100644 index 00000000..877c8979 --- /dev/null +++ b/apps/transfer/serializers/news.py @@ -0,0 +1,37 @@ +from rest_framework import serializers +from news.models import News + + +class NewsSerializer(serializers.ModelSerializer): + image_url = serializers.SerializerMethodField() + preview_image_url = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + + class Meta: + model = News + fields = ( + "state", + "template", + "image_url", + "preview_image_url", + "title", + "description", + "slug", + "news_type" + ) + + def get_description(self, obj): + if not hasattr(obj, "body"): + return "" + else: + return obj.body + + def get_image_url(self, obj): + if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: + return "" + return obj.attachment_file_name + + def get_preview_image_url(self, obj): + if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: + return "" + return obj.attachment_file_name diff --git a/apps/transfer/tests.py b/apps/transfer/tests.py index 7ce503c2..593d2501 100644 --- a/apps/transfer/tests.py +++ b/apps/transfer/tests.py @@ -1,3 +1,14 @@ from django.test import TestCase - -# Create your tests here. +# from django.core.management import call_command +# +# +# class NewsTransferTestCase(TestCase): +# command = "-n" +# +# databases = {'default', 'legacy'} +# +# def test_transfer(self): +# args = [self.command] +# kwargs = {} +# +# call_command('transfer', *args, **kwargs) diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index 818a8220..a045f39e 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -7,27 +7,45 @@ from pprint import pprint def transfer_objects(data_type): - models_list = {} for app in apps.get_app_configs(): - if exists(f"{app.path}/transfer.py"): - card_module = SourceFileLoader("transfer", f"{app.path}/transfer.py").load_module() - if not hasattr(card_module, "card") or len(card_module.card) < 1: + if exists(f"{app.path}/transfer_data.py"): + card_module = SourceFileLoader("transfer", f"{app.path}/transfer_data.py").load_module() + if not hasattr(card_module, "data_types") or not isinstance(card_module.data_types, dict) or len(card_module.data_types) < 1: continue - for model, card in card_module.card.items(): - if "data_type" in card and data_type == card["data_type"]: - card['app_label'] = app.label - models_list[model] = card + for module_data_type, transfer_funcs in card_module.data_types.items(): + if data_type == module_data_type: + # card['app_label'] = app.label + # models_list[model] = card + for transfer_func in transfer_funcs: + print(f"========================== FUNCTION {transfer_func.__name__} ================================") + transfer_func() + + # if len(models_list) < 1: + # print(f"Models with data type {data_type} not found in structure") + # exit(1) + + # models_list = sort_by_dependencies(models_list) + # + # for model, card in models_list.items(): + # transfer_data(model, card) + + + + + + + + + + + + + - if len(models_list) < 1: - print(f"Models with data type {data_type} not found in structure") - exit(1) - models_list = sort_by_dependencies(models_list) - for model, card in models_list.items(): - transfer_data(model, card) # TRANSFER DATA FUNCTION From 0a24b80917a383c70077b53fefcba584e22675ca Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 20 Oct 2019 19:08:14 +0300 Subject: [PATCH 040/173] Add title method --- apps/news/transfer_data.py | 19 +++++++------------ apps/transfer/serializers/news.py | 4 ++++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index ab02ca7f..06b6510d 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -4,23 +4,18 @@ from news.models import NewsType from django.db.models import Value, IntegerField, F from pprint import pprint - def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name="News") - print(news_type) - # + queryset = PageTexts.objects.all().annotate(news_type=Value(news_type.id, output_field=IntegerField())) queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) queryset = queryset.annotate(template=F('page__template')) - pprint(vars(queryset.first())) - print("===================================================================================") - # - serialized_data = NewsSerializer(queryset, many=True) - # data = serialized_data.data - # pprint(serialized_data.data) - for serialized_obj in serialized_data.data: - pprint(serialized_obj) - break + + serialized_data = NewsSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") data_types = { diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 877c8979..fd36e62e 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -6,6 +6,7 @@ class NewsSerializer(serializers.ModelSerializer): image_url = serializers.SerializerMethodField() preview_image_url = serializers.SerializerMethodField() description = serializers.SerializerMethodField() + title = serializers.SerializerMethodField() class Meta: model = News @@ -20,6 +21,9 @@ class NewsSerializer(serializers.ModelSerializer): "news_type" ) + def get_title(self, obj): + return f'{"en-GB": {obj.title}}' + def get_description(self, obj): if not hasattr(obj, "body"): return "" From 019a10db52ae524cd07052c42a8357f32edcf9b6 Mon Sep 17 00:00:00 2001 From: michail Date: Mon, 21 Oct 2019 11:07:54 +0500 Subject: [PATCH 041/173] commit --- apps/transfer/models.py | 89 +++++++++++++++++++++------------------- project/settings/base.py | 3 +- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/apps/transfer/models.py b/apps/transfer/models.py index caab0321..e661b402 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -648,49 +648,51 @@ class EmailAddresses(MigrateMixin): db_table = 'email_addresses' -# class Reviews(MigrateMixin): -# using = 'legacy' -# -# vintage = models.PositiveIntegerField() -# mark = models.FloatField(blank=True, null=True) -# favorite = models.IntegerField(blank=True, null=True) -# account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) -# establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) -# visited_at = models.DateField(blank=True, null=True) -# created_at = models.DateTimeField() -# published_at = models.DateTimeField(blank=True, null=True) -# updated_at = models.DateTimeField() -# aasm_state = models.CharField(max_length=255, blank=True, null=True) -# reviewer_id = models.IntegerField() -# priority = models.IntegerField(blank=True, null=True) -# #TODO: модель Products в postgres закомментирована -# product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) -# received_at = models.DateTimeField(blank=True, null=True) -# reviewer_name = models.CharField(max_length=255, blank=True, null=True) -# type = models.CharField(max_length=255, blank=True, null=True) -# locked = models.IntegerField(blank=True, null=True) -# temporary = models.IntegerField(blank=True, null=True) -# last_state_change_at = models.DateTimeField(blank=True, null=True) -# editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'reviews' -# +class Reviews(MigrateMixin): + using = 'legacy' -# class ReviewTexts(MigrateMixin): -# using = 'legacy' -# -# review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) -# locale = models.CharField(max_length=5, blank=True, null=True) -# text = models.TextField(blank=True, null=True) -# updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) -# created_at = models.DateTimeField() -# updated_at = models.DateTimeField() -# -# class Meta: -# managed = False -# db_table = 'review_texts' + vintage = models.PositiveIntegerField() + mark = models.FloatField(blank=True, null=True) + favorite = models.IntegerField(blank=True, null=True) + # account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="account_reviews") + account_id = models.IntegerField(blank=True, null=True) + establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) + visited_at = models.DateField(blank=True, null=True) + created_at = models.DateTimeField() + published_at = models.DateTimeField(blank=True, null=True) + updated_at = models.DateTimeField() + aasm_state = models.CharField(max_length=255, blank=True, null=True) + reviewer_id = models.IntegerField() + priority = models.IntegerField(blank=True, null=True) + #TODO: модель Products в postgres закомментирована + # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) + received_at = models.DateTimeField(blank=True, null=True) + reviewer_name = models.CharField(max_length=255, blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + locked = models.IntegerField(blank=True, null=True) + temporary = models.IntegerField(blank=True, null=True) + last_state_change_at = models.DateTimeField(blank=True, null=True) + # editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="editor_reviews") + + class Meta: + managed = False + db_table = 'reviews' + + +class ReviewTexts(MigrateMixin): + using = 'legacy' + + # review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) + review_id = models.IntegerField(blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + text = models.TextField(blank=True, null=True) + updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'review_texts' class Comments(MigrateMixin): using = 'legacy' @@ -748,7 +750,8 @@ class PageTexts(MigrateMixin): body = models.TextField(blank=True, null=True) locale = models.CharField(max_length=255, blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) - page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) + # page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) + page_id = models.IntegerField(blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() summary = models.TextField(blank=True, null=True) diff --git a/project/settings/base.py b/project/settings/base.py index 509b9b14..dd6c52cc 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -152,7 +152,8 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.23.0.1', + # 'HOST': '172.23.0.1', + 'HOST': '172.17.0.1', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From c6750400ba4a3f42406468c811f622093988147d Mon Sep 17 00:00:00 2001 From: michail Date: Mon, 21 Oct 2019 11:16:11 +0500 Subject: [PATCH 042/173] fix IntegerField to ForeignKey --- apps/transfer/models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 034bcbaa..f469f698 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -654,8 +654,8 @@ class Reviews(MigrateMixin): vintage = models.PositiveIntegerField() mark = models.FloatField(blank=True, null=True) favorite = models.IntegerField(blank=True, null=True) - # account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="account_reviews") - account_id = models.IntegerField(blank=True, null=True) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="account_reviews") + # account_id = models.IntegerField(blank=True, null=True) establishment = models.ForeignKey(Establishments, models.DO_NOTHING, blank=True, null=True) visited_at = models.DateField(blank=True, null=True) created_at = models.DateTimeField() @@ -672,7 +672,7 @@ class Reviews(MigrateMixin): locked = models.IntegerField(blank=True, null=True) temporary = models.IntegerField(blank=True, null=True) last_state_change_at = models.DateTimeField(blank=True, null=True) - # editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="editor_reviews") + editor = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True, related_name="editor_reviews") class Meta: managed = False @@ -682,8 +682,8 @@ class Reviews(MigrateMixin): class ReviewTexts(MigrateMixin): using = 'legacy' - # review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) - review_id = models.IntegerField(blank=True, null=True) + review = models.ForeignKey('Reviews', models.DO_NOTHING, blank=True, null=True) + # review_id = models.IntegerField(blank=True, null=True) locale = models.CharField(max_length=5, blank=True, null=True) text = models.TextField(blank=True, null=True) updated_by = models.ForeignKey(Accounts, models.DO_NOTHING, db_column='updated_by', blank=True, null=True) @@ -750,8 +750,8 @@ class PageTexts(MigrateMixin): body = models.TextField(blank=True, null=True) locale = models.CharField(max_length=255, blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) - # page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) - page_id = models.IntegerField(blank=True, null=True) + page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) + # page_id = models.IntegerField(blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() summary = models.TextField(blank=True, null=True) From 7ebd3383313545639675c76f22d87a506604dfbb Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 21 Oct 2019 09:59:59 +0300 Subject: [PATCH 043/173] Tmp template fix --- apps/transfer/serializers/news.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index fd36e62e..ae99e45b 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -7,6 +7,7 @@ class NewsSerializer(serializers.ModelSerializer): preview_image_url = serializers.SerializerMethodField() description = serializers.SerializerMethodField() title = serializers.SerializerMethodField() + template = serializers.SerializerMethodField() class Meta: model = News @@ -21,6 +22,9 @@ class NewsSerializer(serializers.ModelSerializer): "news_type" ) + def get_template(self, obj): + return obj.template + def get_title(self, obj): return f'{"en-GB": {obj.title}}' From 3f85aa84cc6bb42aeb7f4995a3f65be272fb1988 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 21 Oct 2019 14:19:01 +0300 Subject: [PATCH 044/173] Remove tmp code --- apps/news/transfer_data.py | 1 + apps/transfer/utils.py | 219 ------------------------------------- 2 files changed, 1 insertion(+), 219 deletions(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 06b6510d..67ec89b0 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -4,6 +4,7 @@ from news.models import NewsType from django.db.models import Value, IntegerField, F from pprint import pprint + def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name="News") diff --git a/apps/transfer/utils.py b/apps/transfer/utils.py index a045f39e..b83d5b2b 100644 --- a/apps/transfer/utils.py +++ b/apps/transfer/utils.py @@ -1,9 +1,6 @@ from os.path import exists from importlib.machinery import SourceFileLoader from django.apps import apps -from collections import OrderedDict -import importlib -from pprint import pprint def transfer_objects(data_type): @@ -16,222 +13,6 @@ def transfer_objects(data_type): for module_data_type, transfer_funcs in card_module.data_types.items(): if data_type == module_data_type: - # card['app_label'] = app.label - # models_list[model] = card for transfer_func in transfer_funcs: print(f"========================== FUNCTION {transfer_func.__name__} ================================") transfer_func() - - # if len(models_list) < 1: - # print(f"Models with data type {data_type} not found in structure") - # exit(1) - - # models_list = sort_by_dependencies(models_list) - # - # for model, card in models_list.items(): - # transfer_data(model, card) - - - - - - - - - - - - - - - - - - -# TRANSFER DATA FUNCTION -def transfer_data(model, card): - legacy_model_queryset = get_legacy_data(card) - print("==============================================================================") - print(model) - fields_map = get_fields_map(card, legacy_model_queryset) - - - - - - - - - - - - - - - - - - - - -# ************************** UTILITIES ****************************** - -# Get map of fields -def get_fields_map(card, queryset): - relations = card['fields']['relations'] if "relations" in card['fields'] else None - if relations: - del (card['fields']['relations']) - - fields_list = list(card['fields'].values())[0] - - app_queryset_list = [] - # print("==============================================================================") - # pprint(card) - # print("==============================================================================") - - for legacy_object in queryset.iterator(): - app_queryset_dict = {} - for app_field, legacy_field in fields_list.items(): - app_value = convert_field_from_legacy_to_app(legacy_object, legacy_field) - if app_value is not None: - app_queryset_dict[app_field] = app_value - - if relations is not None: - for relation_table, relation_data in relations.items(): - # print(f"{relation_table.lower()}_set") - print(legacy_object.pagetext) - # relation_object = getattr(legacy_object, f"{relation_table.lower()}_set") - - # print(relation_object) - - print("==============================") - app_queryset_list.append(app_queryset_dict) - - return app_queryset_list - - -# Convert field -def convert_field_from_legacy_to_app(legacy_object, legacy_field): - result = None - if isinstance(legacy_field, tuple): - if isinstance(legacy_field[0], tuple): - # legacy_attributes = {} - # for legacy_attribute in legacy_field: - print(legacy_field) - else: - if hasattr(legacy_object, legacy_field[0]): - try: - legacy_data = getattr(legacy_object, legacy_field[0]) - except Exception as e: - raise AttributeError(f"Attribute {legacy_field} not found in {legacy_object}: {e}") - - field_module = legacy_field[-1].split(".") - try: - field_type = getattr(importlib.import_module(".".join(field_module[:-1])), field_module[-1]) - except Exception as e: - raise ValueError(f"Cannot set new field type for field {legacy_field}: {e}") - - if len(legacy_field) == 3: - result = field_type(**{legacy_field[1]: legacy_data}) - elif len(legacy_field) == 2: - result = field_type(legacy_data) - - else: - print(f"Attribute {legacy_field} not found in {legacy_object.__dict__}") - - else: - if hasattr(legacy_object, legacy_field): - result = getattr(legacy_object, legacy_field) - else: - print(f"Attribute {legacy_field} not found in {legacy_object.__dict__}") - - return result - - -# Get legacy model data -def get_legacy_data(card): - if "fields" not in card: - print("You must set fields list") - return - card_fields = card['fields'] - relations = card_fields['relations'] if 'relations' in card_fields else None - - if relations: - del (card_fields['relations']) - - if len(card_fields) > 1: - print("You must set one table to transfer") - return - - try: - legacy_table = list(card_fields.keys())[0] - except Exception as e: - print(f"Cannot get legacy table name from {card_fields.keys()}") - return - try: - fields = list(card_fields.values())[0] - except Exception as e: - print(f"Cannot get legacy fields from {card_fields.values()}") - return - - try: - legacy_model = apps.get_model(app_label="transfer", model_name=legacy_table) - except LookupError as e: - print(f"ERROR: legacy model {legacy_table} can not be loaded: {e}") - return - - legacy_fields = [] - - for legacy_field in fields.values(): - if isinstance(legacy_field, str): - legacy_fields.append(legacy_field) - - elif isinstance(legacy_field, tuple): - if isinstance(legacy_field[0], tuple): - """Группа полей""" - for legacy_field_group in legacy_field: - legacy_fields.append(legacy_field_group[1]) - else: - """Одиночное поле""" - legacy_fields.append(legacy_field[0]) - - queryset = legacy_model.objects.all()[:50]# TODO: remove after debug - - # Возвращаем зависимости на место - if relations is not None: - queryset = add_legacy_relation_data(queryset, relations) - card['fields']['relations'] = relations - - return queryset - - -# Add relation data to queryset -def add_legacy_relation_data(queryset, relations): - for relation_table in relations.keys(): - queryset = queryset.prefetch_related(relation_table.lower()) - return queryset - - -# Models sort -def sort_by_dependencies(data): - """Сортировка моделей по зависимостям""" - - """Сначала мы сортируем модели по зависимостям в обратном порядке, используя сортировку вставкой""" - result = [] - for model, card in data.items(): - if "dependencies" in card and isinstance(card['dependencies'], tuple): - for model_dependency in result: - if model_dependency in card['dependencies']: - result.insert(result.index(model_dependency), model) - break - - else: - result.append(model) - - """Затем мы создаём сортированный словарь из реверса получившегося результата""" - - result = OrderedDict( - [(model, data[model]) for model in reversed(result)] - ) - - return result \ No newline at end of file From e7107cfd04ba708b219767e6b40099e1fa1c41fa Mon Sep 17 00:00:00 2001 From: michail Date: Mon, 21 Oct 2019 17:31:44 +0500 Subject: [PATCH 045/173] fixes of news --- .../migrations/0021_auto_20191021_1120.py | 18 +++++++++++ apps/news/models.py | 2 +- apps/news/transfer_data.py | 5 ++- apps/transfer/migrations/__init__.py | 0 apps/transfer/serializers/news.py | 31 +++++++++++++++++-- 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 apps/news/migrations/0021_auto_20191021_1120.py delete mode 100644 apps/transfer/migrations/__init__.py diff --git a/apps/news/migrations/0021_auto_20191021_1120.py b/apps/news/migrations/0021_auto_20191021_1120.py new file mode 100644 index 00000000..4aa1104e --- /dev/null +++ b/apps/news/migrations/0021_auto_20191021_1120.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-21 11:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0020_remove_news_author'), + ] + + operations = [ + migrations.AlterField( + model_name='news', + name='slug', + field=models.SlugField(max_length=255, unique=True, verbose_name='News slug'), + ), + ] diff --git a/apps/news/models.py b/apps/news/models.py index 6e91b912..4558f3dd 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -113,7 +113,7 @@ class News(BaseAttributes, TranslatedFieldsMixin): start = models.DateTimeField(verbose_name=_('Start')) end = models.DateTimeField(blank=True, null=True, default=None, verbose_name=_('End')) - slug = models.SlugField(unique=True, max_length=50, + slug = models.SlugField(unique=True, max_length=255, verbose_name=_('News slug')) playlist = models.IntegerField(_('playlist')) state = models.PositiveSmallIntegerField(default=WAITING, choices=STATE_CHOICES, diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 06b6510d..15c4b032 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -7,10 +7,13 @@ from pprint import pprint def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name="News") - queryset = PageTexts.objects.all().annotate(news_type=Value(news_type.id, output_field=IntegerField())) + queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), + playlist=Value(1, output_field=IntegerField())) + # queryset = PageTexts.objects.all().annotate(news_type=Value(news_type.id, output_field=IntegerField())) queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) queryset = queryset.annotate(template=F('page__template')) + serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() diff --git a/apps/transfer/migrations/__init__.py b/apps/transfer/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index ae99e45b..cd685c7a 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,5 +1,8 @@ from rest_framework import serializers from news.models import News +from uuid import uuid4 +import datetime +from django.conf import settings class NewsSerializer(serializers.ModelSerializer): @@ -8,10 +11,13 @@ class NewsSerializer(serializers.ModelSerializer): description = serializers.SerializerMethodField() title = serializers.SerializerMethodField() template = serializers.SerializerMethodField() + state = serializers.SerializerMethodField() + created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') class Meta: model = News fields = ( + "created_at", "state", "template", "image_url", @@ -19,11 +25,32 @@ class NewsSerializer(serializers.ModelSerializer): "title", "description", "slug", - "news_type" + "news_type", + "playlist" ) def get_template(self, obj): - return obj.template + if obj.template == "main": + return News.MAIN + elif obj.template == "main.pdf.erb": + return News.MAIN_PDF_ERB + else: + return News.NEWSPAPER + + def get_state(self, obj): + if obj.state == "published": + return News.PUBLISHED + elif obj.state == "hidden": + return News.HIDDEN + elif obj.state == "published_exclusive": + return News.PUBLISHED_EXCLUSIVE + else: + return News.WAITING + + # django.db.utils.IntegrityError: "start" NOT NULL + # def get_start(self, obj): + # # return obj.created_at + # return datetime.datetime.now().isoformat() def get_title(self, obj): return f'{"en-GB": {obj.title}}' From 21f10d06f23331cc8a34d0be0b2bc97e290f7783 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 21 Oct 2019 17:22:28 +0300 Subject: [PATCH 046/173] Fix state --- apps/transfer/serializers/news.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index cd685c7a..a61e040c 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -29,6 +29,10 @@ class NewsSerializer(serializers.ModelSerializer): "playlist" ) + def create(self, validated_data): + validated_data['state'] = self.get_state(validated_data) + return News.objects.create(**validated_data) + def get_template(self, obj): if obj.template == "main": return News.MAIN From 863762bdf27c2e542b9073c07e78762f40d9da34 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 21 Oct 2019 17:23:50 +0300 Subject: [PATCH 047/173] Fix state --- apps/transfer/serializers/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index a61e040c..1aa0120a 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -11,7 +11,7 @@ class NewsSerializer(serializers.ModelSerializer): description = serializers.SerializerMethodField() title = serializers.SerializerMethodField() template = serializers.SerializerMethodField() - state = serializers.SerializerMethodField() + state = serializers.CharField() created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') class Meta: From 786ee03d8e45d65d40be9c31f63900587410e070 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 21 Oct 2019 17:24:30 +0300 Subject: [PATCH 048/173] Fix --- apps/transfer/serializers/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 1aa0120a..a9d53cb8 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -11,7 +11,7 @@ class NewsSerializer(serializers.ModelSerializer): description = serializers.SerializerMethodField() title = serializers.SerializerMethodField() template = serializers.SerializerMethodField() - state = serializers.CharField() + state = serializers.IntegerField(source="state") created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') class Meta: From 50434620959502ff982d7a66a0f3919f18db664a Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 22 Oct 2019 17:11:45 +0500 Subject: [PATCH 049/173] prototype --- apps/transfer/serializers/news.py | 95 +++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index a9d53cb8..2d85bea1 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -6,12 +6,14 @@ from django.conf import settings class NewsSerializer(serializers.ModelSerializer): - image_url = serializers.SerializerMethodField() - preview_image_url = serializers.SerializerMethodField() - description = serializers.SerializerMethodField() - title = serializers.SerializerMethodField() - template = serializers.SerializerMethodField() - state = serializers.IntegerField(source="state") + # image_url = serializers.CharField() + # preview_image_url = serializers.CharField() + attachment_file_name = serializers.CharField(allow_null=True) + # body = serializers.CharField(allow_null=True, source="description") + body = serializers.CharField(allow_null=True) + title = serializers.CharField() + template = serializers.CharField() + state = serializers.CharField() created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') class Meta: @@ -20,57 +22,102 @@ class NewsSerializer(serializers.ModelSerializer): "created_at", "state", "template", - "image_url", - "preview_image_url", + # "image_url", + # "preview_image_url", + "attachment_file_name", "title", - "description", + "body", "slug", "news_type", "playlist" ) + def validate(self, data): + data["state"] = self.get_state(data) + data["template"] = self.get_template(data) + data["title"] = self.get_title(data) + data["description"] = self.get_description(data) + data["image_url"] = self.get_image_url(data) + data["preview_image_url"] = self.get_preview_image_url(data) + data.pop("attachment_file_name") + data.pop("body") + return data + def create(self, validated_data): - validated_data['state'] = self.get_state(validated_data) + # validated_data['state'] = self.get_state(validated_data) return News.objects.create(**validated_data) def get_template(self, obj): - if obj.template == "main": + if obj["template"] == "main": return News.MAIN - elif obj.template == "main.pdf.erb": + elif obj["template"] == "main.pdf.erb": return News.MAIN_PDF_ERB else: return News.NEWSPAPER + # if obj.template == "main": + # return News.MAIN + # elif obj.template == "main.pdf.erb": + # return News.MAIN_PDF_ERB + # else: + # return News.NEWSPAPER + def get_state(self, obj): - if obj.state == "published": + if obj["state"] == "published": return News.PUBLISHED - elif obj.state == "hidden": + elif obj["state"] == "hidden": return News.HIDDEN - elif obj.state == "published_exclusive": + elif obj["state"] == "published_exclusive": return News.PUBLISHED_EXCLUSIVE else: return News.WAITING + # if obj.state == "published": + # return News.PUBLISHED + # elif obj.state == "hidden": + # return News.HIDDEN + # elif obj.state == "published_exclusive": + # return News.PUBLISHED_EXCLUSIVE + # else: + # return News.WAITING + # django.db.utils.IntegrityError: "start" NOT NULL # def get_start(self, obj): # # return obj.created_at # return datetime.datetime.now().isoformat() def get_title(self, obj): - return f'{"en-GB": {obj.title}}' + tit = obj.get("title") + return {"en-GB": tit} + # return f'{"en-GB": {obj.title}}' def get_description(self, obj): - if not hasattr(obj, "body"): - return "" - else: - return obj.body + # return {"en-GB": {obj["body"]}} + desc = obj.get("body") + return {"en-GB": desc} + # if not hasattr(obj, "body"): + # return "" + # else: + # return obj.body def get_image_url(self, obj): - if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: + if not obj.get("attachment_file_name"): return "" - return obj.attachment_file_name + + if len(obj.get("attachment_file_name"))>200: + return obj.get("attachment_file_name")[:200] + return obj.get("attachment_file_name") + # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: + # return "" + # return obj.attachment_file_name def get_preview_image_url(self, obj): - if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: + if not obj.get("attachment_file_name"): return "" - return obj.attachment_file_name + + if len(obj.get("attachment_file_name"))>200: + return obj.get("attachment_file_name")[:200] + return obj.get("attachment_file_name") + # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: + # return "" + # return obj.attachment_file_name From 915683fa7bf4c7b9252bcb9212bd361a025c58a0 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 23 Oct 2019 09:16:29 +0300 Subject: [PATCH 050/173] Remove comments --- apps/news/transfer_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 24ef5a9e..deee2fe5 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -10,11 +10,9 @@ def transfer_news(): queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), playlist=Value(1, output_field=IntegerField())) - # queryset = PageTexts.objects.all().annotate(news_type=Value(news_type.id, output_field=IntegerField())) queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) queryset = queryset.annotate(template=F('page__template')) - serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() From fed2aae7f18c554e4e838a1af307bebb9e88db11 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 23 Oct 2019 09:30:36 +0300 Subject: [PATCH 051/173] Fix settings/base Add transfer_data to location --- apps/location/transfer_data.py | 25 +++++++++++++++++++++++++ project/settings/base.py | 3 +-- 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 apps/location/transfer_data.py diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py new file mode 100644 index 00000000..deee2fe5 --- /dev/null +++ b/apps/location/transfer_data.py @@ -0,0 +1,25 @@ +from transfer.serializers.news import NewsSerializer +from transfer.models import PageTexts +from news.models import NewsType +from django.db.models import Value, IntegerField, F +from pprint import pprint + + +def transfer_news(): + news_type, _ = NewsType.objects.get_or_create(name="News") + + queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), + playlist=Value(1, output_field=IntegerField())) + queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) + queryset = queryset.annotate(template=F('page__template')) + + serialized_data = NewsSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "news": [transfer_news] +} diff --git a/project/settings/base.py b/project/settings/base.py index bdfeb73f..69f601d7 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -74,8 +74,7 @@ PROJECT_APPS = [ 'favorites.apps.FavoritesConfig', 'rating.apps.RatingConfig', 'transfer.apps.TransferConfig', - 'tag.apps.TagConfig', -] + 'tag.apps.TagConfig' ] From 9e6d01b1bde2efc4f96531d7b959b4944c6f12f0 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 23 Oct 2019 12:18:25 +0500 Subject: [PATCH 052/173] prototype for transfer account --- apps/account/transfer_data.py | 24 ++++++++++++++ apps/transfer/management/commands/transfer.py | 3 +- apps/transfer/serializers/account.py | 33 +++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 apps/account/transfer_data.py create mode 100644 apps/transfer/serializers/account.py diff --git a/apps/account/transfer_data.py b/apps/account/transfer_data.py new file mode 100644 index 00000000..91f9051f --- /dev/null +++ b/apps/account/transfer_data.py @@ -0,0 +1,24 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from transfer.models import Profiles, Accounts +from transfer.serializers.account import UserSerializer + + +def transfer_user(): + # queryset = Profiles.objects.all() + # queryset = queryset.annotate(nickname=F("account__nickname")) + # queryset = queryset.annotate(email=F("account__email")) + + queryset = Accounts.objects.filter(confirmed_at__isnull=False) + + serialized_data = UserSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "account": [transfer_user] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 8018e38f..653760ac 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -10,7 +10,8 @@ class Command(BaseCommand): """ DATA_TYPES = [ 'dictionaries', - 'news' + 'news', + 'account' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/account.py b/apps/transfer/serializers/account.py new file mode 100644 index 00000000..cd66ef1f --- /dev/null +++ b/apps/transfer/serializers/account.py @@ -0,0 +1,33 @@ +from rest_framework import serializers +from account.models import User + + +class UserSerializer(serializers.ModelSerializer): + nickname = serializers.CharField() + email = serializers.CharField() + confirmed_at = serializers.DateTimeField() + + class Meta: + model = User + + fields = ( + "nickname", + "email", + "confirmed_at" + ) + + def validate(self, data): + data["username"] = self.get_username(data) + data["email_confirmed"] = self.get_email_confirmed(data) + data.pop("nickname") + data.pop("confirmed_at") + return data + + def create(self, validated_data): + User.objects.create(**validated_data) + + def get_email_confirmed(self, obj): + return True + + def get_username(self, obj): + return obj["nickname"] From 1264ba963fa589e00e25cbc97557dbba4fd2d433 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 23 Oct 2019 13:12:13 +0500 Subject: [PATCH 053/173] edited get_username because nickname from legacy is not unique --- apps/transfer/serializers/account.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/transfer/serializers/account.py b/apps/transfer/serializers/account.py index cd66ef1f..123b374b 100644 --- a/apps/transfer/serializers/account.py +++ b/apps/transfer/serializers/account.py @@ -24,10 +24,11 @@ class UserSerializer(serializers.ModelSerializer): return data def create(self, validated_data): + # использовать get_or_create User.objects.create(**validated_data) def get_email_confirmed(self, obj): return True def get_username(self, obj): - return obj["nickname"] + return obj["email"] From 8d0f4a8a00dbc42aa4e4302e1c1529756e8b4652 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 23 Oct 2019 14:17:34 +0500 Subject: [PATCH 054/173] added transfer for notification app --- apps/notification/transfer_data.py | 21 ++++++++++++ apps/transfer/management/commands/transfer.py | 3 +- apps/transfer/serializers/notification.py | 34 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 apps/notification/transfer_data.py create mode 100644 apps/transfer/serializers/notification.py diff --git a/apps/notification/transfer_data.py b/apps/notification/transfer_data.py new file mode 100644 index 00000000..3dd69f56 --- /dev/null +++ b/apps/notification/transfer_data.py @@ -0,0 +1,21 @@ +from transfer.serializers.notification import SubscriberSerializer +from notification.models import Subscriber +from transfer.models import EmailAddresses +from django.db.models import Value, IntegerField, F +from pprint import pprint + + +def transfer_subscriber(): + queryset = EmailAddresses.objects.filter(state="usable") + + serialized_data = SubscriberSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "subscriber": [transfer_subscriber] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 653760ac..18e0de8d 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -11,7 +11,8 @@ class Command(BaseCommand): DATA_TYPES = [ 'dictionaries', 'news', - 'account' + 'account', + 'subscriber' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py new file mode 100644 index 00000000..ccc07eff --- /dev/null +++ b/apps/transfer/serializers/notification.py @@ -0,0 +1,34 @@ +from rest_framework import serializers +from notification.models import Subscriber + + +class SubscriberSerializer(serializers.ModelSerializer): + email = serializers.CharField() + locale = serializers.CharField(allow_null=True) + country_code = serializers.CharField(allow_null=True) + + class Meta: + model = Subscriber + fields = ( + "email", + "locale", + "country_code" + ) + + def validate(self, data): + data["email"] = self.get_email(data) + data["locale"] = self.get_locale(data) + data["country_code"] = self.get_country_code(data) + return data + + def create(self, validated_data): + Subscriber.objects.create(**validated_data) + + def get_email(self, obj): + return obj["email"] + + def get_locale(self, obj): + return obj["locale"] + + def get_country_code(self, obj): + return obj["country_code"] From 6c9ddfbc9867d9090d0e2ce32d91898e4eacae2d Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 23 Oct 2019 14:42:08 +0500 Subject: [PATCH 055/173] added transfer for recipe app --- apps/recipe/transfer_data.py | 19 ++++++++ apps/transfer/management/commands/transfer.py | 3 +- apps/transfer/serializers/recipe.py | 46 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 apps/recipe/transfer_data.py create mode 100644 apps/transfer/serializers/recipe.py diff --git a/apps/recipe/transfer_data.py b/apps/recipe/transfer_data.py new file mode 100644 index 00000000..4a2e97d6 --- /dev/null +++ b/apps/recipe/transfer_data.py @@ -0,0 +1,19 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from transfer.models import PageTexts +from transfer.serializers.recipe import RecipeSerializer + + +def transfer_recipe(): + queryset = PageTexts.objects.filter(page__type="Recipe") + + serialized_data = RecipeSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "recipe": [transfer_recipe] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 18e0de8d..30a87dc4 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -12,7 +12,8 @@ class Command(BaseCommand): 'dictionaries', 'news', 'account', - 'subscriber' + 'subscriber', + 'recipe' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/recipe.py b/apps/transfer/serializers/recipe.py new file mode 100644 index 00000000..890d2b42 --- /dev/null +++ b/apps/transfer/serializers/recipe.py @@ -0,0 +1,46 @@ +from rest_framework import serializers +from recipe.models import Recipe + + +class RecipeSerializer(serializers.ModelSerializer): + body = serializers.CharField(allow_null=True) + title = serializers.CharField() + state = serializers.CharField() + created_at = serializers.DateTimeField(source="published_at", format='%m-%d-%Y %H:%M:%S') + + class Meta: + model = Recipe + fields = ( + "body", + "title", + "state", + "created_at" + ) + + def validate(self, data): + data["state"] = self.get_state(data) + data["title"] = self.get_title(data) + data["description"] = self.get_description(data) + data.pop("body") + return data + + def create(self, validated_data): + return Recipe.objects.create(**validated_data) + + def get_state(self, obj): + if obj["state"] == "published": + return Recipe.PUBLISHED + elif obj["state"] == "hidden": + return Recipe.HIDDEN + elif obj["state"] == "published_exclusive": + return Recipe.PUBLISHED_EXCLUSIVE + else: + return Recipe.WAITING + + def get_title(self, obj): + tit = obj.get("title") + return {"en-GB": tit} + + def get_description(self, obj): + desc = obj.get("body") + return {"en-GB": desc} From 110d0872da8a7a62b9baf9a7de3b7f770e629ab0 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 23 Oct 2019 16:49:15 +0500 Subject: [PATCH 056/173] added transfer for partner app --- apps/partner/transfer_data.py | 19 +++++++++++++++++++ apps/transfer/management/commands/transfer.py | 3 ++- apps/transfer/serializers/partner.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 apps/partner/transfer_data.py create mode 100644 apps/transfer/serializers/partner.py diff --git a/apps/partner/transfer_data.py b/apps/partner/transfer_data.py new file mode 100644 index 00000000..084fd7b1 --- /dev/null +++ b/apps/partner/transfer_data.py @@ -0,0 +1,19 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from transfer.models import EstablishmentBacklinks +from transfer.serializers.partner import PartnerSerializer + + +def transfer_partner(): + queryset = EstablishmentBacklinks.objects.filter(type="Partner") + + serialized_data = PartnerSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "partner": [transfer_partner] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 30a87dc4..90c828d4 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -13,7 +13,8 @@ class Command(BaseCommand): 'news', 'account', 'subscriber', - 'recipe' + 'recipe', + 'partner' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/partner.py b/apps/transfer/serializers/partner.py new file mode 100644 index 00000000..094418c6 --- /dev/null +++ b/apps/transfer/serializers/partner.py @@ -0,0 +1,15 @@ +from rest_framework import serializers +from partner.models import Partner + + +class PartnerSerializer(serializers.ModelSerializer): + backlink_url = serializers.CharField(source="url") + + class Meta: + model = Partner + fields = ( + "backlink_url", + ) + + def create(self, validated_data): + return Partner.objects.create(**validated_data) From 636b15a30c59892f559e9d827ac8dfe6e8d497e4 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 23 Oct 2019 14:59:40 +0300 Subject: [PATCH 057/173] mysql to docker-compose --- docker-compose.yml | 19 +++++++++++++++++++ project/settings/base.py | 2 +- requirements/base.txt | 5 ++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b446101..ac4619dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,20 @@ version: '3.5' services: + + # Legacy MySQL DB + mysql_db: + image: mysql:5.7 + ports: + - "3306:3306" + environment: + MYSQL_DATABASE: dev + MYSQL_USER: dev + MYSQL_PASSWORD: octosecret123 + MYSQL_ROOT_PASSWORD: rootPassword + volumes: + - .:/code + - gm-mysql_db:/var/lib/mysql + # PostgreSQL database db: build: @@ -86,6 +101,7 @@ services: - DB_USERNAME=postgres - DB_PASSWORD=postgres depends_on: + - mysql_db - db # - rabbitmq - redis @@ -99,6 +115,9 @@ services: - "8000:8000" volumes: + gm-mysql_db: + name: gm-mysql_db + gm-db: name: gm-db diff --git a/project/settings/base.py b/project/settings/base.py index 69f601d7..6c52afe6 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -155,7 +155,7 @@ DATABASES = { 'legacy': { 'ENGINE': 'django.db.backends.mysql', # 'HOST': '172.23.0.1', - 'HOST': '172.17.0.1', + 'HOST': '127.0.0.1', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', diff --git a/requirements/base.txt b/requirements/base.txt index 538cbe52..7de1517e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -33,11 +33,10 @@ django-elasticsearch-dsl>=7.0.0,<8.0.0 django-elasticsearch-dsl-drf==0.20.2 sentry-sdk==0.11.2 -<<<<<<< HEAD + mysqlclient==1.4.4 -======= + # temp solution redis==3.2.0 amqp>=2.4.0 celery==4.3.0rc2 ->>>>>>> develop From cb3ad6bb96f13d522d5c2b876fbdabb26f9f8897 Mon Sep 17 00:00:00 2001 From: michail Date: Wed, 23 Oct 2019 17:19:23 +0500 Subject: [PATCH 058/173] added transfer for gallery app --- .../migrations/0002_auto_20191023_1207.py | 20 ++++++++++++++++++ apps/gallery/models.py | 2 +- apps/gallery/transfer_data.py | 21 +++++++++++++++++++ apps/transfer/management/commands/transfer.py | 3 ++- apps/transfer/serializers/gallery.py | 15 +++++++++++++ 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 apps/gallery/migrations/0002_auto_20191023_1207.py create mode 100644 apps/gallery/transfer_data.py create mode 100644 apps/transfer/serializers/gallery.py diff --git a/apps/gallery/migrations/0002_auto_20191023_1207.py b/apps/gallery/migrations/0002_auto_20191023_1207.py new file mode 100644 index 00000000..64b93720 --- /dev/null +++ b/apps/gallery/migrations/0002_auto_20191023_1207.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-23 12:07 + +from django.db import migrations +import easy_thumbnails.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=easy_thumbnails.fields.ThumbnailerImageField(max_length=255, upload_to=utils.methods.image_path, verbose_name='Image file'), + ), + ] diff --git a/apps/gallery/models.py b/apps/gallery/models.py index 022a24e0..aec1c119 100644 --- a/apps/gallery/models.py +++ b/apps/gallery/models.py @@ -9,7 +9,7 @@ class Image(ProjectBaseMixin, ImageMixin): """Image model.""" image = ThumbnailerImageField(upload_to=image_path, - verbose_name=_('Image file')) + verbose_name=_('Image file'), max_length=255) class Meta: """Meta class.""" diff --git a/apps/gallery/transfer_data.py b/apps/gallery/transfer_data.py new file mode 100644 index 00000000..e008f1f0 --- /dev/null +++ b/apps/gallery/transfer_data.py @@ -0,0 +1,21 @@ +from django.db.models import Value, IntegerField, F +from pprint import pprint +from gallery.models import Image +from transfer.models import MercuryImages +from transfer.serializers.gallery import ImageSerializer + + +def transfer_gallery(): + queryset = MercuryImages.objects.all() + + serialized_data = ImageSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "gallery": [transfer_gallery] +} + diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 90c828d4..d4a1c213 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -14,7 +14,8 @@ class Command(BaseCommand): 'account', 'subscriber', 'recipe', - 'partner' + 'partner', + 'gallery' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/gallery.py b/apps/transfer/serializers/gallery.py new file mode 100644 index 00000000..273dd703 --- /dev/null +++ b/apps/transfer/serializers/gallery.py @@ -0,0 +1,15 @@ +from rest_framework import serializers +from gallery.models import Image + + +class ImageSerializer(serializers.ModelSerializer): + attachment_file_name = serializers.CharField(source="image") + + class Meta: + model = Image + fields = ( + "attachment_file_name", + ) + + def create(self, validated_data): + return Image.objects.create(**validated_data) From 938279bd7a04d9e54d36297ce7b557c2f686e9a9 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 23 Oct 2019 16:06:54 +0300 Subject: [PATCH 059/173] mysql to docker-compose.mysql --- docker-compose.mysql.yml | 127 +++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 19 ------ 2 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 docker-compose.mysql.yml diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml new file mode 100644 index 00000000..ac4619dd --- /dev/null +++ b/docker-compose.mysql.yml @@ -0,0 +1,127 @@ +version: '3.5' +services: + + # Legacy MySQL DB + mysql_db: + image: mysql:5.7 + ports: + - "3306:3306" + environment: + MYSQL_DATABASE: dev + MYSQL_USER: dev + MYSQL_PASSWORD: octosecret123 + MYSQL_ROOT_PASSWORD: rootPassword + volumes: + - .:/code + - gm-mysql_db:/var/lib/mysql + + # PostgreSQL database + db: + build: + context: ./_dockerfiles/db + dockerfile: Dockerfile + hostname: db + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + ports: + - "5436:5432" + volumes: + - gm-db:/var/lib/postgresql/data/ + + elasticsearch: + image: elasticsearch:7.3.1 + volumes: + - gm-esdata:/usr/share/elasticsearch/data + hostname: elasticsearch + ports: + - 9200:9200 + - 9300:9300 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - discovery.type=single-node + - xpack.security.enabled=false + + # Redis + redis: + image: redis:2.8.23 + ports: + - "6379:6379" + + # RabbitMQ + #rabbitmq: + # image: rabbitmq:latest + # ports: + # - "5672:5672" + + # Celery + worker: + build: . + command: ./run_celery.sh + environment: + - SETTINGS_CONFIGURATION=local + - DB_NAME=postgres + - DB_USERNAME=postgres + - DB_HOSTNAME=db + - DB_PORT=5432 + - DB_PASSWORD=postgres + volumes: + - .:/code + links: + - db +# - rabbitmq + - redis + + worker_beat: + build: . + command: ./run_celery_beat.sh + environment: + - SETTINGS_CONFIGURATION=local + - DB_NAME=postgres + - DB_USERNAME=postgres + - DB_HOSTNAME=db + - DB_PORT=5432 + - DB_PASSWORD=postgres + volumes: + - .:/code + links: + - db +# - rabbitmq + - redis + # App: G&M + gm_app: + build: . + command: python manage.py runserver 0.0.0.0:8000 + environment: + - SETTINGS_CONFIGURATION=local + - DB_HOSTNAME=db + - DB_PORT=5432 + - DB_NAME=postgres + - DB_USERNAME=postgres + - DB_PASSWORD=postgres + depends_on: + - mysql_db + - db +# - rabbitmq + - redis + - worker + - worker_beat + - elasticsearch + volumes: + - .:/code + - gm-media:/media-data + ports: + - "8000:8000" + +volumes: + gm-mysql_db: + name: gm-mysql_db + + gm-db: + name: gm-db + + gm-media: + name: gm-media + + gm-esdata: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ac4619dd..3b446101 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,5 @@ version: '3.5' services: - - # Legacy MySQL DB - mysql_db: - image: mysql:5.7 - ports: - - "3306:3306" - environment: - MYSQL_DATABASE: dev - MYSQL_USER: dev - MYSQL_PASSWORD: octosecret123 - MYSQL_ROOT_PASSWORD: rootPassword - volumes: - - .:/code - - gm-mysql_db:/var/lib/mysql - # PostgreSQL database db: build: @@ -101,7 +86,6 @@ services: - DB_USERNAME=postgres - DB_PASSWORD=postgres depends_on: - - mysql_db - db # - rabbitmq - redis @@ -115,9 +99,6 @@ services: - "8000:8000" volumes: - gm-mysql_db: - name: gm-mysql_db - gm-db: name: gm-db From 3ce7e186f7181156fb837b4267ba24644b3d3f87 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Oct 2019 14:01:38 +0300 Subject: [PATCH 060/173] migrate from legacy news --- apps/news/transfer_data.py | 16 ++-- apps/transfer/models.py | 1 + apps/transfer/serializers/news.py | 143 +++++++++--------------------- apps/utils/legacy_parser.py | 12 +++ docker-compose.mysql.yml | 20 ++++- project/settings/base.py | 2 +- project/settings/development.py | 2 +- requirements/base.txt | 1 + 8 files changed, 87 insertions(+), 110 deletions(-) create mode 100644 apps/utils/legacy_parser.py diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index deee2fe5..9d13e40e 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,23 +1,23 @@ -from transfer.serializers.news import NewsSerializer -from transfer.models import PageTexts -from news.models import NewsType from django.db.models import Value, IntegerField, F -from pprint import pprint + +from news.models import NewsType +from transfer.models import PageTexts +from transfer.serializers.news import NewsSerializer def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name="News") - queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), - playlist=Value(1, output_field=IntegerField())) - queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) + queryset = PageTexts.objects.filter(page__type="News").annotate( + news_type=Value(news_type.id, output_field=IntegerField()), + ) queryset = queryset.annotate(template=F('page__template')) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + print(f"News serializer errors: {serialized_data.errors}") data_types = { diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f469f698..f2a634bf 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -694,6 +694,7 @@ class ReviewTexts(MigrateMixin): managed = False db_table = 'review_texts' + class Comments(MigrateMixin): using = 'legacy' diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 2d85bea1..6cddb8dc 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,123 +1,68 @@ from rest_framework import serializers + from news.models import News -from uuid import uuid4 -import datetime -from django.conf import settings +from utils.legacy_parser import parse_legacy_content class NewsSerializer(serializers.ModelSerializer): - # image_url = serializers.CharField() - # preview_image_url = serializers.CharField() - attachment_file_name = serializers.CharField(allow_null=True) - # body = serializers.CharField(allow_null=True, source="description") + locale = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() template = serializers.CharField() state = serializers.CharField() - created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') + created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') class Meta: model = News fields = ( - "created_at", - "state", - "template", - # "image_url", - # "preview_image_url", - "attachment_file_name", - "title", - "body", - "slug", - "news_type", - "playlist" + 'created_at', + 'state', + 'template', + 'title', + 'body', + 'slug', + 'news_type', + 'locale', ) def validate(self, data): - data["state"] = self.get_state(data) - data["template"] = self.get_template(data) - data["title"] = self.get_title(data) - data["description"] = self.get_description(data) - data["image_url"] = self.get_image_url(data) - data["preview_image_url"] = self.get_preview_image_url(data) - data.pop("attachment_file_name") - data.pop("body") + data.update({ + 'state': self.get_state(data), + 'template': self.get_template(data), + 'title': self.get_title(data), + 'description': self.get_description(data), + }) + data.pop('body') + data.pop('locale') return data def create(self, validated_data): - # validated_data['state'] = self.get_state(validated_data) return News.objects.create(**validated_data) - def get_template(self, obj): - if obj["template"] == "main": - return News.MAIN - elif obj["template"] == "main.pdf.erb": - return News.MAIN_PDF_ERB - else: - return News.NEWSPAPER + @staticmethod + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + } + return templates.get(data['template'], News.MAIN) - # if obj.template == "main": - # return News.MAIN - # elif obj.template == "main.pdf.erb": - # return News.MAIN_PDF_ERB - # else: - # return News.NEWSPAPER + @staticmethod + def get_state(data): + states = { + 'new': News.PUBLISHED, + 'published': News.PUBLISHED, + 'hidden': News.HIDDEN, + } + return states.get(data['state'], News.WAITING) - def get_state(self, obj): - if obj["state"] == "published": - return News.PUBLISHED - elif obj["state"] == "hidden": - return News.HIDDEN - elif obj["state"] == "published_exclusive": - return News.PUBLISHED_EXCLUSIVE - else: - return News.WAITING + @staticmethod + def get_title(data): + return {data['locale']: data['title']} - # if obj.state == "published": - # return News.PUBLISHED - # elif obj.state == "hidden": - # return News.HIDDEN - # elif obj.state == "published_exclusive": - # return News.PUBLISHED_EXCLUSIVE - # else: - # return News.WAITING - - # django.db.utils.IntegrityError: "start" NOT NULL - # def get_start(self, obj): - # # return obj.created_at - # return datetime.datetime.now().isoformat() - - def get_title(self, obj): - tit = obj.get("title") - return {"en-GB": tit} - # return f'{"en-GB": {obj.title}}' - - def get_description(self, obj): - # return {"en-GB": {obj["body"]}} - desc = obj.get("body") - return {"en-GB": desc} - # if not hasattr(obj, "body"): - # return "" - # else: - # return obj.body - - def get_image_url(self, obj): - if not obj.get("attachment_file_name"): - return "" - - if len(obj.get("attachment_file_name"))>200: - return obj.get("attachment_file_name")[:200] - return obj.get("attachment_file_name") - # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: - # return "" - # return obj.attachment_file_name - - def get_preview_image_url(self, obj): - if not obj.get("attachment_file_name"): - return "" - - if len(obj.get("attachment_file_name"))>200: - return obj.get("attachment_file_name")[:200] - return obj.get("attachment_file_name") - # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: - # return "" - # return obj.attachment_file_name + @staticmethod + def get_description(data): + content = None + if data['body']: + content = parse_legacy_content(data['body']) + return {data['locale']: content} diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py new file mode 100644 index 00000000..a0e526f2 --- /dev/null +++ b/apps/utils/legacy_parser.py @@ -0,0 +1,12 @@ +import yaml + + +def parse_legacy_content(legacy_content): + clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' + content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) + result = '' + try: + result = content_dict['news_content']['value'] + except IndexError: + pass + return result diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index ac4619dd..cabb3808 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -14,6 +14,8 @@ services: volumes: - .:/code - gm-mysql_db:/var/lib/mysql + networks: + - mysql_network # PostgreSQL database db: @@ -29,6 +31,8 @@ services: - "5436:5432" volumes: - gm-db:/var/lib/postgresql/data/ + networks: + - database_network elasticsearch: image: elasticsearch:7.3.1 @@ -42,6 +46,8 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - discovery.type=single-node - xpack.security.enabled=false + networks: + - elasticsearch_network # Redis redis: @@ -113,6 +119,10 @@ services: - gm-media:/media-data ports: - "8000:8000" + networks: + - database_network + - mysql_network + - elasticsearch_network volumes: gm-mysql_db: @@ -124,4 +134,12 @@ volumes: gm-media: name: gm-media - gm-esdata: \ No newline at end of file + gm-esdata: + +networks: + database_network: + driver: bridge + mysql_network: + driver: bridge + elasticsearch_network: + driver: bridge diff --git a/project/settings/base.py b/project/settings/base.py index f5cce87d..f2001779 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -160,7 +160,7 @@ DATABASES = { 'legacy': { 'ENGINE': 'django.db.backends.mysql', # 'HOST': '172.23.0.1', - 'HOST': '127.0.0.1', + 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', diff --git a/project/settings/development.py b/project/settings/development.py index 59691818..3e9968f4 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,7 +19,7 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'localhost:9200' + 'hosts': 'elasticsearch:9200' } } diff --git a/requirements/base.txt b/requirements/base.txt index a4349bea..3a45dedc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -43,6 +43,7 @@ sorl-thumbnail==12.5.0 mysqlclient==1.4.4 +PyYAML==5.1.2 # temp solution redis==3.2.0 From 5826fb247586a8cbc07b851196035efa8a1d8571 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Oct 2019 14:49:59 +0300 Subject: [PATCH 061/173] mysql to docker-compose fix exception --- apps/utils/legacy_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py index a0e526f2..b69e2604 100644 --- a/apps/utils/legacy_parser.py +++ b/apps/utils/legacy_parser.py @@ -7,6 +7,6 @@ def parse_legacy_content(legacy_content): result = '' try: result = content_dict['news_content']['value'] - except IndexError: + except KeyError: pass return result From 6335abe1ea857693a290a8ef3fe9da0d04feb1aa Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Oct 2019 15:24:31 +0300 Subject: [PATCH 062/173] states dict fix --- apps/transfer/serializers/news.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 6cddb8dc..f47b4940 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -50,9 +50,11 @@ class NewsSerializer(serializers.ModelSerializer): @staticmethod def get_state(data): states = { - 'new': News.PUBLISHED, + 'new': News.WAITING, 'published': News.PUBLISHED, 'hidden': News.HIDDEN, + 'published_exclusive': News.PUBLISHED_EXCLUSIVE, + 'scheduled_exclusively': News.WAITING, } return states.get(data['state'], News.WAITING) From a667655c096409adce26e5e05a08027184cb4074 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Oct 2019 15:32:47 +0300 Subject: [PATCH 063/173] pprint --- apps/news/transfer_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 9d13e40e..d46c7613 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,3 +1,5 @@ +from pprint import pprint + from django.db.models import Value, IntegerField, F from news.models import NewsType @@ -17,7 +19,7 @@ def transfer_news(): if serialized_data.is_valid(): serialized_data.save() else: - print(f"News serializer errors: {serialized_data.errors}") + pprint(f"News serializer errors: {serialized_data.errors}") data_types = { From 449bf0f28ceaa9da7ed5fea9a9d9fadeb8bf6e58 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Oct 2019 16:11:26 +0300 Subject: [PATCH 064/173] comment duplicate method --- apps/location/transfer_data.py | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index deee2fe5..4f279daf 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -5,21 +5,21 @@ from django.db.models import Value, IntegerField, F from pprint import pprint -def transfer_news(): - news_type, _ = NewsType.objects.get_or_create(name="News") - - queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), - playlist=Value(1, output_field=IntegerField())) - queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) - queryset = queryset.annotate(template=F('page__template')) - - serialized_data = NewsSerializer(data=list(queryset.values()), many=True) - if serialized_data.is_valid(): - serialized_data.save() - else: - pprint(f"News serializer errors: {serialized_data.errors}") - - -data_types = { - "news": [transfer_news] -} +# def transfer_news(): +# news_type, _ = NewsType.objects.get_or_create(name="News") +# +# queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), +# playlist=Value(1, output_field=IntegerField())) +# queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) +# queryset = queryset.annotate(template=F('page__template')) +# +# serialized_data = NewsSerializer(data=list(queryset.values()), many=True) +# if serialized_data.is_valid(): +# serialized_data.save() +# else: +# pprint(f"News serializer errors: {serialized_data.errors}") +# +# +# data_types = { +# "news": [transfer_news] +# } From 287578eb9eb76fc12bdef6e61d137719ead74b0a Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 24 Oct 2019 17:08:00 +0300 Subject: [PATCH 065/173] base models and funcs --- apps/establishment/transfer.py | 96 +++++++++---------- apps/establishment/transfer_data.py | 18 ++++ apps/transfer/management/commands/transfer.py | 3 +- apps/transfer/serializers/establishment.py | 26 +++++ apps/transfer/serializers/news.py | 4 +- apps/utils/legacy_parser.py | 10 +- 6 files changed, 104 insertions(+), 53 deletions(-) create mode 100644 apps/establishment/transfer_data.py create mode 100644 apps/transfer/serializers/establishment.py diff --git a/apps/establishment/transfer.py b/apps/establishment/transfer.py index 604cc933..677245f9 100644 --- a/apps/establishment/transfer.py +++ b/apps/establishment/transfer.py @@ -10,7 +10,6 @@ field[1] - название поля в таблице legacy """ - card = { "EstablishmentType": { "data_type": "dictionaries", @@ -24,54 +23,54 @@ card = { } }, "Establishment": { - "data_type": "dictionaries", - "dependencies": ("Address", "Collection", "EstablishmentType"), - # нету аналогов для establishment_type, establishment_subtypes, schedule, comments, tags - "fields": { - "Establishments": { - "name": "name", - "slug": "slug", - "is_publish": ("state", "django.db.models.BooleanField") - }, - "relations": { - "EstablishmentInfos": { - "key": "establishment", - "fields": { - "website": "website", - "facebook": "facebook", - "twitter": "twitter", - "lafourchette": "lafourchette", - "booking": "booking_url" - } - }, + "data_type": "dictionaries", + "dependencies": ("Address", "Collection", "EstablishmentType"), + # нету аналогов для establishment_type, establishment_subtypes, schedule, comments, tags + "fields": { + "Establishments": { + "name": "name", + "slug": "slug", + "is_publish": ("state", "django.db.models.BooleanField") + }, + "relations": { + "EstablishmentInfos": { + "key": "establishment", + "fields": { + "website": "website", + "facebook": "facebook", + "twitter": "twitter", + "lafourchette": "lafourchette", + "booking": "booking_url" + } }, }, - # как работать с GenericRelation - ? - # как работать с ManyToManyField - ? "EstablishmentSubType", "schedule" - "relations": { - "Locations": [ - (("location", None), - ("Address", "address", None, None)), - ], - "Establishments": [#TODO правильно ли заполнена связь с EstablishmentType - ? - (("type", "type"), - ("EstablishmentType", "establishment_type", "id", "django.db.models.PositiveIntegerField")) - ] - # # "establishment_subtypes": "EstablishmentSubType", - # "collections": "Collection", - # # TODO: нашел schedules в legacy - # # "schedule": "Timetable", - # "award": "Award", - # # "tags": "MetaDataContent", - # "reviews": "Review", - # # "comments": "Comment", - # # "favorites": "Favorites" # В legacy этой таблицы не было - - } }, + # как работать с GenericRelation - ? + # как работать с ManyToManyField - ? "EstablishmentSubType", "schedule" + "relations": { + "Locations": [ + (("location", None), + ("Address", "address", None, None)), + ], + "Establishments": [ # TODO правильно ли заполнена связь с EstablishmentType - ? + (("type", "type"), + ("EstablishmentType", "establishment_type", "id", "django.db.models.PositiveIntegerField")) + ] + # # "establishment_subtypes": "EstablishmentSubType", + # "collections": "Collection", + # # TODO: нашел schedules в legacy + # # "schedule": "Timetable", + # "award": "Award", + # # "tags": "MetaDataContent", + # "reviews": "Review", + # # "comments": "Comment", + # # "favorites": "Favorites" # В legacy этой таблицы не было + + } + }, "Menu": { "data_type": "objects", - "dependencies": ("Establishment", ), + "dependencies": ("Establishment",), "fields": { "Menus": { "category": ("name", "django.db.models.TJSONField") @@ -86,12 +85,12 @@ card = { }, "ContactPhone": { "data_type": "objects", - "dependencies": ("Establishment", ), + "dependencies": ("Establishment",), "fields": { "Establishments": { "phone": "phone" }, - "relations": { #TODO правильно ли заполнена связь с Establishment - ? + "relations": { # TODO правильно ли заполнена связь с Establishment - ? "Establishments": [ (("id", "id"), ("Establishment", "establishment", "id", "django.db.models.PositiveIntegerField")) @@ -102,7 +101,7 @@ card = { "ContactEmail": { "data_type": "objects", - "dependencies": ("Establishment", ), + "dependencies": ("Establishment",), "fields": { "EstablishmentInfos": { "email": "email" @@ -117,5 +116,4 @@ card = { } } - -used_apps = ("review", "location", "collection", "main", "timetable", "favorites", "comment", ) \ No newline at end of file +used_apps = ("review", "location", "collection", "main", "timetable", "favorites", "comment",) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py new file mode 100644 index 00000000..5f96fcaa --- /dev/null +++ b/apps/establishment/transfer_data.py @@ -0,0 +1,18 @@ +from pprint import pprint + +from transfer.serializers.establishment import EstablishmentSerializer + + +def transfer_establishment(): + result = [] + + serialized_data = EstablishmentSerializer(data=result, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Establishment serializer errors: {serialized_data.errors}") + + +data_types = { + "establishment": [transfer_establishment] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index d4a1c213..7de18f5d 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -15,7 +15,8 @@ class Command(BaseCommand): 'subscriber', 'recipe', 'partner', - 'gallery' + 'gallery', + 'establishment', ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py new file mode 100644 index 00000000..fe81c835 --- /dev/null +++ b/apps/transfer/serializers/establishment.py @@ -0,0 +1,26 @@ +from rest_framework import serializers + +from establishment.models import Establishment + + +class EstablishmentSerializer(serializers.ModelSerializer): + + class Meta: + model = Establishment + fields = '__all__' + + def validate(self, data): + pass + # data.update({ + # 'state': self.get_state(data), + # 'template': self.get_template(data), + # 'title': self.get_title(data), + # 'description': self.get_description(data), + # }) + # data.pop('body') + # data.pop('locale') + # return data + + def create(self, validated_data): + pass + # return News.objects.create(**validated_data) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index f47b4940..c6a70fa9 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,7 +1,7 @@ from rest_framework import serializers from news.models import News -from utils.legacy_parser import parse_legacy_content +from utils.legacy_parser import parse_legacy_news_content class NewsSerializer(serializers.ModelSerializer): @@ -66,5 +66,5 @@ class NewsSerializer(serializers.ModelSerializer): def get_description(data): content = None if data['body']: - content = parse_legacy_content(data['body']) + content = parse_legacy_news_content(data['body']) return {data['locale']: content} diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py index b69e2604..7ec0747d 100644 --- a/apps/utils/legacy_parser.py +++ b/apps/utils/legacy_parser.py @@ -1,7 +1,7 @@ import yaml -def parse_legacy_content(legacy_content): +def parse_legacy_news_content(legacy_content): clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) result = '' @@ -10,3 +10,11 @@ def parse_legacy_content(legacy_content): except KeyError: pass return result + + +def parse_legacy_schedule_content(legacy_content): + clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' + content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) + result = '' + # TODO: вернуть валидные данные расписания для новой модели + return result \ No newline at end of file From 106ddb044e8fa4d183925885e036fdd1808a223d Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 07:48:29 +0300 Subject: [PATCH 066/173] Add country transfer --- apps/location/transfer_data.py | 24 ++++++++++-------------- apps/transfer/serializers/location.py | 26 ++++++++++++++++++++++++++ project/settings/base.py | 4 ++-- 3 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 apps/transfer/serializers/location.py diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index deee2fe5..04c2b163 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,25 +1,21 @@ -from transfer.serializers.news import NewsSerializer -from transfer.models import PageTexts -from news.models import NewsType -from django.db.models import Value, IntegerField, F +from transfer.serializers.location import CountrySerializer +from transfer.models import Cities from pprint import pprint -def transfer_news(): - news_type, _ = NewsType.objects.get_or_create(name="News") +def transfer_countries(): + queryset = Cities.objects.exclude(country_code_2__isnull=True).values_list("country_code_2", flat=True).distinct() - queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), - playlist=Value(1, output_field=IntegerField())) - queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) - queryset = queryset.annotate(template=F('page__template')) - - serialized_data = NewsSerializer(data=list(queryset.values()), many=True) + serialized_data = CountrySerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + pprint(f"Country serializer errors: {serialized_data.errors}") data_types = { - "news": [transfer_news] + "dictionaries": [ + transfer_countries, + + ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py new file mode 100644 index 00000000..f3fe44e6 --- /dev/null +++ b/apps/transfer/serializers/location.py @@ -0,0 +1,26 @@ +from rest_framework import serializers +from location.models import Country + + +class CountrySerializer(serializers.ModelSerializer): + country_code_2 = serializers.CharField() + + class Meta: + model = Country + fields = ( + "country_code_2", + ) + + def validate(self, data): + data["code"] = self.get_country_code(data) + del(data['country_code_2']) + return data + + def create(self, validated_data): + # Some countries already in database + country, _ = Country.objects.get_or_create(**validated_data) + return Country + + def get_country_code(self, obj): + print(f"OBJECT: {obj}") + return obj.get("country_code_2") diff --git a/project/settings/base.py b/project/settings/base.py index 6c52afe6..cd797225 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -154,8 +154,8 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - # 'HOST': '172.23.0.1', - 'HOST': '127.0.0.1', + 'HOST': '172.20.0.1', + # 'HOST': '127.0.0.1', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From c61d5321a99e674d7b1f2ae4039e719851fcf63c Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 09:52:35 +0300 Subject: [PATCH 067/173] Tmp commit --- apps/location/transfer_data.py | 16 +++++++++++- apps/transfer/management/commands/transfer.py | 4 ++- apps/transfer/serializers/location.py | 25 +++++++++++++++++-- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 04c2b163..745ff23c 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,4 +1,4 @@ -from transfer.serializers.location import CountrySerializer +from transfer.serializers.location import CountrySerializer, RegionSerializer from transfer.models import Cities from pprint import pprint @@ -12,10 +12,24 @@ def transfer_countries(): else: pprint(f"Country serializer errors: {serialized_data.errors}") +def transfer_regions(): + regions_without_subregion_queryset = Cities.objects.\ + exclude(subregion_code__isnull=False).values('region_code', 'country_code_2', 'subregion_code').distinct() + + serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) + if serialized_without_subregion.is_valid(): + serialized_without_subregion.save() + else: + pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") + + data_types = { "dictionaries": [ transfer_countries, + ], + "tmp": [ + transfer_regions ] } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index d4a1c213..96b4155a 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -15,7 +15,9 @@ class Command(BaseCommand): 'subscriber', 'recipe', 'partner', - 'gallery' + 'gallery', + + 'tmp' ] def handle(self, *args, **options): diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index f3fe44e6..a6d000ef 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from location.models import Country +from location.models import Country, Region class CountrySerializer(serializers.ModelSerializer): @@ -22,5 +22,26 @@ class CountrySerializer(serializers.ModelSerializer): return Country def get_country_code(self, obj): - print(f"OBJECT: {obj}") return obj.get("country_code_2") + + +class RegionSerializer(serializers.ModelSerializer): + region_code = serializers.CharField() + subregion_code = serializers.CharField() + country_code_2 = serializers.CharField() + + class Meta: + model = Region + fields = ( + "region_code", + "country_code_2", + "subregion_code" + ) + + def validate(self, data): + return data + + def create(self, validated_data): + # Some regions may be already in database + region, _ = Region.objects.get_or_create(**validated_data) + return region From bf375f33707bf662ae73805c434eb8cefda823a9 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 10:01:30 +0300 Subject: [PATCH 068/173] Remove errors Add version to timezonefinder --- apps/gallery/models.py | 6 ------ requirements/base.txt | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/gallery/models.py b/apps/gallery/models.py index 1a11bb35..bf5bc51c 100644 --- a/apps/gallery/models.py +++ b/apps/gallery/models.py @@ -27,13 +27,7 @@ class Image(ProjectBaseMixin, SORLImageMixin, PlatformMixin): blank=True, null=True, default=None, verbose_name=_('image orientation')) title = models.CharField(_('title'), max_length=255, default='') - -<<<<<<< HEAD - image = ThumbnailerImageField(upload_to=image_path, - verbose_name=_('Image file'), max_length=255) -======= objects = ImageQuerySet.as_manager() ->>>>>>> develop class Meta: """Meta class.""" diff --git a/requirements/base.txt b/requirements/base.txt index bd073996..b9fbcdfc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,7 +9,7 @@ fcm-django django-easy-select2 bootstrap-admin drf-yasg==1.16.0 -timezonefinder +timezonefinder==4.1.0 PySocks!=1.5.7,>=1.5.6; djangorestframework==3.9.4 From 95811c74373514f10cd59daa9c0c4ef3fe874fda Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Oct 2019 10:33:31 +0300 Subject: [PATCH 069/173] return to localhost --- project/settings/base.py | 4 ++-- project/settings/development.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/project/settings/base.py b/project/settings/base.py index 3d085978..752691ea 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,8 +159,8 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - # 'HOST': '172.23.0.1', - 'HOST': 'mysql_db', + 'HOST': '172.23.0.1', + # 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', diff --git a/project/settings/development.py b/project/settings/development.py index 6871fbf9..73e0095d 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,7 +19,8 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'elasticsearch:9200' + 'hosts': 'localhost:9200' + # 'hosts': 'elasticsearch:9200' } } From d79946633e1b2ce13760329678fe56851bee145d Mon Sep 17 00:00:00 2001 From: michail Date: Fri, 25 Oct 2019 13:17:35 +0500 Subject: [PATCH 070/173] fix get_description for recipe.py --- apps/transfer/serializers/recipe.py | 19 ++++++++++++++----- project/settings/base.py | 3 ++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/apps/transfer/serializers/recipe.py b/apps/transfer/serializers/recipe.py index 890d2b42..3948bbdd 100644 --- a/apps/transfer/serializers/recipe.py +++ b/apps/transfer/serializers/recipe.py @@ -1,8 +1,10 @@ from rest_framework import serializers from recipe.models import Recipe +from utils.legacy_parser import parse_legacy_content class RecipeSerializer(serializers.ModelSerializer): + locale = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() state = serializers.CharField() @@ -14,7 +16,8 @@ class RecipeSerializer(serializers.ModelSerializer): "body", "title", "state", - "created_at" + "created_at", + 'locale', ) def validate(self, data): @@ -22,6 +25,7 @@ class RecipeSerializer(serializers.ModelSerializer): data["title"] = self.get_title(data) data["description"] = self.get_description(data) data.pop("body") + data.pop("locale") return data def create(self, validated_data): @@ -38,9 +42,14 @@ class RecipeSerializer(serializers.ModelSerializer): return Recipe.WAITING def get_title(self, obj): - tit = obj.get("title") - return {"en-GB": tit} + # tit = obj.get("title") + # return {"en-GB": tit} + return {obj['locale']: obj['title']} def get_description(self, obj): - desc = obj.get("body") - return {"en-GB": desc} + # desc = obj.get("body") + # return {"en-GB": desc} + content = None + if obj['body']: + content = parse_legacy_content(obj['body']) + return {obj['locale']: content} diff --git a/project/settings/base.py b/project/settings/base.py index 752691ea..676a67d1 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,7 +159,8 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.23.0.1', + 'HOST': '172.17.0.1', + # 'HOST': '172.23.0.1', # 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', From 09647fa4d361b061a361668a36b3c849eab8c8c5 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Oct 2019 12:03:43 +0300 Subject: [PATCH 071/173] start transfer_data --- apps/establishment/transfer_data.py | 39 +++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 5f96fcaa..ebb13c4a 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -1,16 +1,45 @@ from pprint import pprint +from transfer.models import Establishments from transfer.serializers.establishment import EstablishmentSerializer def transfer_establishment(): result = [] - serialized_data = EstablishmentSerializer(data=result, many=True) - if serialized_data.is_valid(): - serialized_data.save() - else: - pprint(f"Establishment serializer errors: {serialized_data.errors}") + old_establishments = Establishments.objects.all().prefetch_related('establishmentinfos_set', 'schedules_set') + for item in old_establishments: + data = { + 'name': item.name, + 'slug': item.slug, + 'type': item.type, + 'location': item.location.id, + 'schedules': [], + } + info = item.establishmentinfos_set.first() + if info: + data.update({ + 'website': info.website, + 'facebook': info.facebook, + 'twitter': info.twitter, + 'lafourchette': info.lafourchette, + 'booking': info.booking_url, + }) + for schedule in item.schedules_set.all(): + data['schedules'].append({ + 'raw_timetable': schedule.timetable + }) + result.append(data) + + print('-' * 30) + print(len(result)) + pprint(result[0]) + + # serialized_data = EstablishmentSerializer(data=result, many=True) + # if serialized_data.is_valid(): + # serialized_data.save() + # else: + # pprint(f"Establishment serializer errors: {serialized_data.errors}") data_types = { From 0f595ee40bc5a2e3807107ab0b813b1657863c2a Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Oct 2019 12:07:02 +0300 Subject: [PATCH 072/173] merge migrations --- .../gallery/migrations/0004_merge_20191025_0906.py | 14 ++++++++++++++ apps/news/migrations/0029_merge_20191025_0906.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 apps/gallery/migrations/0004_merge_20191025_0906.py create mode 100644 apps/news/migrations/0029_merge_20191025_0906.py diff --git a/apps/gallery/migrations/0004_merge_20191025_0906.py b/apps/gallery/migrations/0004_merge_20191025_0906.py new file mode 100644 index 00000000..4372fa95 --- /dev/null +++ b/apps/gallery/migrations/0004_merge_20191025_0906.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0002_auto_20191023_1207'), + ('gallery', '0003_auto_20191003_1228'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0029_merge_20191025_0906.py b/apps/news/migrations/0029_merge_20191025_0906.py new file mode 100644 index 00000000..e5967a8d --- /dev/null +++ b/apps/news/migrations/0029_merge_20191025_0906.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0028_auto_20191024_1649'), + ('news', '0021_auto_20191021_1120'), + ] + + operations = [ + ] From abe8eae6551ad4c6a5cc029869f001d73009b7ad Mon Sep 17 00:00:00 2001 From: littlewolf Date: Fri, 25 Oct 2019 12:27:38 +0300 Subject: [PATCH 073/173] Tmp --- apps/location/transfer_data.py | 7 +++++-- apps/transfer/serializers/location.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 745ff23c..885c47e0 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -12,9 +12,13 @@ def transfer_countries(): else: pprint(f"Country serializer errors: {serialized_data.errors}") + def transfer_regions(): regions_without_subregion_queryset = Cities.objects.\ - exclude(subregion_code__isnull=False).values('region_code', 'country_code_2', 'subregion_code').distinct() + exclude(subregion_code__isnull=True).\ + exclude(region_code__isnull=True).\ + exclude(country_code_2__isnull=True).\ + values('region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) if serialized_without_subregion.is_valid(): @@ -23,7 +27,6 @@ def transfer_regions(): pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - data_types = { "dictionaries": [ transfer_countries, diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index a6d000ef..61df6baf 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -27,7 +27,7 @@ class CountrySerializer(serializers.ModelSerializer): class RegionSerializer(serializers.ModelSerializer): region_code = serializers.CharField() - subregion_code = serializers.CharField() + subregion_code = serializers.CharField(allow_null=True) country_code_2 = serializers.CharField() class Meta: @@ -39,6 +39,17 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): + data['code'] = data.pop('region_code') + if "country_code_2" in data and data["country_code_2"] is not None: + try: + country = Country.objects.get(code=data['country_code_2']) + data.country = country + del (data['country_code_2']) + + except Country.DoesNotExist as e: + print(f"Country error: {e}") + + del(data['subregion_code']) return data def create(self, validated_data): From 76d6e22fb92852772fa1a96a3ccbcf134eccccd3 Mon Sep 17 00:00:00 2001 From: michail Date: Fri, 25 Oct 2019 15:42:54 +0500 Subject: [PATCH 074/173] added transfer for advertisement --- .../migrations/0004_auto_20191025_0903.py | 18 +++++++++ apps/advertisement/models.py | 2 +- apps/advertisement/transfer_data.py | 20 ++++++++++ apps/transfer/management/commands/transfer.py | 3 +- apps/transfer/models.py | 20 ++++++++++ apps/transfer/serializers/advertisement.py | 39 +++++++++++++++++++ 6 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 apps/advertisement/migrations/0004_auto_20191025_0903.py create mode 100644 apps/advertisement/transfer_data.py create mode 100644 apps/transfer/serializers/advertisement.py diff --git a/apps/advertisement/migrations/0004_auto_20191025_0903.py b/apps/advertisement/migrations/0004_auto_20191025_0903.py new file mode 100644 index 00000000..f633d67c --- /dev/null +++ b/apps/advertisement/migrations/0004_auto_20191025_0903.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('advertisement', '0003_auto_20190919_1344'), + ] + + operations = [ + migrations.AlterField( + model_name='advertisement', + name='block_level', + field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'), + ), + ] diff --git a/apps/advertisement/models.py b/apps/advertisement/models.py index fdc25988..62b41428 100644 --- a/apps/advertisement/models.py +++ b/apps/advertisement/models.py @@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin): url = models.URLField(verbose_name=_('Ad URL')) width = models.PositiveIntegerField(verbose_name=_('Block width')) height = models.PositiveIntegerField(verbose_name=_('Block height')) - block_level = models.CharField(verbose_name=_('Block level'), max_length=10) + block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True) target_languages = models.ManyToManyField(Language) class Meta: diff --git a/apps/advertisement/transfer_data.py b/apps/advertisement/transfer_data.py new file mode 100644 index 00000000..3af16286 --- /dev/null +++ b/apps/advertisement/transfer_data.py @@ -0,0 +1,20 @@ +from pprint import pprint +from django.db.models import Value, IntegerField, F +from transfer.models import Ads +from transfer.serializers.advertisement import AdvertisementSerializer + + +def transfer_advertisement(): + queryset = Ads.objects.filter(href__isnull=False) + + serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "commercial": [transfer_advertisement] +} \ No newline at end of file diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index d4a1c213..68383877 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -15,7 +15,8 @@ class Command(BaseCommand): 'subscriber', 'recipe', 'partner', - 'gallery' + 'gallery', + 'commercial' ] def handle(self, *args, **options): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f2a634bf..f6dd2dbc 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -788,3 +788,23 @@ class PageMetadata(MigrateMixin): class Meta: managed = False db_table = 'page_metadata' + + +class Ads(MigrateMixin): + using = 'legacy' + + site_id = models.IntegerField(blank=True, null=True) + href = models.CharField(max_length=255, blank=True, null=True) + start_at = models.DateTimeField(blank=True, null=True) + expire_at = models.DateTimeField(blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'ads' \ No newline at end of file diff --git a/apps/transfer/serializers/advertisement.py b/apps/transfer/serializers/advertisement.py new file mode 100644 index 00000000..2e3a488f --- /dev/null +++ b/apps/transfer/serializers/advertisement.py @@ -0,0 +1,39 @@ +from rest_framework import serializers +from advertisement.models import Advertisement +import yaml + + +class AdvertisementSerializer(serializers.ModelSerializer): + href = serializers.CharField() + geometries = serializers.CharField(max_length=1024) + + class Meta: + model = Advertisement + fields = ( + "href", + "geometries" + ) + + def validate(self, data): + data["url"] = data["href"] + data["width"] = self.get_width(data["geometries"]) + data["height"] = self.get_height(data["geometries"]) + data.pop("href") + data.pop("geometries") + return data + + def create(self, validated_data): + return Advertisement.objects.create(**validated_data) + + def get_width(self, data): + data = self.parse_geometries(data) + return int(float(data["width"])) + + def get_height(self, data): + data = self.parse_geometries(data) + return int(float(data["height"])) + + def parse_geometries(self, geo_str): + clear_str = "!ruby/object:Paperclip::Geometry" + content_dict = yaml.safe_load(geo_str.replace(clear_str, '')) + return content_dict[':original'] \ No newline at end of file From e72e92244b8d465e40bff68cd3dfd2daebed48ff Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 08:42:00 +0300 Subject: [PATCH 075/173] Update country serializer --- apps/location/transfer_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 885c47e0..e4ac90a7 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,3 +1,5 @@ +from django.db.models import Q + from transfer.serializers.location import CountrySerializer, RegionSerializer from transfer.models import Cities from pprint import pprint @@ -15,9 +17,7 @@ def transfer_countries(): def transfer_regions(): regions_without_subregion_queryset = Cities.objects.\ - exclude(subregion_code__isnull=True).\ - exclude(region_code__isnull=True).\ - exclude(country_code_2__isnull=True).\ + exclude(Q(subregion_code__isnull=False) | Q(region_code__isnull=True) | Q(country_code_2__isnull=True)).\ values('region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) From 06b12f94279f8132daa8ca46e3b387d5da4a947a Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 09:18:45 +0300 Subject: [PATCH 076/173] Add country transfer --- apps/location/transfer_data.py | 9 +++++++-- apps/transfer/serializers/location.py | 20 +++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index e4ac90a7..db020e7a 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -6,7 +6,8 @@ from pprint import pprint def transfer_countries(): - queryset = Cities.objects.exclude(country_code_2__isnull=True).values_list("country_code_2", flat=True).distinct() + queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2=""))\ + .values_list("country_code_2", flat=True).distinct() serialized_data = CountrySerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): @@ -17,7 +18,11 @@ def transfer_countries(): def transfer_regions(): regions_without_subregion_queryset = Cities.objects.\ - exclude(Q(subregion_code__isnull=False) | Q(region_code__isnull=True) | Q(country_code_2__isnull=True)).\ + exclude(Q(subregion_code__isnull=False) | + Q(region_code__isnull=True) | + Q(country_code_2__isnull=True) | + Q(region_code="") | + Q(country_code_2="")).\ values('region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 61df6baf..b53cad6a 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -40,19 +40,21 @@ class RegionSerializer(serializers.ModelSerializer): def validate(self, data): data['code'] = data.pop('region_code') - if "country_code_2" in data and data["country_code_2"] is not None: - try: - country = Country.objects.get(code=data['country_code_2']) - data.country = country - del (data['country_code_2']) - - except Country.DoesNotExist as e: - print(f"Country error: {e}") + try: + country = Country.objects.get(code=data['country_code_2']) + except Exception as e: + raise ValueError(f"{data}: {e}") + data["country"] = country + del (data['country_code_2']) del(data['subregion_code']) + return data def create(self, validated_data): # Some regions may be already in database - region, _ = Region.objects.get_or_create(**validated_data) + try: + region, _ = Region.objects.get_or_create(**validated_data) + except Exception as e: + raise ValueError(f"{validated_data}: {e}") return region From cd724de65fc532a0643c2f1ae6272f0c7b83d050 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 09:19:34 +0300 Subject: [PATCH 077/173] Move countries to dictionaries --- apps/location/transfer_data.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index db020e7a..a5a659c0 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -35,9 +35,6 @@ def transfer_regions(): data_types = { "dictionaries": [ transfer_countries, - - ], - "tmp": [ - transfer_regions + transfer_regions, ] } From 8bcf4c9ca00472952250e6d2649880c15e304146 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sat, 26 Oct 2019 09:35:09 +0300 Subject: [PATCH 078/173] Add child regions --- apps/location/transfer_data.py | 25 ++++++++++++++++++++++++- apps/transfer/serializers/location.py | 17 +++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index a5a659c0..cbf5954f 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,6 +1,6 @@ from django.db.models import Q -from transfer.serializers.location import CountrySerializer, RegionSerializer +from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer from transfer.models import Cities from pprint import pprint @@ -31,10 +31,33 @@ def transfer_regions(): else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") + regions_with_subregion_queryset = Cities.objects. \ + exclude(Q(subregion_code__isnull=True) | + Q(subregion_code="") | + Q(region_code__isnull=True) | + Q(country_code_2__isnull=True) | + Q(region_code="") | + Q(country_code_2="")). \ + values('region_code', 'country_code_2', 'subregion_code').distinct() + + serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) + if serialized_with_subregion.is_valid(): + serialized_with_subregion.save() + else: + pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + + + +def transfer_cities(): + pass + data_types = { "dictionaries": [ transfer_countries, + ], + "tmp": [ transfer_regions, + # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index b53cad6a..654dc246 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -39,11 +39,20 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): - data['code'] = data.pop('region_code') + if "subregion" in data and data["subregion"] is not None: + try: + parent_region = Region.objects.get(code=data['region_code']) + except Exception as e: + raise ValueError(f"Parent region error with {data}: {e}") + data['parent_region'] = parent_region + data['code'] = data.pop('subregion_code') + + else: + data['code'] = data.pop('region_code') try: country = Country.objects.get(code=data['country_code_2']) except Exception as e: - raise ValueError(f"{data}: {e}") + raise ValueError(f"Country error with {data}: {e}") data["country"] = country del (data['country_code_2']) @@ -58,3 +67,7 @@ class RegionSerializer(serializers.ModelSerializer): except Exception as e: raise ValueError(f"{validated_data}: {e}") return region + + +class CitySerializer(serializers.ModelSerializer): + pass From 16f06ce2f1bbc134c1a39d08087bf98a72a2e78e Mon Sep 17 00:00:00 2001 From: alex Date: Sat, 26 Oct 2019 22:02:38 +0300 Subject: [PATCH 079/173] fix news slug --- apps/news/models.py | 6 +++--- apps/transfer/serializers/news.py | 3 +++ apps/transfer/serializers/recipe.py | 4 ++-- apps/utils/slug_generator.py | 15 +++++++++++++++ 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 apps/utils/slug_generator.py diff --git a/apps/news/models.py b/apps/news/models.py index dbb2f5bf..306e4906 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -1,11 +1,12 @@ """News app models.""" -from django.db import models from django.contrib.contenttypes import fields as generic +from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework.reverse import reverse -from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin + from rating.models import Rating +from utils.models import BaseAttributes, TJSONField, TranslatedFieldsMixin, ProjectBaseMixin class NewsType(models.Model): @@ -217,7 +218,6 @@ class NewsGalleryQuerySet(models.QuerySet): class NewsGallery(models.Model): - news = models.ForeignKey(News, null=True, related_name='news_gallery', on_delete=models.CASCADE, diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index c6a70fa9..e46c5b4a 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -2,10 +2,12 @@ from rest_framework import serializers from news.models import News from utils.legacy_parser import parse_legacy_news_content +from utils.slug_generator import generate_unique_slug class NewsSerializer(serializers.ModelSerializer): locale = serializers.CharField() + slug = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() template = serializers.CharField() @@ -27,6 +29,7 @@ class NewsSerializer(serializers.ModelSerializer): def validate(self, data): data.update({ + 'slug': generate_unique_slug(News, data['slug']), 'state': self.get_state(data), 'template': self.get_template(data), 'title': self.get_title(data), diff --git a/apps/transfer/serializers/recipe.py b/apps/transfer/serializers/recipe.py index 3948bbdd..11698ba1 100644 --- a/apps/transfer/serializers/recipe.py +++ b/apps/transfer/serializers/recipe.py @@ -1,6 +1,6 @@ from rest_framework import serializers from recipe.models import Recipe -from utils.legacy_parser import parse_legacy_content +from utils.legacy_parser import parse_legacy_news_content class RecipeSerializer(serializers.ModelSerializer): @@ -51,5 +51,5 @@ class RecipeSerializer(serializers.ModelSerializer): # return {"en-GB": desc} content = None if obj['body']: - content = parse_legacy_content(obj['body']) + content = parse_legacy_news_content(obj['body']) return {obj['locale']: content} diff --git a/apps/utils/slug_generator.py b/apps/utils/slug_generator.py new file mode 100644 index 00000000..fa1ec772 --- /dev/null +++ b/apps/utils/slug_generator.py @@ -0,0 +1,15 @@ +from django.utils.text import slugify + + +def generate_unique_slug(klass, text): + """ + return unique slug if origin slug is exist. + eg: `foo-bar` => `foo-bar-1` + """ + origin_slug = slugify(text) + unique_slug = origin_slug + numb = 1 + while klass.objects.filter(slug=unique_slug).exists(): + unique_slug = f'{origin_slug}-{numb}' + numb += 1 + return unique_slug From 7eb241ffeda79a35029d902323ea8f74307f3bda Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 12:47:45 +0300 Subject: [PATCH 080/173] Tmp commit --- apps/location/models.py | 1 + apps/location/transfer.py | 6 -- apps/location/transfer_data.py | 19 ++++-- apps/transfer/serializers/location.py | 83 ++++++++++++++++++++++++--- 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index da645de6..c562d300 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -21,6 +21,7 @@ class Country(TranslatedFieldsMixin, SVGImageMixin, ProjectBaseMixin): low_price = models.IntegerField(default=25, verbose_name=_('Low price')) high_price = models.IntegerField(default=50, verbose_name=_('High price')) languages = models.ManyToManyField(Language, verbose_name=_('Languages')) + old_id = models.IntegerField(null=True, blank=True, default=None) @property def country_id(self): diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 74099303..9ad1c15a 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -112,12 +112,6 @@ card = { "fields": { "name": "name", } - }, - "CityPhotos": { - "key": "city", - "fields": { - "coordinates": "geometries" - } } } }, diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index cbf5954f..681efa51 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,4 +1,4 @@ -from django.db.models import Q +from django.db.models import Q, F from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer from transfer.models import Cities @@ -47,17 +47,26 @@ def transfer_regions(): pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") - def transfer_cities(): - pass + queryset = Cities.objects.exclude(Q(region_code__isnull=True) | + Q(country_code_2__isnull=True) | + Q(region_code="") | + Q(country_code_2="")).only("name", "latitude", "longitude", "subregion_code", + "country_code_2", "zip_code", "is_island") + + serialized_data = CitySerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"City serializer errors: {serialized_data.errors}") data_types = { "dictionaries": [ - transfer_countries, + transfer_regions, ], "tmp": [ - transfer_regions, + transfer_countries, # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 654dc246..93437187 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -1,25 +1,33 @@ from rest_framework import serializers -from location.models import Country, Region +from location.models import Country, Region, City class CountrySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() + id = serializers.IntegerField() class Meta: model = Country fields = ( + "id", "country_code_2", ) def validate(self, data): data["code"] = self.get_country_code(data) del(data['country_code_2']) + + data['old_id'] = data.pop('id') + return data def create(self, validated_data): # Some countries already in database - country, _ = Country.objects.get_or_create(**validated_data) - return Country + try: + country = Country.objects.get(code=validated_data['code']) + except Country.DoesNotExists: + country = Country.objects.create(**validated_data) + return country def get_country_code(self, obj): return obj.get("country_code_2") @@ -41,14 +49,15 @@ class RegionSerializer(serializers.ModelSerializer): def validate(self, data): if "subregion" in data and data["subregion"] is not None: try: - parent_region = Region.objects.get(code=data['region_code']) + parent_region = Region.objects.get(code=str(data['region_code'])) except Exception as e: raise ValueError(f"Parent region error with {data}: {e}") data['parent_region'] = parent_region - data['code'] = data.pop('subregion_code') + data['code'] = str(data.pop('subregion_code')) else: - data['code'] = data.pop('region_code') + data['code'] = str(data.pop('region_code')) + try: country = Country.objects.get(code=data['country_code_2']) except Exception as e: @@ -70,4 +79,64 @@ class RegionSerializer(serializers.ModelSerializer): class CitySerializer(serializers.ModelSerializer): - pass + country_code_2 = serializers.CharField() + subregion_code = serializers.CharField() + zip_code = serializers.CharField() + is_island = serializers.IntegerField() + name = serializers.CharField() + + class Meta: + model = City + fields = ( + "country_code_2", + "subregion_code", + "zip_code", + "is_island", + "name", + ) + + def validate(self, data): + data = self.set_is_island(data) + data = self.set_code(data) + data = self.set_zip_code(data) + data = self.set_relations(data) + return data + + def create(self, validated_data): + return City.objects.create(**validated_data) + + def set_is_island(self, data): + data['is_island'] = True if "is_island" in data \ + and data['is_island'] is not None \ + and data['is_island'] > 0 \ + else False + return data + + def set_code(self, data): + data['code'] = data.pop('country_code_2') + return data + + def set_relations(self, data): + try: + region = Region.objects.get(code=data['code']) + except: + try: + region = Region.objects.get(code=data['subregion_code']) + except Exception as e: + raise ValueError(f"Region not found with {data}: {e}") + data['region'] = region + del(data['subregion_code']) + + try: + country = Country.objects.get(code=data['code']) + except Exception as e: + raise ValueError(f"Region not found with {data}: {e}") + + data['country'] = country + + return data + + def set_zip_code(self, data): + data['postal_code'] = data.pop('zip_code') + return data + From 25a2edf17d2f9e731a230cbe6d8c63b7ce102f53 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 27 Oct 2019 16:23:30 +0300 Subject: [PATCH 081/173] establishment fin --- .../migrations/0044_establishment_old_id.py | 18 +++ apps/establishment/models.py | 6 +- apps/establishment/transfer_data.py | 64 ++++++-- .../migrations/0005_auto_20191027_0756.py | 20 +++ apps/transfer/models.py | 24 ++- apps/transfer/serializers/establishment.py | 151 ++++++++++++++++-- apps/utils/legacy_parser.py | 40 ++++- 7 files changed, 282 insertions(+), 41 deletions(-) create mode 100644 apps/establishment/migrations/0044_establishment_old_id.py create mode 100644 apps/gallery/migrations/0005_auto_20191027_0756.py diff --git a/apps/establishment/migrations/0044_establishment_old_id.py b/apps/establishment/migrations/0044_establishment_old_id.py new file mode 100644 index 00000000..9f630c2b --- /dev/null +++ b/apps/establishment/migrations/0044_establishment_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 07:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0043_establishment_currency'), + ] + + operations = [ + migrations.AddField( + model_name='establishment', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 304ea2a6..3fc623a9 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -14,6 +14,8 @@ from django.db.models import When, Case, F, ExpressionWrapper, Subquery, Q from django.utils import timezone from django.utils.translation import gettext_lazy as _ from phonenumber_field.modelfields import PhoneNumberField +from pytz import timezone as ptz +from timezone_field import TimeZoneField from collection.models import Collection from location.models import Address @@ -21,7 +23,6 @@ from main.models import Award, Currency from review.models import Review from utils.models import (ProjectBaseMixin, TJSONField, URLImageMixin, TranslatedFieldsMixin, BaseAttributes) -from timezone_field import TimeZoneField # todo: establishment type&subtypes check @@ -296,6 +297,7 @@ class EstablishmentQuerySet(models.QuerySet): class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): """Establishment model.""" + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) name = models.CharField(_('name'), max_length=255, default='') name_translated = models.CharField(_('Transliterated name'), max_length=255, default='') @@ -436,7 +438,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): @property def works_now(self): """ Is establishment working now """ - now_at_est_tz = datetime.now(tz=self.tz) + now_at_est_tz = datetime.now(tz=ptz(self.tz)) current_week = now_at_est_tz.weekday() schedule_for_today = self.schedule.filter(weekday=current_week).first() if schedule_for_today is None or schedule_for_today.closed_at is None or schedule_for_today.opening_at is None: diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index ebb13c4a..5716087d 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -7,15 +7,41 @@ from transfer.serializers.establishment import EstablishmentSerializer def transfer_establishment(): result = [] - old_establishments = Establishments.objects.all().prefetch_related('establishmentinfos_set', 'schedules_set') + old_establishments = Establishments.objects.exclude( + type='Wineyard', + ).prefetch_related( + 'establishmentinfos_set', + 'schedules_set', + 'descriptions_set', + ) for item in old_establishments: data = { + 'old_id': item.id, 'name': item.name, + 'name_translated': item.index_name, 'slug': item.slug, 'type': item.type, - 'location': item.location.id, - 'schedules': [], + 'phone': item.phone, + 'created': item.created_at, + 'description': [], + 'tz': None, + 'website': None, + 'facebook': None, + 'twitter': None, + 'lafourchette': None, + 'booking': None, + 'schedules': None, + 'location': None, + 'email': None, } + + if item.location: + data.update({ + 'location': item.location.id, + 'tz': item.location.timezone, + }) + + # Инфо info = item.establishmentinfos_set.first() if info: data.update({ @@ -24,22 +50,30 @@ def transfer_establishment(): 'twitter': info.twitter, 'lafourchette': info.lafourchette, 'booking': info.booking_url, + 'email': info.email, }) - for schedule in item.schedules_set.all(): - data['schedules'].append({ - 'raw_timetable': schedule.timetable + + # Время работы + schedule = item.schedules_set.first() + if schedule: + data.update({ + 'schedules': schedule.timetable, }) + + # Описание + descriptions = item.descriptions_set.all() + for description in descriptions: + data['description'].append({ + description.locale: description.text, + }) + result.append(data) - print('-' * 30) - print(len(result)) - pprint(result[0]) - - # serialized_data = EstablishmentSerializer(data=result, many=True) - # if serialized_data.is_valid(): - # serialized_data.save() - # else: - # pprint(f"Establishment serializer errors: {serialized_data.errors}") + serialized_data = EstablishmentSerializer(data=result, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Establishment serializer errors: {serialized_data.errors}") data_types = { diff --git a/apps/gallery/migrations/0005_auto_20191027_0756.py b/apps/gallery/migrations/0005_auto_20191027_0756.py new file mode 100644 index 00000000..d60745f1 --- /dev/null +++ b/apps/gallery/migrations/0005_auto_20191027_0756.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-27 07:56 + +from django.db import migrations +import sorl.thumbnail.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0004_merge_20191025_0906'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'), + ), + ] diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f6dd2dbc..896d478b 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -277,7 +277,6 @@ class Collections(MigrateMixin): db_table = 'collections' - # class CollectionEvents(MigrateMixin): # using = 'legacy' # @@ -296,7 +295,7 @@ class Collections(MigrateMixin): # class CollectionEventAvailabilities(MigrateMixin): # using = 'legacy' -#TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts +# TODO: collection_event - внешний ключ к CollectionEvents, которая имеет внешний ключ к Accounts # collection_event = models.ForeignKey('CollectionEvents', models.DO_NOTHING, blank=True, null=True) # establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) @@ -369,6 +368,7 @@ class GuideFilters(MigrateMixin): managed = False db_table = 'guide_filters' + # # class GuideSections(MigrateMixin): # using = 'legacy' @@ -447,6 +447,20 @@ class Establishments(MigrateMixin): db_table = 'establishments' +class Descriptions(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + locale = models.CharField(max_length=5, blank=True, null=True) + text = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'descriptions' + + # class EstablishmentAssets(MigrateMixin): # using = 'legacy' # @@ -540,7 +554,7 @@ class EstablishmentInfos(MigrateMixin): # # establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) - #TODO: модели Merchandises нету в гугл таблице Check Migrations +# TODO: модели Merchandises нету в гугл таблице Check Migrations # merchandise = models.ForeignKey('Merchandises', models.DO_NOTHING, blank=True, null=True) # gifted = models.IntegerField(blank=True, null=True) @@ -664,7 +678,7 @@ class Reviews(MigrateMixin): aasm_state = models.CharField(max_length=255, blank=True, null=True) reviewer_id = models.IntegerField() priority = models.IntegerField(blank=True, null=True) - #TODO: модель Products в postgres закомментирована + # TODO: модель Products в postgres закомментирована # product = models.ForeignKey("Products", models.DO_NOTHING, blank=True, null=True) received_at = models.DateTimeField(blank=True, null=True) reviewer_name = models.CharField(max_length=255, blank=True, null=True) @@ -807,4 +821,4 @@ class Ads(MigrateMixin): class Meta: managed = False - db_table = 'ads' \ No newline at end of file + db_table = 'ads' diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index fe81c835..171d7837 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -1,26 +1,149 @@ +from django.core.exceptions import MultipleObjectsReturned, ValidationError +from django.db import transaction from rest_framework import serializers -from establishment.models import Establishment +from establishment.models import Establishment, ContactEmail, ContactPhone, EstablishmentType +from location.models import Address +from timetable.models import Timetable +from utils.legacy_parser import parse_legacy_schedule_content +from utils.slug_generator import generate_unique_slug class EstablishmentSerializer(serializers.ModelSerializer): + slug = serializers.CharField(allow_null=True, allow_blank=True) + type = serializers.CharField() + description = serializers.ListField( + allow_null=True, + child=serializers.DictField( + child=serializers.CharField(allow_null=True) + ) + ) + schedules = serializers.CharField(allow_null=True, allow_blank=True) + location = serializers.IntegerField(allow_null=True) + email = serializers.CharField(allow_null=True, allow_blank=True) + phone = serializers.CharField(allow_null=True, allow_blank=True) + website = serializers.CharField(allow_null=True, allow_blank=True) + facebook = serializers.CharField(allow_null=True, allow_blank=True) + twitter = serializers.CharField(allow_null=True, allow_blank=True) + booking = serializers.CharField(allow_null=True, allow_blank=True) + tz = serializers.CharField(allow_null=True, allow_blank=True) + created = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') class Meta: model = Establishment - fields = '__all__' + fields = ( + 'created', + 'old_id', # + + 'name', # + + 'name_translated', # + + 'tz', # + + 'website', # + + 'facebook', # + + 'twitter', # + + 'lafourchette', # + + 'booking', # + + 'type', # + см в JIRA + 'slug', # + сгенерировать уникальный слаг + 'description', # + (разобрал в transfer_data) + 'schedules', # + разобрать RUBY словать (2 варианта) + 'location', # + получить новые объекты Address по old_id + 'email', # + создать объект для ContactEmail + 'phone', # + создать объект для ContactPhone + ) def validate(self, data): - pass - # data.update({ - # 'state': self.get_state(data), - # 'template': self.get_template(data), - # 'title': self.get_title(data), - # 'description': self.get_description(data), - # }) - # data.pop('body') - # data.pop('locale') - # return data + data.update({ + 'slug': generate_unique_slug(Establishment, data['slug'] if data['slug'] else data['name']), + 'address_id': self.get_address(data['location']), + 'establishment_type_id': self.get_type(data), + }) + data.pop('location') + data.pop('type') + return data + @transaction.atomic def create(self, validated_data): - pass - # return News.objects.create(**validated_data) + email = validated_data.pop('email') + phone = validated_data.pop('phone') + schedules = validated_data.pop('schedules') + + establishment = Establishment.objects.create(**validated_data) + if email: + ContactEmail.objects.get_or_create( + email=email, + establishment=establishment, + ) + if phone: + ContactPhone.objects.get_or_create( + phone=phone, + establishment=establishment, + ) + if schedules: + new_schedules = self.get_schedules(schedules) + for schedule in new_schedules: + establishment.schedule.add(schedule) + establishment.save() + + return establishment + + @staticmethod + def get_address(address): + # return Address.objects.filter(old_id=address).first() + return None + + @staticmethod + def get_type(data): + types = { + 'Restaurant': EstablishmentType.RESTAURANT, + 'Shop': EstablishmentType.ARTISAN, + } + obj, _ = EstablishmentType.objects.get_or_create(index_name=types[data['type']]) + return obj.id + + @staticmethod + def get_schedules(schedules): + result = [] + legacy_dict = parse_legacy_schedule_content(schedules) + + weekdays = { + 'su': Timetable.SUNDAY, + 'mo': Timetable.MONDAY, + 'tu': Timetable.THURSDAY, + 'we': Timetable.WEDNESDAY, + 'th': Timetable.THURSDAY, + 'fr': Timetable.FRIDAY, + 'sa': Timetable.SATURDAY, + } + + for key, val in legacy_dict.items(): + payload = { + 'weekday': weekdays[key], + 'lunch_start': None, + 'lunch_end': None, + 'dinner_start': None, + 'dinner_end': None, + 'opening_at': None, + 'closed_at': None, + } + if val['morning']: + payload.update({ + 'lunch_start': val['morning']['start'], + 'lunch_end': val['morning']['end'], + }) + if val['afternoon']: + payload.update({ + 'dinner_start': val['afternoon']['start'], + 'dinner_end': val['afternoon']['end'], + }) + + try: + obj, _ = Timetable.objects.get_or_create(**payload) + except ValidationError: + obj = None + except MultipleObjectsReturned: + obj = Timetable.objects.filter(**payload).first() + + if obj: + result.append(obj) + + return result diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py index 7ec0747d..7fbbd133 100644 --- a/apps/utils/legacy_parser.py +++ b/apps/utils/legacy_parser.py @@ -1,3 +1,5 @@ +from pprint import pprint + import yaml @@ -13,8 +15,36 @@ def parse_legacy_news_content(legacy_content): def parse_legacy_schedule_content(legacy_content): - clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' - content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) - result = '' - # TODO: вернуть валидные данные расписания для новой модели - return result \ No newline at end of file + initial = legacy_content + + s1 = "!ruby/object:ActionController::Parameters" + s2 = "!ruby/hash:ActiveSupport::HashWithIndifferentAccess" + + for _ in (s1, s2): + legacy_content = legacy_content.replace(_, "") + content_dict = yaml.safe_load(legacy_content) + + if s1 not in initial or s2 not in initial: + return yaml.safe_load(legacy_content) + + result = {} + for k, v in content_dict.items(): + try: + if v["parameters"]["afternoon"]: + afternoon = v["parameters"]["afternoon"]["parameters"] + else: + afternoon = None + + if v["parameters"]["morning"]: + morning = v["parameters"]["morning"]["parameters"] + else: + morning = None + + data = {"afternoon": afternoon, "morning": morning} + result.update({k: data}) + except KeyError: + print('--------' * 7) + pprint(initial) + raise KeyError + + return result From 3b68a4dc19bc38577e1dc11b380d1e51bf71c541 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 27 Oct 2019 16:47:08 +0300 Subject: [PATCH 082/173] establishment fin fix 1 --- apps/establishment/transfer_data.py | 4 ++-- apps/transfer/serializers/establishment.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 5716087d..bc0ae034 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -23,7 +23,7 @@ def transfer_establishment(): 'type': item.type, 'phone': item.phone, 'created': item.created_at, - 'description': [], + 'description': {}, 'tz': None, 'website': None, 'facebook': None, @@ -63,7 +63,7 @@ def transfer_establishment(): # Описание descriptions = item.descriptions_set.all() for description in descriptions: - data['description'].append({ + data['description'].update({ description.locale: description.text, }) diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index 171d7837..f5babf7b 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -12,11 +12,9 @@ from utils.slug_generator import generate_unique_slug class EstablishmentSerializer(serializers.ModelSerializer): slug = serializers.CharField(allow_null=True, allow_blank=True) type = serializers.CharField() - description = serializers.ListField( + description = serializers.DictField( allow_null=True, - child=serializers.DictField( - child=serializers.CharField(allow_null=True) - ) + child=serializers.CharField(allow_null=True), ) schedules = serializers.CharField(allow_null=True, allow_blank=True) location = serializers.IntegerField(allow_null=True) From 0db2638eb8c547eabad8d685d80b73b426016be2 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 17:23:04 +0300 Subject: [PATCH 083/173] Fix country queryset --- apps/location/models.py | 2 ++ apps/location/transfer_data.py | 22 +++++++++++++++------- apps/transfer/serializers/location.py | 19 ++++++++++++++----- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index c562d300..abf58da2 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -44,6 +44,7 @@ class Region(models.Model): blank=True, default=None, on_delete=models.CASCADE) country = models.ForeignKey( Country, verbose_name=_('country'), on_delete=models.CASCADE) + old_id = models.IntegerField(null=True, blank=True, default=None) class Meta: """Meta class.""" @@ -77,6 +78,7 @@ class City(models.Model): _('postal code'), max_length=10, default='', help_text=_('Ex.: 350018')) is_island = models.BooleanField(_('is island'), default=False) + old_id = models.IntegerField(null=True, blank=True, default=None) objects = CityQuerySet.as_manager() diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 681efa51..6b001e46 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,4 +1,4 @@ -from django.db.models import Q, F +from django.db.models import Q, QuerySet from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer from transfer.models import Cities @@ -6,10 +6,18 @@ from pprint import pprint def transfer_countries(): - queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2=""))\ - .values_list("country_code_2", flat=True).distinct() + # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \ + # .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2") + queryset = Cities.objects.raw("SELECT cities.id, cities.country_code_2 " + "FROM cities " + "WHERE NOT ((cities.country_code_2 IS NULL " + "OR (cities.country_code_2 = '' " + "AND cities.country_code_2 IS NOT NULL))) " + "GROUP BY cities.country_code_2") - serialized_data = CountrySerializer(data=list(queryset.values()), many=True) + queryset = [vars(query) for query in queryset] + + serialized_data = CountrySerializer(data=queryset, many=True) if serialized_data.is_valid(): serialized_data.save() else: @@ -23,7 +31,7 @@ def transfer_regions(): Q(country_code_2__isnull=True) | Q(region_code="") | Q(country_code_2="")).\ - values('region_code', 'country_code_2', 'subregion_code').distinct() + values('id', 'region_code', 'country_code_2', 'subregion_code').distinct() serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) if serialized_without_subregion.is_valid(): @@ -51,7 +59,7 @@ def transfer_cities(): queryset = Cities.objects.exclude(Q(region_code__isnull=True) | Q(country_code_2__isnull=True) | Q(region_code="") | - Q(country_code_2="")).only("name", "latitude", "longitude", "subregion_code", + Q(country_code_2="")).only("id", "name", "latitude", "longitude", "subregion_code", "country_code_2", "zip_code", "is_island") serialized_data = CitySerializer(data=list(queryset.values()), many=True) @@ -63,10 +71,10 @@ def transfer_cities(): data_types = { "dictionaries": [ - transfer_regions, ], "tmp": [ transfer_countries, + # transfer_regions, # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 93437187..c9795957 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -25,7 +25,7 @@ class CountrySerializer(serializers.ModelSerializer): # Some countries already in database try: country = Country.objects.get(code=validated_data['code']) - except Country.DoesNotExists: + except Country.DoesNotExist: country = Country.objects.create(**validated_data) return country @@ -37,13 +37,15 @@ class RegionSerializer(serializers.ModelSerializer): region_code = serializers.CharField() subregion_code = serializers.CharField(allow_null=True) country_code_2 = serializers.CharField() + id = serializers.IntegerField() class Meta: model = Region fields = ( "region_code", "country_code_2", - "subregion_code" + "subregion_code", + "id" ) def validate(self, data): @@ -67,14 +69,16 @@ class RegionSerializer(serializers.ModelSerializer): del (data['country_code_2']) del(data['subregion_code']) + data['old_id'] = data.pop("id") + return data def create(self, validated_data): # Some regions may be already in database try: - region, _ = Region.objects.get_or_create(**validated_data) - except Exception as e: - raise ValueError(f"{validated_data}: {e}") + region = Region.objects.get(old_id=validated_data['old_id']) + except Region.DoesNotExist: + region = Region.objects.create(**validated_data) return region @@ -84,6 +88,7 @@ class CitySerializer(serializers.ModelSerializer): zip_code = serializers.CharField() is_island = serializers.IntegerField() name = serializers.CharField() + id = serializers.IntegerField() class Meta: model = City @@ -93,6 +98,7 @@ class CitySerializer(serializers.ModelSerializer): "zip_code", "is_island", "name", + "id" ) def validate(self, data): @@ -100,6 +106,7 @@ class CitySerializer(serializers.ModelSerializer): data = self.set_code(data) data = self.set_zip_code(data) data = self.set_relations(data) + data = self.set_old_id(data) return data def create(self, validated_data): @@ -140,3 +147,5 @@ class CitySerializer(serializers.ModelSerializer): data['postal_code'] = data.pop('zip_code') return data + def set_old_id(self, data): + data['old_id'] = data.pop('id') From 07f226604e813b172a8265b66608e08d857207f7 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 20:31:41 +0300 Subject: [PATCH 084/173] Fix regions without subregion --- apps/location/transfer_data.py | 68 +++++++++++++++------------ apps/transfer/serializers/location.py | 55 +++++++++++++--------- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 6b001e46..657a8853 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -8,12 +8,12 @@ from pprint import pprint def transfer_countries(): # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \ # .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2") - queryset = Cities.objects.raw("SELECT cities.id, cities.country_code_2 " - "FROM cities " - "WHERE NOT ((cities.country_code_2 IS NULL " - "OR (cities.country_code_2 = '' " - "AND cities.country_code_2 IS NOT NULL))) " - "GROUP BY cities.country_code_2") + queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 + FROM cities + WHERE NOT ((cities.country_code_2 IS NULL + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL))) + GROUP BY cities.country_code_2""") queryset = [vars(query) for query in queryset] @@ -25,34 +25,40 @@ def transfer_countries(): def transfer_regions(): - regions_without_subregion_queryset = Cities.objects.\ - exclude(Q(subregion_code__isnull=False) | - Q(region_code__isnull=True) | - Q(country_code_2__isnull=True) | - Q(region_code="") | - Q(country_code_2="")).\ - values('id', 'region_code', 'country_code_2', 'subregion_code').distinct() + regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities + WHERE NOT ((cities.subregion_code IS NOT NULL + OR cities.region_code IS NULL + OR cities.country_code_2 IS NULL + OR (cities.region_code = '' + AND cities.region_code IS NOT NULL) + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL))) + GROUP BY region_code""") - serialized_without_subregion = RegionSerializer(data=list(regions_without_subregion_queryset.values()), many=True) + regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] + + serialized_without_subregion = RegionSerializer(data=regions_without_subregion_queryset, many=True) if serialized_without_subregion.is_valid(): serialized_without_subregion.save() else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - - regions_with_subregion_queryset = Cities.objects. \ - exclude(Q(subregion_code__isnull=True) | - Q(subregion_code="") | - Q(region_code__isnull=True) | - Q(country_code_2__isnull=True) | - Q(region_code="") | - Q(country_code_2="")). \ - values('region_code', 'country_code_2', 'subregion_code').distinct() - - serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) - if serialized_with_subregion.is_valid(): - serialized_with_subregion.save() - else: - pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + # + # regions_with_subregion_queryset = Cities.objects. \ + # exclude(Q(subregion_code__isnull=True) | + # Q(subregion_code="") | + # Q(region_code__isnull=True) | + # Q(country_code_2__isnull=True) | + # Q(region_code="") | + # Q(country_code_2="")). \ + # values('region_code', 'country_code_2', 'subregion_code').distinct() + # + # serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) + # if serialized_with_subregion.is_valid(): + # serialized_with_subregion.save() + # else: + # pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") def transfer_cities(): @@ -71,10 +77,10 @@ def transfer_cities(): data_types = { "dictionaries": [ + transfer_countries, ], "tmp": [ - transfer_countries, - # transfer_regions, + transfer_regions, # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index c9795957..4895b85e 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -49,27 +49,9 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): - if "subregion" in data and data["subregion"] is not None: - try: - parent_region = Region.objects.get(code=str(data['region_code'])) - except Exception as e: - raise ValueError(f"Parent region error with {data}: {e}") - data['parent_region'] = parent_region - data['code'] = str(data.pop('subregion_code')) - - else: - data['code'] = str(data.pop('region_code')) - - try: - country = Country.objects.get(code=data['country_code_2']) - except Exception as e: - raise ValueError(f"Country error with {data}: {e}") - data["country"] = country - - del (data['country_code_2']) - del(data['subregion_code']) - - data['old_id'] = data.pop("id") + data = self.set_code(data) + data = self.set_country(data) + data = self.set_old_id(data) return data @@ -81,6 +63,37 @@ class RegionSerializer(serializers.ModelSerializer): region = Region.objects.create(**validated_data) return region + def set_code(self, data): + if "subregion_code" in data and data["subregion_code"] is not None: + try: + parent_region = Region.objects.get(code=str(data['region_code'])) + except Exception as e: + raise ValueError(f"Parent region error with {data}: {e}") + + data['parent_region'] = parent_region + data['code'] = data.pop('subregion_code') + + else: + data['code'] = data.pop('region_code') + del(data['subregion_code']) + + return data + + def set_country(self, data): + try: + country = Country.objects.get(code=data['country_code_2']) + except Exception as e: + raise ValueError(f"Country error with {data}: {e}") + + data["country"] = country + del(data['country_code_2']) + + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + class CitySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() From 0d30543664ff0c5bc4c610ad8cf6395640f40fb4 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Sun, 27 Oct 2019 21:43:34 +0300 Subject: [PATCH 085/173] Fix regions with subregions --- apps/location/transfer_data.py | 57 +++++++++++++++------------ apps/transfer/serializers/location.py | 5 ++- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 657a8853..3dc52c05 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -25,16 +25,16 @@ def transfer_countries(): def transfer_regions(): - regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, - cities.country_code_2, cities.subregion_code - FROM cities - WHERE NOT ((cities.subregion_code IS NOT NULL - OR cities.region_code IS NULL - OR cities.country_code_2 IS NULL - OR (cities.region_code = '' - AND cities.region_code IS NOT NULL) - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL))) + regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities + WHERE NOT ((cities.subregion_code IS NOT NULL + OR cities.region_code IS NULL + OR cities.country_code_2 IS NULL + OR (cities.region_code = '' + AND cities.region_code IS NOT NULL) + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL))) GROUP BY region_code""") regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] @@ -44,21 +44,28 @@ def transfer_regions(): serialized_without_subregion.save() else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - # - # regions_with_subregion_queryset = Cities.objects. \ - # exclude(Q(subregion_code__isnull=True) | - # Q(subregion_code="") | - # Q(region_code__isnull=True) | - # Q(country_code_2__isnull=True) | - # Q(region_code="") | - # Q(country_code_2="")). \ - # values('region_code', 'country_code_2', 'subregion_code').distinct() - # - # serialized_with_subregion = RegionSerializer(data=list(regions_with_subregion_queryset.values()), many=True) - # if serialized_with_subregion.is_valid(): - # serialized_with_subregion.save() - # else: - # pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") + + regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities + WHERE NOT ((cities.subregion_code IS NULL + OR (cities.subregion_code = '' + OR cities.region_code IS NULL + OR cities.country_code_2 IS NULL + OR (cities.region_code = '' + AND cities.region_code IS NOT NULL) + OR (cities.country_code_2 = '' + AND cities.country_code_2 IS NOT NULL)))) + AND cities.subregion_code in (SELECT region_code FROM cities) + GROUP BY region_code""") + + regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] + + serialized_with_subregion = RegionSerializer(data=regions_with_subregion_queryset, many=True) + if serialized_with_subregion.is_valid(): + serialized_with_subregion.save() + else: + pprint(f"Child regions serializer errors: {serialized_with_subregion.errors}") def transfer_cities(): diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 4895b85e..00c2e834 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -49,9 +49,9 @@ class RegionSerializer(serializers.ModelSerializer): ) def validate(self, data): + data = self.set_old_id(data) data = self.set_code(data) data = self.set_country(data) - data = self.set_old_id(data) return data @@ -61,6 +61,8 @@ class RegionSerializer(serializers.ModelSerializer): region = Region.objects.get(old_id=validated_data['old_id']) except Region.DoesNotExist: region = Region.objects.create(**validated_data) + except Exception as e: + raise ValueError(f"REGION ERROR: {validated_data}: {e}") return region def set_code(self, data): @@ -72,6 +74,7 @@ class RegionSerializer(serializers.ModelSerializer): data['parent_region'] = parent_region data['code'] = data.pop('subregion_code') + del(data['region_code']) else: data['code'] = data.pop('region_code') From 255b1ad5f1c193ead0a6c786970762f0ca131f35 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 00:15:37 +0300 Subject: [PATCH 086/173] Fix queryset conditions --- apps/location/transfer.py | 10 +---- apps/location/transfer_data.py | 61 ++++++++++++++------------- apps/transfer/serializers/location.py | 28 +++++------- 3 files changed, 44 insertions(+), 55 deletions(-) diff --git a/apps/location/transfer.py b/apps/location/transfer.py index 9ad1c15a..fe4e8084 100644 --- a/apps/location/transfer.py +++ b/apps/location/transfer.py @@ -101,18 +101,10 @@ card = { "dependencies": ("Country", "Region"), "fields": { "Cities": { - "coordinates": (("lat", "latitude"), ("long", "longitude")), + "name": "name", "code": "country_code_2", "postal_code": "zip_code", "is_island": ("is_island", "is_island", "django.db.models.Boolean") - }, - "relations": { - "CityNames": { - "key": "city", - "fields": { - "name": "name", - } - } } }, "relations": { diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 3dc52c05..6d6861cc 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -6,13 +6,10 @@ from pprint import pprint def transfer_countries(): - # queryset = Cities.objects.exclude(Q(country_code_2__isnull=True) | Q(country_code_2="")) \ - # .only("id", "country_code_2").annotate(Count("country_code_2")).order_by("country_code_2") queryset = Cities.objects.raw("""SELECT cities.id, cities.country_code_2 - FROM cities - WHERE NOT ((cities.country_code_2 IS NULL - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL))) + FROM cities WHERE + country_code_2 IS NOT NULL AND + country_code_2 != "" GROUP BY cities.country_code_2""") queryset = [vars(query) for query in queryset] @@ -27,14 +24,13 @@ def transfer_countries(): def transfer_regions(): regions_without_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, cities.country_code_2, cities.subregion_code - FROM cities - WHERE NOT ((cities.subregion_code IS NOT NULL - OR cities.region_code IS NULL - OR cities.country_code_2 IS NULL - OR (cities.region_code = '' - AND cities.region_code IS NOT NULL) - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL))) + FROM cities WHERE + (subregion_code IS NULL OR + subregion_code = "") AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" GROUP BY region_code""") regions_without_subregion_queryset = [vars(query) for query in regions_without_subregion_queryset] @@ -45,18 +41,24 @@ def transfer_regions(): else: pprint(f"Parent regions serializer errors: {serialized_without_subregion.errors}") - regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, - cities.country_code_2, cities.subregion_code - FROM cities - WHERE NOT ((cities.subregion_code IS NULL - OR (cities.subregion_code = '' - OR cities.region_code IS NULL - OR cities.country_code_2 IS NULL - OR (cities.region_code = '' - AND cities.region_code IS NOT NULL) - OR (cities.country_code_2 = '' - AND cities.country_code_2 IS NOT NULL)))) - AND cities.subregion_code in (SELECT region_code FROM cities) + regions_with_subregion_queryset = Cities.objects.raw("""SELECT cities.id, cities.region_code, + cities.country_code_2, cities.subregion_code + FROM cities WHERE + subregion_code IS NOT NULL AND + subregion_code != "" AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + AND cities.subregion_code in ( + SELECT region_code FROM cities WHERE + (subregion_code IS NULL OR + subregion_code = "") AND + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + ) GROUP BY region_code""") regions_with_subregion_queryset = [vars(query) for query in regions_with_subregion_queryset] @@ -72,12 +74,13 @@ def transfer_cities(): queryset = Cities.objects.exclude(Q(region_code__isnull=True) | Q(country_code_2__isnull=True) | Q(region_code="") | - Q(country_code_2="")).only("id", "name", "latitude", "longitude", "subregion_code", - "country_code_2", "zip_code", "is_island") + Q(country_code_2="")).only("id", "name", "region_code", + "country_code_2", "zip_code", "is_island").distinct() serialized_data = CitySerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): - serialized_data.save() + # serialized_data.save() + print(serialized_data.validated_data) else: pprint(f"City serializer errors: {serialized_data.errors}") diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 00c2e834..5a95e223 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -35,7 +35,7 @@ class CountrySerializer(serializers.ModelSerializer): class RegionSerializer(serializers.ModelSerializer): region_code = serializers.CharField() - subregion_code = serializers.CharField(allow_null=True) + subregion_code = serializers.CharField(allow_null=True, allow_blank=True) country_code_2 = serializers.CharField() id = serializers.IntegerField() @@ -52,7 +52,6 @@ class RegionSerializer(serializers.ModelSerializer): data = self.set_old_id(data) data = self.set_code(data) data = self.set_country(data) - return data def create(self, validated_data): @@ -66,7 +65,8 @@ class RegionSerializer(serializers.ModelSerializer): return region def set_code(self, data): - if "subregion_code" in data and data["subregion_code"] is not None: + print(data) + if "subregion_code" in data and data["subregion_code"] is not None and data["subregion_code"].strip() != "": try: parent_region = Region.objects.get(code=str(data['region_code'])) except Exception as e: @@ -75,7 +75,6 @@ class RegionSerializer(serializers.ModelSerializer): data['parent_region'] = parent_region data['code'] = data.pop('subregion_code') del(data['region_code']) - else: data['code'] = data.pop('region_code') del(data['subregion_code']) @@ -100,7 +99,6 @@ class RegionSerializer(serializers.ModelSerializer): class CitySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() - subregion_code = serializers.CharField() zip_code = serializers.CharField() is_island = serializers.IntegerField() name = serializers.CharField() @@ -110,7 +108,6 @@ class CitySerializer(serializers.ModelSerializer): model = City fields = ( "country_code_2", - "subregion_code", "zip_code", "is_island", "name", @@ -118,11 +115,11 @@ class CitySerializer(serializers.ModelSerializer): ) def validate(self, data): + data = self.set_old_id(data) + data = self.set_relations(data) data = self.set_is_island(data) data = self.set_code(data) data = self.set_zip_code(data) - data = self.set_relations(data) - data = self.set_old_id(data) return data def create(self, validated_data): @@ -141,18 +138,14 @@ class CitySerializer(serializers.ModelSerializer): def set_relations(self, data): try: - region = Region.objects.get(code=data['code']) - except: - try: - region = Region.objects.get(code=data['subregion_code']) - except Exception as e: - raise ValueError(f"Region not found with {data}: {e}") + region = Region.objects.get(old_id=data['old_id']) + except Region.DoesNotExist as e: + raise ValueError(f"Region not found with {data}: {e}") data['region'] = region - del(data['subregion_code']) try: - country = Country.objects.get(code=data['code']) - except Exception as e: + country = Country.objects.get(old_id=data['old_id']) + except Country.DoesNotExist as e: raise ValueError(f"Region not found with {data}: {e}") data['country'] = country @@ -165,3 +158,4 @@ class CitySerializer(serializers.ModelSerializer): def set_old_id(self, data): data['old_id'] = data.pop('id') + return data From 25d58a8bc50d385f1decd5b839a4d5b349ac2920 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 07:19:17 +0300 Subject: [PATCH 087/173] Fix and add city transfer --- apps/location/models.py | 1 - apps/location/transfer_data.py | 24 ++++++++++++++---------- apps/transfer/serializers/location.py | 26 +++++++++++++++++++------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index abf58da2..0b650bce 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -79,7 +79,6 @@ class City(models.Model): is_island = models.BooleanField(_('is island'), default=False) old_id = models.IntegerField(null=True, blank=True, default=None) - objects = CityQuerySet.as_manager() class Meta: diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index 6d6861cc..dca432e7 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -71,16 +71,20 @@ def transfer_regions(): def transfer_cities(): - queryset = Cities.objects.exclude(Q(region_code__isnull=True) | - Q(country_code_2__isnull=True) | - Q(region_code="") | - Q(country_code_2="")).only("id", "name", "region_code", - "country_code_2", "zip_code", "is_island").distinct() + queryset = Cities.objects.raw("""SELECT cities.id, cities.name, cities.country_code_2, cities.zip_code, + cities.is_island, cities.region_code, cities.subregion_code + FROM cities WHERE + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "" + """) - serialized_data = CitySerializer(data=list(queryset.values()), many=True) + queryset = [vars(query) for query in queryset] + + serialized_data = CitySerializer(data=queryset, many=True) if serialized_data.is_valid(): - # serialized_data.save() - print(serialized_data.validated_data) + serialized_data.save() else: pprint(f"City serializer errors: {serialized_data.errors}") @@ -88,9 +92,9 @@ def transfer_cities(): data_types = { "dictionaries": [ transfer_countries, + transfer_regions, + transfer_cities ], "tmp": [ - transfer_regions, - # transfer_cities ] } diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 5a95e223..11d809ec 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -99,8 +99,10 @@ class RegionSerializer(serializers.ModelSerializer): class CitySerializer(serializers.ModelSerializer): country_code_2 = serializers.CharField() - zip_code = serializers.CharField() - is_island = serializers.IntegerField() + region_code = serializers.CharField() + subregion_code = serializers.CharField(allow_null=True, allow_blank=True) + zip_code = serializers.CharField(allow_null=True, allow_blank=True) + is_island = serializers.IntegerField(allow_null=True) name = serializers.CharField() id = serializers.IntegerField() @@ -108,6 +110,8 @@ class CitySerializer(serializers.ModelSerializer): model = City fields = ( "country_code_2", + "region_code", + "subregion_code", "zip_code", "is_island", "name", @@ -133,27 +137,35 @@ class CitySerializer(serializers.ModelSerializer): return data def set_code(self, data): - data['code'] = data.pop('country_code_2') + data['code'] = data.pop('region_code') return data def set_relations(self, data): try: - region = Region.objects.get(old_id=data['old_id']) + region = Region.objects.filter(code=data['region_code']).first() except Region.DoesNotExist as e: - raise ValueError(f"Region not found with {data}: {e}") + try: + region = Region.objects.filter(code=data['subregion_code']).first() + except Region.DoesNotExist as e: + raise ValueError(f"Region not found with {data}: {e}") + data['region'] = region + del(data['subregion_code']) try: - country = Country.objects.get(old_id=data['old_id']) + country = Country.objects.get(code=data['country_code_2']) except Country.DoesNotExist as e: - raise ValueError(f"Region not found with {data}: {e}") + raise ValueError(f"Country not found with {data}: {e}") data['country'] = country + del(data['country_code_2']) return data def set_zip_code(self, data): data['postal_code'] = data.pop('zip_code') + if data['postal_code'] is None: + data['postal_code'] = "" return data def set_old_id(self, data): From 281d57996ef68e27035343211052491a8842ed66 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 28 Oct 2019 09:24:56 +0300 Subject: [PATCH 088/173] fix null timezone --- apps/establishment/transfer_data.py | 7 ++++--- apps/transfer/serializers/establishment.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index bc0ae034..4670e6cf 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -1,5 +1,7 @@ from pprint import pprint +from django.db.models import Q + from transfer.models import Establishments from transfer.serializers.establishment import EstablishmentSerializer @@ -8,7 +10,8 @@ def transfer_establishment(): result = [] old_establishments = Establishments.objects.exclude( - type='Wineyard', + Q(type='Wineyard') | + Q(location__timezone__isnull=True) ).prefetch_related( 'establishmentinfos_set', 'schedules_set', @@ -24,14 +27,12 @@ def transfer_establishment(): 'phone': item.phone, 'created': item.created_at, 'description': {}, - 'tz': None, 'website': None, 'facebook': None, 'twitter': None, 'lafourchette': None, 'booking': None, 'schedules': None, - 'location': None, 'email': None, } diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index f5babf7b..9735f5fb 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -3,7 +3,7 @@ from django.db import transaction from rest_framework import serializers from establishment.models import Establishment, ContactEmail, ContactPhone, EstablishmentType -from location.models import Address +# from location.models import Address from timetable.models import Timetable from utils.legacy_parser import parse_legacy_schedule_content from utils.slug_generator import generate_unique_slug @@ -24,7 +24,7 @@ class EstablishmentSerializer(serializers.ModelSerializer): facebook = serializers.CharField(allow_null=True, allow_blank=True) twitter = serializers.CharField(allow_null=True, allow_blank=True) booking = serializers.CharField(allow_null=True, allow_blank=True) - tz = serializers.CharField(allow_null=True, allow_blank=True) + tz = serializers.CharField() created = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') class Meta: From 7d556df8d553e4d9092318271463dc05f477c10b Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 10:13:01 +0300 Subject: [PATCH 089/173] Add address serializer --- apps/location/models.py | 1 + apps/location/transfer_data.py | 29 +++++++- apps/transfer/models.py | 2 +- apps/transfer/serializers/location.py | 97 ++++++++++++++++++++++++++- 4 files changed, 125 insertions(+), 4 deletions(-) diff --git a/apps/location/models.py b/apps/location/models.py index 0b650bce..5b440560 100644 --- a/apps/location/models.py +++ b/apps/location/models.py @@ -102,6 +102,7 @@ class Address(models.Model): default='', help_text=_('Ex.: 350018')) coordinates = models.PointField( _('Coordinates'), blank=True, null=True, default=None) + old_id = models.IntegerField(null=True, blank=True, default=None) class Meta: """Meta class.""" diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index dca432e7..c079f97b 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -1,7 +1,7 @@ from django.db.models import Q, QuerySet -from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer -from transfer.models import Cities +from transfer.serializers.location import CountrySerializer, RegionSerializer, CitySerializer, AddressSerializer +from transfer.models import Cities, Locations from pprint import pprint @@ -89,6 +89,30 @@ def transfer_cities(): pprint(f"City serializer errors: {serialized_data.errors}") +def transfer_addresses(): + queryset = Locations.objects.raw("""SELECT locations.id, locations.zip_code, locations.longitude, + locations.latitude, locations.address, locations.city_id + FROM locations WHERE + locations.address != "" AND + locations.address IS NOT NULL AND + locations.city_id IS NOT NULL AND + locations.city_id IN (SELECT cities.id + FROM cities WHERE + region_code IS NOT NULL AND + region_code != "" AND + country_code_2 IS NOT NULL AND + country_code_2 != "")""") + + queryset = [vars(query) for query in queryset] + + serialized_data = AddressSerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Address serializer errors: {serialized_data.errors}") + + + data_types = { "dictionaries": [ transfer_countries, @@ -96,5 +120,6 @@ data_types = { transfer_cities ], "tmp": [ + transfer_addresses ] } diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f6dd2dbc..05f1e175 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -232,7 +232,7 @@ class CityPhotos(MigrateMixin): db_table = 'city_photos' -class Locations(models.Model): +class Locations(MigrateMixin): using = 'legacy' country_code = models.CharField(max_length=3) diff --git a/apps/transfer/serializers/location.py b/apps/transfer/serializers/location.py index 11d809ec..d4ea51cc 100644 --- a/apps/transfer/serializers/location.py +++ b/apps/transfer/serializers/location.py @@ -1,5 +1,8 @@ from rest_framework import serializers -from location.models import Country, Region, City +from location.models import Country, Region, City, Address +from django.contrib.gis.geos import Point +from django.contrib.gis.geos import GEOSGeometry +import json class CountrySerializer(serializers.ModelSerializer): @@ -171,3 +174,95 @@ class CitySerializer(serializers.ModelSerializer): def set_old_id(self, data): data['old_id'] = data.pop('id') return data + + +class AddressSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + city_id = serializers.IntegerField() + zip_code = serializers.CharField(allow_null=True, allow_blank=True) + latitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True) + longitude = serializers.DecimalField(max_digits=10, decimal_places=6, allow_null=True) + address = serializers.CharField() + + class Meta: + model = Address + fields = ( + "id", + "city_id", + "zip_code", + "latitude", + "longitude", + "address" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_address(data) + data = self.set_postal_code(data) + data = self.set_city(data) + data = self.set_point(data) + return data + + def create(self, validated_data): + return Address.objects.create(**validated_data) + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + + def set_postal_code(self, data): + data['postal_code'] = data.pop('zip_code', None) + if data['postal_code'] is None: + data['postal_code'] = "" + return data + + def set_city(self, data): + try: + city = City.objects.filter(old_id=data['city_id']).first() + except City.DoesNotExist as e: + raise ValueError(f"City not found with {data}: {e}") + + data['city'] = city + del(data['city_id']) + return data + + def set_address(self, data): + address_list = data.pop('address').split(' ') + is_first_street = False + data['street_name_1'] = [] + data['street_name_2'] = [] + while len(address_list) > 0: + address_part = address_list.pop() + try: + address_part = int(address_part) + data['number'] = address_part + is_first_street = True + except: + if is_first_street: + data['street_name_1'].append(address_part) + else: + data['street_name_2'].append(address_part) + + data['street_name_1'] = " ".join(data['street_name_1']) + data['street_name_2'] = " ".join(data['street_name_2']) + if "number" not in data: + data['number'] = 0 + + return data + + def set_point(self, data): + if data['latitude'] is not None and data['longitude'] is not None: + data['coordinates'] = Point(float(data['longitude']), float(data['latitude'])) + # data['coordinates'] = GEOSGeometry( + # json.dumps({ + # "type": "Point", + # "coordinates": [data['longitude'], data['latitude']] + # }, ensure_ascii=False, default=str) + # ) + else: + data['coordinates'] = None + + del(data['latitude']) + del(data['longitude']) + + return data From 3622ac22b9adf031849125fcc83b4f464ddb2aab Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 10:13:43 +0300 Subject: [PATCH 090/173] Add migrations --- .../location/migrations/0013_country_old_id.py | 18 ++++++++++++++++++ .../migrations/0014_remove_country_old_id.py | 17 +++++++++++++++++ .../location/migrations/0015_country_old_id.py | 18 ++++++++++++++++++ apps/location/migrations/0016_region_old_id.py | 18 ++++++++++++++++++ apps/location/migrations/0017_city_old_id.py | 18 ++++++++++++++++++ .../location/migrations/0018_address_old_id.py | 18 ++++++++++++++++++ 6 files changed, 107 insertions(+) create mode 100644 apps/location/migrations/0013_country_old_id.py create mode 100644 apps/location/migrations/0014_remove_country_old_id.py create mode 100644 apps/location/migrations/0015_country_old_id.py create mode 100644 apps/location/migrations/0016_region_old_id.py create mode 100644 apps/location/migrations/0017_city_old_id.py create mode 100644 apps/location/migrations/0018_address_old_id.py diff --git a/apps/location/migrations/0013_country_old_id.py b/apps/location/migrations/0013_country_old_id.py new file mode 100644 index 00000000..7a6a3029 --- /dev/null +++ b/apps/location/migrations/0013_country_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0012_data_migrate'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0014_remove_country_old_id.py b/apps/location/migrations/0014_remove_country_old_id.py new file mode 100644 index 00000000..6b04c9b4 --- /dev/null +++ b/apps/location/migrations/0014_remove_country_old_id.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:04 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0013_country_old_id'), + ] + + operations = [ + migrations.RemoveField( + model_name='country', + name='old_id', + ), + ] diff --git a/apps/location/migrations/0015_country_old_id.py b/apps/location/migrations/0015_country_old_id.py new file mode 100644 index 00000000..56e9b251 --- /dev/null +++ b/apps/location/migrations/0015_country_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 06:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0014_remove_country_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0016_region_old_id.py b/apps/location/migrations/0016_region_old_id.py new file mode 100644 index 00000000..c4bd0d34 --- /dev/null +++ b/apps/location/migrations/0016_region_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-27 10:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0015_country_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='region', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0017_city_old_id.py b/apps/location/migrations/0017_city_old_id.py new file mode 100644 index 00000000..b38a30bd --- /dev/null +++ b/apps/location/migrations/0017_city_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-28 04:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0016_region_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='city', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/location/migrations/0018_address_old_id.py b/apps/location/migrations/0018_address_old_id.py new file mode 100644 index 00000000..549375f4 --- /dev/null +++ b/apps/location/migrations/0018_address_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-28 05:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0017_city_old_id'), + ] + + operations = [ + migrations.AddField( + model_name='address', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] From 7feed3ac8f002812d243ff20f50e32d5e30c1d0d Mon Sep 17 00:00:00 2001 From: littlewolf Date: Mon, 28 Oct 2019 10:14:11 +0300 Subject: [PATCH 091/173] Move address transfer to dictionaries --- apps/location/transfer_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/location/transfer_data.py b/apps/location/transfer_data.py index c079f97b..b19eb42b 100644 --- a/apps/location/transfer_data.py +++ b/apps/location/transfer_data.py @@ -117,9 +117,7 @@ data_types = { "dictionaries": [ transfer_countries, transfer_regions, - transfer_cities - ], - "tmp": [ + transfer_cities, transfer_addresses ] } From 4e0f3c9a4d83fdc68cac0919219f9d1e2b1d904d Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 28 Oct 2019 10:59:28 +0300 Subject: [PATCH 092/173] with address --- apps/establishment/admin.py | 1 + .../migrations/0005_auto_20191025_0914.py | 20 +++++++++++++++++++ .../migrations/0006_merge_20191027_1758.py | 14 +++++++++++++ .../migrations/0023_merge_20191028_0725.py | 14 +++++++++++++ .../migrations/0030_merge_20191028_0725.py | 14 +++++++++++++ apps/transfer/serializers/establishment.py | 7 +++++-- docker-compose.mysql.yml | 10 ++++++++++ 7 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 apps/gallery/migrations/0005_auto_20191025_0914.py create mode 100644 apps/gallery/migrations/0006_merge_20191027_1758.py create mode 100644 apps/main/migrations/0023_merge_20191028_0725.py create mode 100644 apps/news/migrations/0030_merge_20191028_0725.py diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 50c21b90..4b845c60 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -53,6 +53,7 @@ class EstablishmentAdmin(admin.ModelAdmin): inlines = [ AwardInline, ContactPhoneInline, ContactEmailInline, ReviewInline, CommentInline] + raw_id_fields = ('address',) @admin.register(models.Position) diff --git a/apps/gallery/migrations/0005_auto_20191025_0914.py b/apps/gallery/migrations/0005_auto_20191025_0914.py new file mode 100644 index 00000000..61c158b6 --- /dev/null +++ b/apps/gallery/migrations/0005_auto_20191025_0914.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:14 + +from django.db import migrations +import sorl.thumbnail.fields +import utils.methods + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0004_merge_20191025_0906'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='image', + field=sorl.thumbnail.fields.ImageField(upload_to=utils.methods.image_path, verbose_name='image file'), + ), + ] diff --git a/apps/gallery/migrations/0006_merge_20191027_1758.py b/apps/gallery/migrations/0006_merge_20191027_1758.py new file mode 100644 index 00000000..5fca7c74 --- /dev/null +++ b/apps/gallery/migrations/0006_merge_20191027_1758.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-27 17:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0005_auto_20191027_0756'), + ('gallery', '0005_auto_20191025_0914'), + ] + + operations = [ + ] diff --git a/apps/main/migrations/0023_merge_20191028_0725.py b/apps/main/migrations/0023_merge_20191028_0725.py new file mode 100644 index 00000000..e551fe2e --- /dev/null +++ b/apps/main/migrations/0023_merge_20191028_0725.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-28 07:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0020_merge_20191025_0423'), + ('main', '0022_auto_20191023_1113'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0030_merge_20191028_0725.py b/apps/news/migrations/0030_merge_20191028_0725.py new file mode 100644 index 00000000..671e06d7 --- /dev/null +++ b/apps/news/migrations/0030_merge_20191028_0725.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-28 07:25 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0029_merge_20191025_0906'), + ('news', '0023_merge_20191025_0423'), + ] + + operations = [ + ] diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index 9735f5fb..bfc89bd5 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -3,7 +3,7 @@ from django.db import transaction from rest_framework import serializers from establishment.models import Establishment, ContactEmail, ContactPhone, EstablishmentType -# from location.models import Address +from location.models import Address from timetable.models import Timetable from utils.legacy_parser import parse_legacy_schedule_content from utils.slug_generator import generate_unique_slug @@ -86,8 +86,11 @@ class EstablishmentSerializer(serializers.ModelSerializer): @staticmethod def get_address(address): - # return Address.objects.filter(old_id=address).first() + address = Address.objects.filter(old_id=address).first() + if address: + return address.id return None + # return None @staticmethod def get_type(data): diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index cabb3808..3c891dce 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -54,6 +54,8 @@ services: image: redis:2.8.23 ports: - "6379:6379" + networks: + - redis_network # RabbitMQ #rabbitmq: @@ -78,6 +80,8 @@ services: - db # - rabbitmq - redis + networks: + - redis_network worker_beat: build: . @@ -95,6 +99,9 @@ services: - db # - rabbitmq - redis + networks: + - redis_network + # App: G&M gm_app: build: . @@ -120,6 +127,7 @@ services: ports: - "8000:8000" networks: + - redis_network - database_network - mysql_network - elasticsearch_network @@ -143,3 +151,5 @@ networks: driver: bridge elasticsearch_network: driver: bridge + redis_network: + driver: bridge From 853b0e047e4b0d9e0cca6acfaa82872fb197509d Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Mon, 28 Oct 2019 11:41:47 +0300 Subject: [PATCH 093/173] add settings for dev server --- project/settings/base.py | 4 ++-- project/settings/development.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/project/settings/base.py b/project/settings/base.py index 676a67d1..2f05ca52 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,9 +159,9 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.17.0.1', + # 'HOST': '172.17.0.1', # 'HOST': '172.23.0.1', - # 'HOST': 'mysql_db', + 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', diff --git a/project/settings/development.py b/project/settings/development.py index 73e0095d..a9cfe5be 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -39,3 +39,25 @@ sentry_sdk.init( # TMP ( TODO remove it later) # Временный хардкод для демонстрации 4 ноября, потом удалить! HARDCODED_INTERNATIONAL_NEWS_IDS = [8, 9, 10, 11, 15, 17] + +# Database +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.contrib.gis.db.backends.postgis', + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USERNAME'), + 'PASSWORD': os.environ.get('DB_PASSWORD'), + 'HOST': os.environ.get('DB_HOSTNAME'), + 'PORT': os.environ.get('DB_PORT'), + }, + 'legacy': { + 'ENGINE': 'django.db.backends.mysql', + 'HOST': os.environ.get('MYSQL_HOSTNAME'), + 'PORT': os.environ.get('MYSQL_PORT'), + 'NAME': os.environ.get('MYSQL_DATABASE'), + 'USER': os.environ.get('MYSQL_USER'), + 'PASSWORD': os.environ.get('MYSQL_PASSWORD') + } +} \ No newline at end of file From be31c75c19fa439d6a4b590a25c7ed3449f85a7e Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Mon, 28 Oct 2019 14:19:22 +0300 Subject: [PATCH 094/173] disable elastic --- project/settings/development.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/settings/development.py b/project/settings/development.py index a9cfe5be..6c643301 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -26,8 +26,8 @@ ELASTICSEARCH_DSL = { ELASTICSEARCH_INDEX_NAMES = { - 'search_indexes.documents.news': 'development_news', # temporarily disabled - 'search_indexes.documents.establishment': 'development_establishment', + # 'search_indexes.documents.news': 'development_news', # temporarily disabled + # 'search_indexes.documents.establishment': 'development_establishment', } From c15b5976fb23e2ebd618d5f8dbd31b7cc67b2ed0 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 28 Oct 2019 17:07:32 +0300 Subject: [PATCH 095/173] fix ruby schedules --- .gitignore | 1 + apps/transfer/serializers/establishment.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 5d44eda8..9cb2b74f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ logs/ ./docker-compose.override.yml celerybeat-schedule +local_files \ No newline at end of file diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index bfc89bd5..314705de 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -126,15 +126,15 @@ class EstablishmentSerializer(serializers.ModelSerializer): 'opening_at': None, 'closed_at': None, } - if val['morning']: + if val.get('morning'): payload.update({ - 'lunch_start': val['morning']['start'], - 'lunch_end': val['morning']['end'], + 'lunch_start': val['morning'].get('start'), + 'lunch_end': val['morning'].get('end'), }) - if val['afternoon']: + if val.get('afternoon'): payload.update({ - 'dinner_start': val['afternoon']['start'], - 'dinner_end': val['afternoon']['end'], + 'dinner_start': val['afternoon'].get('start'), + 'dinner_end': val['afternoon'].get('end'), }) try: From bff4c70fc80d09bb2a977fadd83a95aeb6fc291f Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Mon, 28 Oct 2019 17:32:15 +0300 Subject: [PATCH 096/173] disable signal of elsatic --- apps/search_indexes/signals.py | 142 ++++++++++++++++----------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/apps/search_indexes/signals.py b/apps/search_indexes/signals.py index 3da50fb8..e38fa028 100644 --- a/apps/search_indexes/signals.py +++ b/apps/search_indexes/signals.py @@ -4,74 +4,74 @@ from django.dispatch import receiver from django_elasticsearch_dsl.registries import registry -@receiver(post_save) -def update_document(sender, **kwargs): - from establishment.models import Establishment - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - - if app_label == 'location': - if model_name == 'country': - establishments = Establishment.objects.filter( - address__city__country=instance) - for establishment in establishments: - registry.update(establishment) - if model_name == 'city': - establishments = Establishment.objects.filter( - address__city=instance) - for establishment in establishments: - registry.update(establishment) - if model_name == 'address': - establishments = Establishment.objects.filter( - address=instance) - for establishment in establishments: - registry.update(establishment) - - if app_label == 'establishment': - # todo: remove after migration - from establishment import models as establishment_models - if model_name == 'establishmenttype': - if isinstance(instance, establishment_models.EstablishmentType): - establishments = Establishment.objects.filter( - establishment_type=instance) - for establishment in establishments: - registry.update(establishment) - if model_name == 'establishmentsubtype': - if isinstance(instance, establishment_models.EstablishmentSubType): - establishments = Establishment.objects.filter( - establishment_subtypes=instance) - for establishment in establishments: - registry.update(establishment) - - if app_label == 'tag': - if model_name == 'tag': - establishments = Establishment.objects.filter(tags=instance) - for establishment in establishments: - registry.update(establishment) - - -@receiver(post_save) -def update_news(sender, **kwargs): - from news.models import News - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - - if app_label == 'location': - if model_name == 'country': - qs = News.objects.filter(country=instance) - for news in qs: - registry.update(news) - - if app_label == 'news': - if model_name == 'newstype': - qs = News.objects.filter(news_type=instance) - for news in qs: - registry.update(news) - - if app_label == 'tag': - if model_name == 'tag': - qs = News.objects.filter(tags=instance) - for news in qs: - registry.update(news) +# @receiver(post_save) +# def update_document(sender, **kwargs): +# from establishment.models import Establishment +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# +# if app_label == 'location': +# if model_name == 'country': +# establishments = Establishment.objects.filter( +# address__city__country=instance) +# for establishment in establishments: +# registry.update(establishment) +# if model_name == 'city': +# establishments = Establishment.objects.filter( +# address__city=instance) +# for establishment in establishments: +# registry.update(establishment) +# if model_name == 'address': +# establishments = Establishment.objects.filter( +# address=instance) +# for establishment in establishments: +# registry.update(establishment) +# +# if app_label == 'establishment': +# # todo: remove after migration +# from establishment import models as establishment_models +# if model_name == 'establishmenttype': +# if isinstance(instance, establishment_models.EstablishmentType): +# establishments = Establishment.objects.filter( +# establishment_type=instance) +# for establishment in establishments: +# registry.update(establishment) +# if model_name == 'establishmentsubtype': +# if isinstance(instance, establishment_models.EstablishmentSubType): +# establishments = Establishment.objects.filter( +# establishment_subtypes=instance) +# for establishment in establishments: +# registry.update(establishment) +# +# if app_label == 'tag': +# if model_name == 'tag': +# establishments = Establishment.objects.filter(tags=instance) +# for establishment in establishments: +# registry.update(establishment) +# +# +# @receiver(post_save) +# def update_news(sender, **kwargs): +# from news.models import News +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# +# if app_label == 'location': +# if model_name == 'country': +# qs = News.objects.filter(country=instance) +# for news in qs: +# registry.update(news) +# +# if app_label == 'news': +# if model_name == 'newstype': +# qs = News.objects.filter(news_type=instance) +# for news in qs: +# registry.update(news) +# +# if app_label == 'tag': +# if model_name == 'tag': +# qs = News.objects.filter(tags=instance) +# for news in qs: +# registry.update(news) From 5bfbafa8655cb3fe685515fdb43d4b1078106a50 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Mon, 28 Oct 2019 17:33:53 +0300 Subject: [PATCH 097/173] fix disable signal of elsatic --- apps/search_indexes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/__init__.py b/apps/search_indexes/__init__.py index 6c812219..c45cbc38 100644 --- a/apps/search_indexes/__init__.py +++ b/apps/search_indexes/__init__.py @@ -1 +1 @@ -from search_indexes.signals import update_document +# from search_indexes.signals import update_document From 4e868c2d8d45f1c441ee640c50fae12f8542cbd5 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 08:25:03 +0300 Subject: [PATCH 098/173] transfer dishes --- apps/establishment/transfer_data.py | 27 ++++++++- apps/transfer/management/commands/transfer.py | 3 +- apps/transfer/models.py | 17 ++++++ apps/transfer/serializers/plate.py | 55 +++++++++++++++++++ apps/utils/constants.py | 26 +++++++++ project/settings/base.py | 2 +- project/settings/development.py | 4 +- 7 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 apps/transfer/serializers/plate.py create mode 100644 apps/utils/constants.py diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 4670e6cf..3005e6c4 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -1,9 +1,10 @@ from pprint import pprint -from django.db.models import Q +from django.db.models import Q, F -from transfer.models import Establishments +from transfer.models import Establishments, Dishes from transfer.serializers.establishment import EstablishmentSerializer +from transfer.serializers.plate import PlateSerializer def transfer_establishment(): @@ -77,6 +78,26 @@ def transfer_establishment(): pprint(f"Establishment serializer errors: {serialized_data.errors}") +def transfer_menu(): + dishes = Dishes.objects.exclude( + Q(establishment_id__isnull=True) | + Q(dish_type__isnull=True) | + Q(price__isnull=True) | + Q(currency__isnull=True) | + Q(name__isnull=True) | + Q(name__exact='') + ).annotate( + country_code=F('establishment__location__country_code'), + ) + + serialized_data = PlateSerializer(data=list(dishes.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Menu serializer errors: {serialized_data.errors}") + + data_types = { - "establishment": [transfer_establishment] + "establishment": [transfer_establishment], + "menu": [transfer_menu], } diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 1c146018..2e4d5de5 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -18,7 +18,8 @@ class Command(BaseCommand): 'establishment', 'gallery', 'commercial', - 'tmp' + 'tmp', + 'menu', ] def handle(self, *args, **options): diff --git a/apps/transfer/models.py b/apps/transfer/models.py index ffd0664e..e9d7b1e6 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -461,6 +461,23 @@ class Descriptions(MigrateMixin): db_table = 'descriptions' +class Dishes(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + price = models.FloatField(blank=True, null=True) + currency = models.CharField(max_length=255, blank=True, null=True) + dish_type = models.CharField(max_length=255, blank=True, null=True) + signature = models.IntegerField(blank=True, null=True) + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'dishes' + + # class EstablishmentAssets(MigrateMixin): # using = 'legacy' # diff --git a/apps/transfer/serializers/plate.py b/apps/transfer/serializers/plate.py new file mode 100644 index 00000000..7d5eb99b --- /dev/null +++ b/apps/transfer/serializers/plate.py @@ -0,0 +1,55 @@ +from rest_framework import serializers + +from establishment.models import Menu, Plate, Establishment +from main.models import Currency +from utils.constants import CODE_LOCALES + + +class PlateSerializer(serializers.Serializer): + name = serializers.CharField() + price = serializers.DecimalField(decimal_places=2, max_digits=10) + currency = serializers.CharField() + dish_type = serializers.CharField() + country_code = serializers.CharField() + establishment_id = serializers.IntegerField() + signature = serializers.IntegerField(allow_null=True) + + def create(self, validated_data): + establishment = Establishment.objects.filter(old_id=validated_data['establishment_id']).first() + if not establishment: + return + return Plate.objects.create(**self.get_plate_data(validated_data, establishment.id)) + + def get_plate_data(self, validated_data, est_id): + locale = CODE_LOCALES.get(validated_data['country_code'], 'en-GB') + payload = { + 'name': {locale: validated_data['name']}, + 'price': validated_data['price'], + 'currency_id': self.get_currency(validated_data), + 'menu_id': self.get_menu(validated_data, est_id), + 'is_signature_plate': bool(validated_data['signature']), + } + return payload + + @staticmethod + def get_menu(validated_data, est_id): + payload = { + 'establishment_id': est_id, + 'category': {'en-GB': validated_data['dish_type']}, + } + menu, _ = Menu.objects.get_or_create(**payload) + return menu.id + + @staticmethod + def get_currency(validated_data): + try: + currency = Currency.objects.get( + slug=validated_data['currency'], + sign='$', + ) + except Currency.DoesNotExist: + currency = Currency.objects.create( + slug=validated_data['currency'], + sign='$', + ) + return currency.id diff --git a/apps/utils/constants.py b/apps/utils/constants.py new file mode 100644 index 00000000..fe24c008 --- /dev/null +++ b/apps/utils/constants.py @@ -0,0 +1,26 @@ +CODE_LOCALES = { + 'AAA': 'en-GB', + 'AUS': 'en-AU', # Австралия + 'AUT': 'de-AT', # Австрия + 'BEL': 'de-BE', # Бельгия + 'BRA': 'pt-BR', # Бразилия + 'CAN': 'fr-CA', # Канада + 'DEU': 'da-DE', # Германия + 'FRA': 'fr-FR', # Франция + 'GEO': 'ka', # Грузия + 'GRC': 'el-GR', # Греция + 'HRV': 'hr-HR', # Хорватия + 'HUN': 'hu-HU', # Венгрия + 'ISR': 'en-IL', # Израиль + 'ITA': 'it-IT', # Италия + 'JPN': 'ja', # Япония + 'LUX': 'fr-LU', # Люксембург + 'MAR': 'ar-MA', # Марокко + 'MDV': 'dv', # Мальдивы + 'NLD': 'nl-NL', # Нидерланды + 'POL': 'pl', # Польша + 'ROU': 'ro', # Румыния + 'RUS': 'ru-RU', # Россия + 'SVN': 'hu-SI', # Словения + 'USA': 'en-US', # США +} diff --git a/project/settings/base.py b/project/settings/base.py index 2f05ca52..659a96ba 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -329,7 +329,7 @@ REDOC_SETTINGS = { # RabbitMQ # BROKER_URL = 'amqp://rabbitmq:5672' # Redis -BROKER_URL = 'redis://localhost:6379/1' +BROKER_URL = 'redis://redis:6379/1' CELERY_RESULT_BACKEND = BROKER_URL CELERY_BROKER_URL = BROKER_URL CELERY_ACCEPT_CONTENT = ['application/json'] diff --git a/project/settings/development.py b/project/settings/development.py index 6c643301..1cebcc87 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,8 +19,8 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'localhost:9200' - # 'hosts': 'elasticsearch:9200' + # 'hosts': 'localhost:9200' + 'hosts': 'elasticsearch:9200' } } From adcc43309bef723b575e6c34d4d5aebd87fc2530 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 08:26:52 +0300 Subject: [PATCH 099/173] fix settings --- project/settings/base.py | 3 ++- project/settings/development.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/project/settings/base.py b/project/settings/base.py index 659a96ba..3d3ec8ca 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -329,7 +329,8 @@ REDOC_SETTINGS = { # RabbitMQ # BROKER_URL = 'amqp://rabbitmq:5672' # Redis -BROKER_URL = 'redis://redis:6379/1' +BROKER_URL = 'redis://localhost:6379/1' +# BROKER_URL = 'redis://redis:6379/1' CELERY_RESULT_BACKEND = BROKER_URL CELERY_BROKER_URL = BROKER_URL CELERY_ACCEPT_CONTENT = ['application/json'] diff --git a/project/settings/development.py b/project/settings/development.py index 1cebcc87..6c643301 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,8 +19,8 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - # 'hosts': 'localhost:9200' - 'hosts': 'elasticsearch:9200' + 'hosts': 'localhost:9200' + # 'hosts': 'elasticsearch:9200' } } From 939d7f139f3d3a29bbe048da648c5837077bb999 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 10:20:30 +0300 Subject: [PATCH 100/173] fix max max length error --- .../migrations/0045_auto_20191029_0719.py | 43 +++++++++++++++++++ apps/establishment/models.py | 12 +++--- 2 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 apps/establishment/migrations/0045_auto_20191029_0719.py diff --git a/apps/establishment/migrations/0045_auto_20191029_0719.py b/apps/establishment/migrations/0045_auto_20191029_0719.py new file mode 100644 index 00000000..4ceb137b --- /dev/null +++ b/apps/establishment/migrations/0045_auto_20191029_0719.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.4 on 2019-10-29 07:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0044_establishment_old_id'), + ] + + operations = [ + migrations.AlterField( + model_name='establishment', + name='booking', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Booking URL'), + ), + migrations.AlterField( + model_name='establishment', + name='facebook', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Facebook URL'), + ), + migrations.AlterField( + model_name='establishment', + name='lafourchette', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Lafourchette URL'), + ), + migrations.AlterField( + model_name='establishment', + name='preview_image_url', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Preview image URL path'), + ), + migrations.AlterField( + model_name='establishment', + name='twitter', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Twitter URL'), + ), + migrations.AlterField( + model_name='establishment', + name='website', + field=models.URLField(blank=True, default=None, max_length=255, null=True, verbose_name='Web site URL'), + ), + ] diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 3fc623a9..4eacdd0d 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -324,19 +324,19 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): price_level = models.PositiveIntegerField(blank=True, null=True, default=None, verbose_name=_('price level')) - website = models.URLField(blank=True, null=True, default=None, + website = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Web site URL')) - facebook = models.URLField(blank=True, null=True, default=None, + facebook = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Facebook URL')) - twitter = models.URLField(blank=True, null=True, default=None, + twitter = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Twitter URL')) - lafourchette = models.URLField(blank=True, null=True, default=None, + lafourchette = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Lafourchette URL')) guestonline_id = models.PositiveIntegerField(blank=True, verbose_name=_('guestonline id'), null=True, default=None,) lastable_id = models.TextField(blank=True, verbose_name=_('lastable id'), unique=True, null=True, default=None,) - booking = models.URLField(blank=True, null=True, default=None, + booking = models.URLField(blank=True, null=True, default=None, max_length=255, verbose_name=_('Booking URL')) is_publish = models.BooleanField(default=False, verbose_name=_('Publish status')) schedule = models.ManyToManyField(to='timetable.Timetable', @@ -352,7 +352,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): related_name='establishments', blank=True, default=None, verbose_name=_('Collections')) - preview_image_url = models.URLField(verbose_name=_('Preview image URL path'), + preview_image_url = models.URLField(verbose_name=_('Preview image URL path'), max_length=255, blank=True, null=True, default=None) slug = models.SlugField(unique=True, max_length=255, null=True, verbose_name=_('Establishment slug')) From 411bef5272e278480cdf7b0211c8ea92dea2df64 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 10:30:08 +0300 Subject: [PATCH 101/173] change admin for news --- apps/news/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/news/admin.py b/apps/news/admin.py index 5c1cbba7..91fb284c 100644 --- a/apps/news/admin.py +++ b/apps/news/admin.py @@ -26,6 +26,7 @@ send_email_action.short_description = "Send the selected news by email" @admin.register(models.News) class NewsAdmin(admin.ModelAdmin): """News admin.""" + raw_id_fields = ('address',) actions = [send_email_action] From 4a9361e04f056d6f389c43ec09c23fd4ca1651d4 Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 29 Oct 2019 12:50:05 +0500 Subject: [PATCH 102/173] fix notification.py --- apps/transfer/serializers/notification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py index ccc07eff..e6bb7f7b 100644 --- a/apps/transfer/serializers/notification.py +++ b/apps/transfer/serializers/notification.py @@ -22,7 +22,7 @@ class SubscriberSerializer(serializers.ModelSerializer): return data def create(self, validated_data): - Subscriber.objects.create(**validated_data) + Subscriber.objects.get_or_create(**validated_data) def get_email(self, obj): return obj["email"] From 7613ece6f820e2efd80f209e771d9496dce6ddf7 Mon Sep 17 00:00:00 2001 From: michail Date: Tue, 29 Oct 2019 13:09:13 +0500 Subject: [PATCH 103/173] new fix notification.py --- apps/transfer/serializers/notification.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/transfer/serializers/notification.py b/apps/transfer/serializers/notification.py index e6bb7f7b..c179dd2a 100644 --- a/apps/transfer/serializers/notification.py +++ b/apps/transfer/serializers/notification.py @@ -22,7 +22,8 @@ class SubscriberSerializer(serializers.ModelSerializer): return data def create(self, validated_data): - Subscriber.objects.get_or_create(**validated_data) + inst, created = Subscriber.objects.get_or_create(**validated_data) + return inst def get_email(self, obj): return obj["email"] From 72c6545a34d1479ac3b03e7da95edd5d044c22fc Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 29 Oct 2019 11:30:33 +0300 Subject: [PATCH 104/173] Add old_id to account_user --- apps/account/models.py | 2 ++ apps/transfer/serializers/account.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/apps/account/models.py b/apps/account/models.py index 2f8c97cc..13943878 100644 --- a/apps/account/models.py +++ b/apps/account/models.py @@ -94,6 +94,8 @@ class User(AbstractUser): unconfirmed_email = models.EmailField(_('unconfirmed email'), blank=True, null=True, default=None) email_confirmed = models.BooleanField(_('email status'), default=False) newsletter = models.NullBooleanField(default=True) + old_id = models.IntegerField(null=True, blank=True, default=None) + EMAIL_FIELD = 'email' USERNAME_FIELD = 'username' diff --git a/apps/transfer/serializers/account.py b/apps/transfer/serializers/account.py index 123b374b..5b0e1369 100644 --- a/apps/transfer/serializers/account.py +++ b/apps/transfer/serializers/account.py @@ -6,17 +6,20 @@ class UserSerializer(serializers.ModelSerializer): nickname = serializers.CharField() email = serializers.CharField() confirmed_at = serializers.DateTimeField() + id = serializers.CharField() class Meta: model = User fields = ( + "id", "nickname", "email", "confirmed_at" ) def validate(self, data): + data["old_id"] = data.pop("id") data["username"] = self.get_username(data) data["email_confirmed"] = self.get_email_confirmed(data) data.pop("nickname") From 338c4a4901617cd37d1e4e14b51b627dd4bc22e8 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 29 Oct 2019 12:16:18 +0300 Subject: [PATCH 105/173] Add migration --- apps/account/migrations/0018_user_old_id.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 apps/account/migrations/0018_user_old_id.py diff --git a/apps/account/migrations/0018_user_old_id.py b/apps/account/migrations/0018_user_old_id.py new file mode 100644 index 00000000..aa409bed --- /dev/null +++ b/apps/account/migrations/0018_user_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 08:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0017_merge_20191024_1233'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] From dc11fb055f8fd3814f9388410ea86a4814920cdc Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 12:34:08 +0300 Subject: [PATCH 106/173] fix news and paris establishments --- .../commands/rm_old_establishments.py | 13 +++++++++++++ apps/establishment/transfer_data.py | 5 ++++- apps/news/admin.py | 1 + apps/news/management/__init__.py | 0 apps/news/management/commands/__init__.py | 0 apps/news/management/commands/rm_all_news.py | 13 +++++++++++++ apps/news/management/commands/rm_old_news.py | 13 +++++++++++++ apps/news/migrations/0030_news_old_id.py | 18 ++++++++++++++++++ .../migrations/0031_merge_20191029_0858.py | 14 ++++++++++++++ apps/news/models.py | 1 + apps/news/transfer_data.py | 1 + apps/transfer/models.py | 2 +- apps/transfer/serializers/news.py | 10 ++++++++++ 13 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 apps/establishment/management/commands/rm_old_establishments.py create mode 100644 apps/news/management/__init__.py create mode 100644 apps/news/management/commands/__init__.py create mode 100644 apps/news/management/commands/rm_all_news.py create mode 100644 apps/news/management/commands/rm_old_news.py create mode 100644 apps/news/migrations/0030_news_old_id.py create mode 100644 apps/news/migrations/0031_merge_20191029_0858.py diff --git a/apps/establishment/management/commands/rm_old_establishments.py b/apps/establishment/management/commands/rm_old_establishments.py new file mode 100644 index 00000000..ec9fbbaa --- /dev/null +++ b/apps/establishment/management/commands/rm_old_establishments.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment + + +class Command(BaseCommand): + help = 'Remove old establishments from new bd' + + def handle(self, *args, **kwargs): + old_establishments = Establishment.objects.exclude(old_id__isnull=True) + count = old_establishments.count() + old_establishments.delete() + self.stdout.write(self.style.WARNING(f'Deleted {count} objects.')) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 3005e6c4..9911d8f5 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -10,7 +10,9 @@ from transfer.serializers.plate import PlateSerializer def transfer_establishment(): result = [] - old_establishments = Establishments.objects.exclude( + old_establishments = Establishments.objects.filter( + location__city__name__icontains='paris', + ).exclude( Q(type='Wineyard') | Q(location__timezone__isnull=True) ).prefetch_related( @@ -18,6 +20,7 @@ def transfer_establishment(): 'schedules_set', 'descriptions_set', ) + for item in old_establishments: data = { 'old_id': item.id, diff --git a/apps/news/admin.py b/apps/news/admin.py index 91fb284c..fa2aca8b 100644 --- a/apps/news/admin.py +++ b/apps/news/admin.py @@ -28,6 +28,7 @@ class NewsAdmin(admin.ModelAdmin): """News admin.""" raw_id_fields = ('address',) actions = [send_email_action] + raw_id_fields = ('news_type', 'address', 'country') @admin.register(models.NewsGallery) diff --git a/apps/news/management/__init__.py b/apps/news/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/news/management/commands/__init__.py b/apps/news/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/news/management/commands/rm_all_news.py b/apps/news/management/commands/rm_all_news.py new file mode 100644 index 00000000..6b27199e --- /dev/null +++ b/apps/news/management/commands/rm_all_news.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from news.models import News + + +class Command(BaseCommand): + help = 'Remove all news from new bd' + + def handle(self, *args, **kwargs): + old_news = News.objects.all() + count = old_news.count() + old_news.delete() + self.stdout.write(self.style.WARNING(f'Deleted {count} objects.')) diff --git a/apps/news/management/commands/rm_old_news.py b/apps/news/management/commands/rm_old_news.py new file mode 100644 index 00000000..cbcf0f6a --- /dev/null +++ b/apps/news/management/commands/rm_old_news.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand + +from news.models import News + + +class Command(BaseCommand): + help = 'Remove old news from new bd' + + def handle(self, *args, **kwargs): + old_news = News.objects.exclude(old_id__isnull=True) + count = old_news.count() + old_news.delete() + self.stdout.write(self.style.WARNING(f'Deleted {count} objects.')) diff --git a/apps/news/migrations/0030_news_old_id.py b/apps/news/migrations/0030_news_old_id.py new file mode 100644 index 00000000..038e5e35 --- /dev/null +++ b/apps/news/migrations/0030_news_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 08:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0029_merge_20191025_0906'), + ] + + operations = [ + migrations.AddField( + model_name='news', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='odl id'), + ), + ] diff --git a/apps/news/migrations/0031_merge_20191029_0858.py b/apps/news/migrations/0031_merge_20191029_0858.py new file mode 100644 index 00000000..6f00fcf0 --- /dev/null +++ b/apps/news/migrations/0031_merge_20191029_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-29 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0030_news_old_id'), + ('news', '0030_merge_20191028_0725'), + ] + + operations = [ + ] diff --git a/apps/news/models.py b/apps/news/models.py index 306e4906..f0caa13c 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -126,6 +126,7 @@ class News(BaseAttributes, TranslatedFieldsMixin): (PUBLISHED_EXCLUSIVE, _('Published exclusive')), ) + old_id = models.PositiveIntegerField(_('odl id'), blank=True, null=True, default=None) news_type = models.ForeignKey(NewsType, on_delete=models.PROTECT, verbose_name=_('news type')) title = TJSONField(blank=True, null=True, default=None, diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index d46c7613..791b54ce 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -12,6 +12,7 @@ def transfer_news(): queryset = PageTexts.objects.filter(page__type="News").annotate( news_type=Value(news_type.id, output_field=IntegerField()), + country_code=F('page__site__country_code_2'), ) queryset = queryset.annotate(template=F('page__template')) diff --git a/apps/transfer/models.py b/apps/transfer/models.py index e9d7b1e6..c431c504 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -751,7 +751,7 @@ class Pages(MigrateMixin): using = 'legacy' root_title = models.CharField(max_length=255, blank=True, null=True) - site_id = models.IntegerField() + site = models.ForeignKey(Sites, models.DO_NOTHING, blank=True, null=True) account_id = models.IntegerField(blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) template = models.CharField(max_length=255, blank=True, null=True) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index e46c5b4a..c0ed6f79 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,5 +1,6 @@ from rest_framework import serializers +from location.models import Country from news.models import News from utils.legacy_parser import parse_legacy_news_content from utils.slug_generator import generate_unique_slug @@ -12,11 +13,13 @@ class NewsSerializer(serializers.ModelSerializer): title = serializers.CharField() template = serializers.CharField() state = serializers.CharField() + country_code = serializers.CharField(allow_null=True) created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') class Meta: model = News fields = ( + 'old_id', 'created_at', 'state', 'template', @@ -25,6 +28,7 @@ class NewsSerializer(serializers.ModelSerializer): 'slug', 'news_type', 'locale', + 'country_code', ) def validate(self, data): @@ -34,7 +38,9 @@ class NewsSerializer(serializers.ModelSerializer): 'template': self.get_template(data), 'title': self.get_title(data), 'description': self.get_description(data), + 'country': self.get_country(data), }) + data.pop('country_code') data.pop('body') data.pop('locale') return data @@ -42,6 +48,10 @@ class NewsSerializer(serializers.ModelSerializer): def create(self, validated_data): return News.objects.create(**validated_data) + @staticmethod + def get_country(data): + return Country.objects.filter(code__iexact=data['country_code']).first() + @staticmethod def get_template(data): templates = { From 8cf2cebe85083d23cbafd604063653bac75bcddc Mon Sep 17 00:00:00 2001 From: Anatoly Date: Tue, 29 Oct 2019 14:01:27 +0300 Subject: [PATCH 107/173] added BaseModelAdminMixin --- apps/establishment/admin.py | 5 +++-- apps/news/admin.py | 3 ++- apps/review/admin.py | 3 ++- apps/utils/admin.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 apps/utils/admin.py diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 4b845c60..38016a18 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -4,6 +4,7 @@ from django.contrib.contenttypes.admin import GenericTabularInline from django.utils.translation import gettext_lazy as _ from comment.models import Comment +from utils.admin import BaseModelAdminMixin from establishment import models from main.models import Award from review import models as review_models @@ -57,7 +58,7 @@ class EstablishmentAdmin(admin.ModelAdmin): @admin.register(models.Position) -class PositionAdmin(admin.ModelAdmin): +class PositionAdmin(BaseModelAdminMixin): """Position admin.""" @@ -68,7 +69,7 @@ class PlateInline(admin.TabularInline): @admin.register(models.Menu) -class MenuAdmin(admin.ModelAdmin): +class MenuAdmin(BaseModelAdminMixin): """Menu admin.""" list_display = ['id', 'category_translated'] inlines = [ diff --git a/apps/news/admin.py b/apps/news/admin.py index 91fb284c..a6ca3279 100644 --- a/apps/news/admin.py +++ b/apps/news/admin.py @@ -3,6 +3,7 @@ from django.conf import settings from news import models from .tasks import send_email_with_news +from utils.admin import BaseModelAdminMixin @admin.register(models.NewsType) @@ -24,7 +25,7 @@ send_email_action.short_description = "Send the selected news by email" @admin.register(models.News) -class NewsAdmin(admin.ModelAdmin): +class NewsAdmin(BaseModelAdminMixin): """News admin.""" raw_id_fields = ('address',) actions = [send_email_action] diff --git a/apps/review/admin.py b/apps/review/admin.py index 5820f557..03c2419a 100644 --- a/apps/review/admin.py +++ b/apps/review/admin.py @@ -1,8 +1,9 @@ """Admin page for app Review""" from . import models from django.contrib import admin +from utils.admin import BaseModelAdminMixin @admin.register(models.Review) -class ReviewAdminModel(admin.ModelAdmin): +class ReviewAdminModel(BaseModelAdminMixin): """Admin model for model Review.""" diff --git a/apps/utils/admin.py b/apps/utils/admin.py new file mode 100644 index 00000000..43680e20 --- /dev/null +++ b/apps/utils/admin.py @@ -0,0 +1,36 @@ +"""Mixins for admin models.""" +from django.contrib import admin +from django.db.models import ForeignKey + + +class BaseModelAdminMixin(admin.ModelAdmin): + """ + Class that overridden ModelAdmin and adds to readonly_fields attr + persisted fields like created_by, modified_by. + """ + + _PERSISTENT_READ_ONLY_FIELDS = ['created_by', 'modified_by'] + + def _get_fk_field_names(self, fields: iter): + """ + Return an iterable object which contains FK model fields. + :param fields: iterable + :return: iterable + """ + foreign_key_fields = [] + for field in fields: + if isinstance(field, ForeignKey): + foreign_key_fields.append(field.name) + return foreign_key_fields + + def get_readonly_fields(self, request, obj=None): + """ + Hook for specifying custom readonly fields. + """ + _readonly_fields = list(self.readonly_fields) + fk_field_names = self._get_fk_field_names(self.model._meta.fields) + + for field_name in fk_field_names: + if field_name in self._PERSISTENT_READ_ONLY_FIELDS: + _readonly_fields.append(field_name) + return tuple(_readonly_fields) From 97f75dfc0546b07834eddfdb431fccc8c80c4eb3 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 14:32:51 +0300 Subject: [PATCH 108/173] add filter accounts for existing users --- apps/account/transfer_data.py | 8 ++++++-- apps/utils/models.py | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/account/transfer_data.py b/apps/account/transfer_data.py index 91f9051f..c39e0ac1 100644 --- a/apps/account/transfer_data.py +++ b/apps/account/transfer_data.py @@ -8,8 +8,12 @@ def transfer_user(): # queryset = Profiles.objects.all() # queryset = queryset.annotate(nickname=F("account__nickname")) # queryset = queryset.annotate(email=F("account__email")) - - queryset = Accounts.objects.filter(confirmed_at__isnull=False) + stop_list = ['cyril@tomatic.net', + 'cyril2@tomatic.net', + 'd.sadykova@id-east.ru', + 'd.sadykova@octopod.ru', + 'n.yurchenko@id-east.ru'] + queryset = Accounts.objects.filter(confirmed_at__isnull=False).exclude(email__in=stop_list) serialized_data = UserSerializer(data=list(queryset.values()), many=True) diff --git a/apps/utils/models.py b/apps/utils/models.py index fb1de17c..ae6e15b0 100644 --- a/apps/utils/models.py +++ b/apps/utils/models.py @@ -331,3 +331,6 @@ class GMTokenGenerator(PasswordResetTokenGenerator): used. """ return self.get_fields(user, timestamp) + + +timezone.datetime.now().date().isoformat() \ No newline at end of file From 631ffd265a38fbec0af6bd53620861a8b3a9d044 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 14:46:44 +0300 Subject: [PATCH 109/173] add mixin for admin panel --- apps/establishment/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 38016a18..735ccc4b 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -48,7 +48,7 @@ class CommentInline(GenericTabularInline): @admin.register(models.Establishment) -class EstablishmentAdmin(admin.ModelAdmin): +class EstablishmentAdmin(BaseModelAdminMixin): """Establishment admin.""" list_display = ['id', '__str__', 'image_tag', ] inlines = [ From 30ba313f54b7fe770d07997d5157a7c5a460a248 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 14:47:57 +0300 Subject: [PATCH 110/173] add filter accounts for existing users --- apps/establishment/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 735ccc4b..c9a7d33e 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -11,12 +11,12 @@ from review import models as review_models @admin.register(models.EstablishmentType) -class EstablishmentTypeAdmin(admin.ModelAdmin): +class EstablishmentTypeAdmin(BaseModelAdminMixin): """EstablishmentType admin.""" @admin.register(models.EstablishmentSubType) -class EstablishmentSubTypeAdmin(admin.ModelAdmin): +class EstablishmentSubTypeAdmin(BaseModelAdminMixin): """EstablishmentSubType admin.""" From 03bf648ef9f08b05f757704dcab9f31db73e40fa Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 15:08:54 +0300 Subject: [PATCH 111/173] add change admin for establishments --- apps/establishment/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index c9a7d33e..0397c407 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -55,6 +55,7 @@ class EstablishmentAdmin(BaseModelAdminMixin): AwardInline, ContactPhoneInline, ContactEmailInline, ReviewInline, CommentInline] raw_id_fields = ('address',) + fields = ['old_id', 'name'] @admin.register(models.Position) From 251edda2ea876fb9fe73f6c9e9dc0a19c49137bb Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Tue, 29 Oct 2019 15:19:07 +0300 Subject: [PATCH 112/173] update Document ViewSet's pagination class --- apps/search_indexes/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index fd1c5b7a..42f4a87e 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -8,7 +8,7 @@ from django_elasticsearch_dsl_drf.filter_backends import ( from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet from search_indexes import serializers, filters from search_indexes.documents import EstablishmentDocument, NewsDocument -from utils.pagination import ProjectPageNumberPagination +from utils.pagination import ProjectMobilePagination class NewsDocumentViewSet(BaseDocumentViewSet): @@ -16,7 +16,7 @@ class NewsDocumentViewSet(BaseDocumentViewSet): document = NewsDocument lookup_field = 'slug' - pagination_class = ProjectPageNumberPagination + pagination_class = ProjectMobilePagination permission_classes = (permissions.AllowAny,) serializer_class = serializers.NewsDocumentSerializer ordering = ('id',) @@ -53,7 +53,7 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): document = EstablishmentDocument lookup_field = 'slug' - pagination_class = ProjectPageNumberPagination + pagination_class = ProjectMobilePagination permission_classes = (permissions.AllowAny,) serializer_class = serializers.EstablishmentDocumentSerializer ordering = ('id',) From 13eaa942617f697f012977da3b28d0f804ee3ed5 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Tue, 29 Oct 2019 15:23:26 +0300 Subject: [PATCH 113/173] added filter to FavoritesEstablishmentListView --- apps/favorites/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/favorites/views.py b/apps/favorites/views.py index 5d99ed4b..6b7bdab0 100644 --- a/apps/favorites/views.py +++ b/apps/favorites/views.py @@ -1,6 +1,7 @@ """Views for app favorites.""" from rest_framework import generics from establishment.models import Establishment +from establishment.filters import EstablishmentFilter from establishment.serializers import EstablishmentBaseSerializer from .models import Favorites @@ -17,6 +18,7 @@ class FavoritesEstablishmentListView(generics.ListAPIView): """List views for favorites""" serializer_class = EstablishmentBaseSerializer + filter_class = EstablishmentFilter def get_queryset(self): """Override get_queryset method""" From f929c096bc7acbc541bf9bd4b616183fcc4e95d1 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 16:42:43 +0300 Subject: [PATCH 114/173] add subtitle like a description --- apps/transfer/serializers/news.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index c0ed6f79..82853cad 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -38,6 +38,7 @@ class NewsSerializer(serializers.ModelSerializer): 'template': self.get_template(data), 'title': self.get_title(data), 'description': self.get_description(data), + 'subtitle': self.get_description(data), 'country': self.get_country(data), }) data.pop('country_code') From 03c75efeced62f941d30362481a921ae019fdca7 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Tue, 29 Oct 2019 17:16:39 +0300 Subject: [PATCH 115/173] gm-259, gm-260, gm-261 --- apps/account/models.py | 14 ++++++++ apps/establishment/models.py | 8 ++--- apps/establishment/serializers/common.py | 34 +++++------------- apps/establishment/urls/common.py | 3 +- apps/establishment/views/web.py | 24 +++---------- apps/favorites/urls.py | 2 ++ apps/favorites/views.py | 17 ++++++++- apps/product/filters.py | 2 +- apps/product/migrations/0002_product_slug.py | 18 ++++++++++ apps/product/models.py | 4 +++ apps/product/serializers/common.py | 37 ++++++++++++++++++-- apps/product/urls/common.py | 4 ++- apps/product/views/common.py | 21 ++++++++++- apps/recipe/models.py | 8 ++--- apps/utils/serializers.py | 26 ++++++++++++++ 15 files changed, 160 insertions(+), 62 deletions(-) create mode 100644 apps/product/migrations/0002_product_slug.py diff --git a/apps/account/models.py b/apps/account/models.py index 2f8c97cc..cc80721a 100644 --- a/apps/account/models.py +++ b/apps/account/models.py @@ -241,6 +241,20 @@ class User(AbstractUser): template_name=settings.CHANGE_EMAIL_TEMPLATE, context=context) + @property + def favorite_establishment_ids(self): + """Return establishment IDs that in favorites for current user.""" + return self.favorites.by_content_type(app_label='establishment', + model='establishment')\ + .values_list('object_id', flat=True) + + @property + def favorite_recipe_ids(self): + """Return recipe IDs that in favorites for current user.""" + return self.favorites.by_content_type(app_label='recipe', + model='recipe')\ + .values_list('object_id', flat=True) + class UserRole(ProjectBaseMixin): """UserRole model.""" diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 0ca2463f..c31f3a60 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -248,14 +248,12 @@ class EstablishmentQuerySet(models.QuerySet): def annotate_in_favorites(self, user): """Annotate flag in_favorites""" - favorite_establishments = [] + favorite_establishment_ids = [] if user.is_authenticated: - favorite_establishments = user.favorites.by_content_type(app_label='establishment', - model='establishment') \ - .values_list('object_id', flat=True) + favorite_establishment_ids = user.favorite_establishment_ids return self.annotate(in_favorites=Case( When( - id__in=favorite_establishments, + id__in=favorite_establishment_ids, then=True), default=False, output_field=models.BooleanField(default=False))) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 14be142a..5e7c76d4 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -5,15 +5,14 @@ from rest_framework import serializers from comment import models as comment_models from comment.serializers import common as comment_serializers from establishment import models -from favorites.models import Favorites from location.serializers import AddressBaseSerializer from main.serializers import AwardSerializer, CurrencySerializer from review import models as review_models from tag.serializers import TagBaseSerializer from timetable.serialziers import ScheduleRUDSerializer from utils import exceptions as utils_exceptions -from utils.serializers import ProjectModelSerializer -from utils.serializers import TranslatedField +from utils.serializers import (ProjectModelSerializer, TranslatedField, + FavoritesCreateSerializer) class ContactPhonesSerializer(serializers.ModelSerializer): @@ -281,26 +280,13 @@ class EstablishmentCommentRUDSerializer(comment_serializers.CommentSerializer): ] -class EstablishmentFavoritesCreateSerializer(serializers.ModelSerializer): - """Create comment serializer""" - - class Meta: - """Serializer for model Comment""" - model = Favorites - fields = [ - 'id', - 'created', - ] - - def get_user(self): - """Get user from request""" - return self.context.get('request').user +class EstablishmentFavoritesCreateSerializer(FavoritesCreateSerializer): + """Serializer to favorite object w/ model Establishment.""" def validate(self, attrs): - """Override validate method""" + """Overridden validate method""" # Check establishment object - establishment_slug = self.context.get('request').parser_context.get('kwargs').get('slug') - establishment_qs = models.Establishment.objects.filter(slug=establishment_slug) + establishment_qs = models.Establishment.objects.filter(slug=self.slug) # Check establishment obj by slug from lookup_kwarg if not establishment_qs.exists(): @@ -309,18 +295,16 @@ class EstablishmentFavoritesCreateSerializer(serializers.ModelSerializer): establishment = establishment_qs.first() # Check existence in favorites - if self.get_user().favorites.by_content_type(app_label='establishment', - model='establishment')\ - .by_object_id(object_id=establishment.id).exists(): + if establishment.favorites.filter(user=self.user).exists(): raise utils_exceptions.FavoritesError() attrs['establishment'] = establishment return attrs def create(self, validated_data, *args, **kwargs): - """Override create method""" + """Overridden create method""" validated_data.update({ - 'user': self.get_user(), + 'user': self.user, 'content_object': validated_data.pop('establishment') }) return super().create(validated_data) diff --git a/apps/establishment/urls/common.py b/apps/establishment/urls/common.py index 8d9453c1..49cd3631 100644 --- a/apps/establishment/urls/common.py +++ b/apps/establishment/urls/common.py @@ -9,7 +9,6 @@ urlpatterns = [ path('', views.EstablishmentListView.as_view(), name='list'), path('recent-reviews/', views.EstablishmentRecentReviewListView.as_view(), name='recent-reviews'), - # path('wineries/', views.WineriesListView.as_view(), name='wineries-list'), path('slug//', views.EstablishmentRetrieveView.as_view(), name='detail'), path('slug//similar/', views.EstablishmentSimilarListView.as_view(), name='similar'), path('slug//comments/', views.EstablishmentCommentListView.as_view(), name='list-comments'), @@ -18,5 +17,5 @@ urlpatterns = [ path('slug//comments//', views.EstablishmentCommentRUDView.as_view(), name='rud-comment'), path('slug//favorites/', views.EstablishmentFavoritesCreateDestroyView.as_view(), - name='add-to-favorites') + name='create-destroy-favorites') ] diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 0699d9d0..6a1de00e 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -138,15 +138,12 @@ class EstablishmentFavoritesCreateDestroyView(generics.CreateAPIView, generics.D """ Returns the object the view is displaying. """ - establishment_obj = get_object_or_404(models.Establishment, - slug=self.kwargs['slug']) - obj = get_object_or_404( - self.request.user.favorites.by_content_type(app_label='establishment', - model='establishment') - .by_object_id(object_id=establishment_obj.pk)) + establishment = get_object_or_404(models.Establishment, + slug=self.kwargs['slug']) + favorites = get_object_or_404(establishment.favorites.filter(user=self.request.user)) # May raise a permission denied - self.check_object_permissions(self.request, obj) - return obj + self.check_object_permissions(self.request, favorites) + return favorites class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIView): @@ -170,14 +167,3 @@ class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIVi return qs.by_distance_from_point(**{k: v for k, v in filter_kwargs.items() if v is not None}) return qs - - -# Wineries -# todo: find out about difference between subtypes data -# class WineriesListView(EstablishmentListView): -# """Return list establishments with type Wineries""" -# -# def get_queryset(self): -# """Overridden get_queryset method.""" -# qs = super(WineriesListView, self).get_queryset() -# return qs.with_type_related().wineries() diff --git a/apps/favorites/urls.py b/apps/favorites/urls.py index bd0c1d16..ad4c6e9d 100644 --- a/apps/favorites/urls.py +++ b/apps/favorites/urls.py @@ -8,4 +8,6 @@ app_name = 'favorites' urlpatterns = [ path('establishments/', views.FavoritesEstablishmentListView.as_view(), name='establishment-list'), + path('products/', views.FavoritesProductListView.as_view(), + name='product-list'), ] diff --git a/apps/favorites/views.py b/apps/favorites/views.py index 6b7bdab0..d2973142 100644 --- a/apps/favorites/views.py +++ b/apps/favorites/views.py @@ -3,6 +3,9 @@ from rest_framework import generics from establishment.models import Establishment from establishment.filters import EstablishmentFilter from establishment.serializers import EstablishmentBaseSerializer +from product.models import Product +from product.serializers import ProductBaseSerializer +from product.filters import ProductFilterSet from .models import Favorites @@ -15,7 +18,7 @@ class FavoritesBaseView(generics.GenericAPIView): class FavoritesEstablishmentListView(generics.ListAPIView): - """List views for favorites""" + """List views for establishments in favorites.""" serializer_class = EstablishmentBaseSerializer filter_class = EstablishmentFilter @@ -24,3 +27,15 @@ class FavoritesEstablishmentListView(generics.ListAPIView): """Override get_queryset method""" return Establishment.objects.filter(favorites__user=self.request.user)\ .order_by('-favorites') + + +class FavoritesProductListView(generics.ListAPIView): + """List views for products in favorites.""" + + serializer_class = ProductBaseSerializer + filter_class = ProductFilterSet + + def get_queryset(self): + """Override get_queryset method""" + return Product.objects.filter(favorites__user=self.request.user)\ + .order_by('-favorites') diff --git a/apps/product/filters.py b/apps/product/filters.py index 7894a6e6..a30147eb 100644 --- a/apps/product/filters.py +++ b/apps/product/filters.py @@ -5,7 +5,7 @@ from django_filters import rest_framework as filters from product import models -class ProductListFilterSet(filters.FilterSet): +class ProductFilterSet(filters.FilterSet): """Product filter set.""" establishment_id = filters.NumberFilter() diff --git a/apps/product/migrations/0002_product_slug.py b/apps/product/migrations/0002_product_slug.py new file mode 100644 index 00000000..a4605464 --- /dev/null +++ b/apps/product/migrations/0002_product_slug.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 14:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('product', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='product', + name='slug', + field=models.SlugField(max_length=255, null=True, unique=True, verbose_name='Establishment slug'), + ), + ] diff --git a/apps/product/models.py b/apps/product/models.py index 40007f18..d4011fa0 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -1,5 +1,6 @@ """Product app models.""" from django.db import models +from django.contrib.contenttypes import fields as generic from django.core.exceptions import ValidationError from django.contrib.postgres.fields import JSONField from django.utils.translation import gettext_lazy as _ @@ -141,6 +142,9 @@ class Product(TranslatedFieldsMixin, BaseAttributes): wine_appellation = models.ForeignKey('location.WineAppellation', on_delete=models.PROTECT, blank=True, null=True, verbose_name=_('wine appellation')) + slug = models.SlugField(unique=True, max_length=255, null=True, + verbose_name=_('Establishment slug')) + favorites = generic.GenericRelation(to='favorites.Favorites') objects = ProductManager.from_queryset(ProductQuerySet)() diff --git a/apps/product/serializers/common.py b/apps/product/serializers/common.py index fb2f7aeb..7ebfaa28 100644 --- a/apps/product/serializers/common.py +++ b/apps/product/serializers/common.py @@ -1,9 +1,11 @@ """Product app serializers.""" from rest_framework import serializers -from utils.serializers import TranslatedField +from utils.serializers import TranslatedField, FavoritesCreateSerializer from product.models import Product, ProductSubType, ProductType +from utils import exceptions as utils_exceptions +from django.utils.translation import gettext_lazy as _ from location.serializers import (WineRegionBaseSerializer, WineAppellationBaseSerializer, - CountrySimpleSerializer) + CountrySimpleSerializer) class ProductSubTypeBaseSerializer(serializers.ModelSerializer): @@ -50,6 +52,7 @@ class ProductBaseSerializer(serializers.ModelSerializer): model = Product fields = [ 'id', + 'slug', 'name_translated', 'category_display', 'description_translated', @@ -61,3 +64,33 @@ class ProductBaseSerializer(serializers.ModelSerializer): 'wine_appellation', 'available_countries', ] + + +class ProductFavoritesCreateSerializer(FavoritesCreateSerializer): + """Serializer to create favorite object w/ model Product.""" + + def validate(self, attrs): + """Overridden validate method""" + # Check establishment object + product_qs = Product.objects.filter(slug=self.slug) + + # Check establishment obj by slug from lookup_kwarg + if not product_qs.exists(): + raise serializers.ValidationError({'detail': _('Object not found.')}) + else: + product = product_qs.first() + + # Check existence in favorites + if product.favorites.filter(user=self.user).exists(): + raise utils_exceptions.FavoritesError() + + attrs['product'] = product + return attrs + + def create(self, validated_data, *args, **kwargs): + """Overridden create method""" + validated_data.update({ + 'user': self.user, + 'content_object': validated_data.pop('product') + }) + return super().create(validated_data) \ No newline at end of file diff --git a/apps/product/urls/common.py b/apps/product/urls/common.py index 57abf4f0..d0dbb8a9 100644 --- a/apps/product/urls/common.py +++ b/apps/product/urls/common.py @@ -6,5 +6,7 @@ from product import views app_name = 'product' urlpatterns = [ - path('', views.ProductListView.as_view(), name='list') + path('', views.ProductListView.as_view(), name='list'), + path('slug//favorites/', views.CreateFavoriteProductView.as_view(), + name='create-destroy-favorites') ] diff --git a/apps/product/views/common.py b/apps/product/views/common.py index 403781e4..63b9677f 100644 --- a/apps/product/views/common.py +++ b/apps/product/views/common.py @@ -1,5 +1,6 @@ """Product app views.""" from rest_framework import generics, permissions +from django.shortcuts import get_object_or_404 from product.models import Product from product import serializers from product import filters @@ -17,4 +18,22 @@ class ProductListView(ProductBaseView, generics.ListAPIView): """List view for model Product.""" permission_classes = (permissions.AllowAny, ) serializer_class = serializers.ProductBaseSerializer - filter_class = filters.ProductListFilterSet + filter_class = filters.ProductFilterSet + + +class CreateFavoriteProductView(generics.CreateAPIView, + generics.DestroyAPIView): + """View for create/destroy product in favorites.""" + serializer_class = serializers.ProductFavoritesCreateSerializer + lookup_field = 'slug' + + def get_object(self): + """ + Returns the object the view is displaying. + """ + product = get_object_or_404(Product, slug=self.kwargs['slug']) + favorites = get_object_or_404(product.favorites.filter(user=self.request.user)) + + # May raise a permission denied + self.check_object_permissions(self.request, favorites) + return favorites diff --git a/apps/recipe/models.py b/apps/recipe/models.py index 6df7adc2..349fed7b 100644 --- a/apps/recipe/models.py +++ b/apps/recipe/models.py @@ -15,14 +15,12 @@ class RecipeQuerySet(models.QuerySet): def annotate_in_favorites(self, user): """Annotate flag in_favorites""" - favorite_establishments = [] + favorite_recipe_ids = [] if user.is_authenticated: - favorite_establishments = user.favorites.by_content_type(app_label='recipe', - model='recipe') \ - .values_list('object_id', flat=True) + favorite_recipe_ids = user.favorite_recipe_ids return self.annotate(in_favorites=models.Case( models.When( - id__in=favorite_establishments, + id__in=favorite_recipe_ids, then=True), default=False, output_field=models.BooleanField(default=False))) diff --git a/apps/utils/serializers.py b/apps/utils/serializers.py index eeff1043..bd7e8b8e 100644 --- a/apps/utils/serializers.py +++ b/apps/utils/serializers.py @@ -4,6 +4,7 @@ from django.core import exceptions from rest_framework import serializers from utils import models from translation.models import Language +from favorites.models import Favorites class EmptySerializer(serializers.Serializer): @@ -72,3 +73,28 @@ class ProjectModelSerializer(serializers.ModelSerializer): """Overrided ModelSerializer.""" serializers.ModelSerializer.serializer_field_mapping[models.TJSONField] = TJSONField + + +class FavoritesCreateSerializer(serializers.ModelSerializer): + """Serializer to favorite object.""" + + class Meta: + """Serializer for model Comment.""" + model = Favorites + fields = [ + 'id', + 'created', + ] + + @property + def request(self): + return self.context.get('request') + + @property + def user(self): + """Get user from request""" + return self.request.user + + @property + def slug(self): + return self.request.parser_context.get('kwargs').get('slug') From b41ec3eebd64a953eaf1920e0bab01888ee711d6 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 17:16:51 +0300 Subject: [PATCH 116/173] add is_publish --- .../management/commands/add_publish_data.py | 18 ++++++++++++++++++ apps/establishment/transfer_data.py | 1 + apps/transfer/serializers/establishment.py | 4 ++++ 3 files changed, 23 insertions(+) create mode 100644 apps/establishment/management/commands/add_publish_data.py diff --git a/apps/establishment/management/commands/add_publish_data.py b/apps/establishment/management/commands/add_publish_data.py new file mode 100644 index 00000000..1c5aae3e --- /dev/null +++ b/apps/establishment/management/commands/add_publish_data.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment +from transfer.models import Establishments + + +class Command(BaseCommand): + help = 'Add publish values from old db to new db' + + def handle(self, *args, **kwargs): + old_establishments = Establishments.objects.all() + count = 0 + for item in old_establishments: + obj = Establishment.objects.filter(old_id=item.id).first() + if obj: + count += 1 + obj.is_publish = item.state == 'published' + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 9911d8f5..b398bc3e 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -30,6 +30,7 @@ def transfer_establishment(): 'type': item.type, 'phone': item.phone, 'created': item.created_at, + 'state': item.state, 'description': {}, 'website': None, 'facebook': None, diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index 314705de..c9979430 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -24,6 +24,7 @@ class EstablishmentSerializer(serializers.ModelSerializer): facebook = serializers.CharField(allow_null=True, allow_blank=True) twitter = serializers.CharField(allow_null=True, allow_blank=True) booking = serializers.CharField(allow_null=True, allow_blank=True) + state = serializers.CharField(allow_null=True) tz = serializers.CharField() created = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') @@ -47,6 +48,7 @@ class EstablishmentSerializer(serializers.ModelSerializer): 'location', # + получить новые объекты Address по old_id 'email', # + создать объект для ContactEmail 'phone', # + создать объект для ContactPhone + 'state', # + создать объект для ContactPhone ) def validate(self, data): @@ -54,9 +56,11 @@ class EstablishmentSerializer(serializers.ModelSerializer): 'slug': generate_unique_slug(Establishment, data['slug'] if data['slug'] else data['name']), 'address_id': self.get_address(data['location']), 'establishment_type_id': self.get_type(data), + 'is_publish': data.get('state') == 'published', }) data.pop('location') data.pop('type') + data.pop('state') return data @transaction.atomic From c32b665b9218af1ed25375b3074ccc39704c863d Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 17:17:24 +0300 Subject: [PATCH 117/173] enable elastic --- apps/search_indexes/__init__.py | 2 +- apps/search_indexes/signals.py | 142 ++++++++++++++++---------------- project/settings/development.py | 4 +- 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/apps/search_indexes/__init__.py b/apps/search_indexes/__init__.py index c45cbc38..6c812219 100644 --- a/apps/search_indexes/__init__.py +++ b/apps/search_indexes/__init__.py @@ -1 +1 @@ -# from search_indexes.signals import update_document +from search_indexes.signals import update_document diff --git a/apps/search_indexes/signals.py b/apps/search_indexes/signals.py index e38fa028..3da50fb8 100644 --- a/apps/search_indexes/signals.py +++ b/apps/search_indexes/signals.py @@ -4,74 +4,74 @@ from django.dispatch import receiver from django_elasticsearch_dsl.registries import registry -# @receiver(post_save) -# def update_document(sender, **kwargs): -# from establishment.models import Establishment -# app_label = sender._meta.app_label -# model_name = sender._meta.model_name -# instance = kwargs['instance'] -# -# if app_label == 'location': -# if model_name == 'country': -# establishments = Establishment.objects.filter( -# address__city__country=instance) -# for establishment in establishments: -# registry.update(establishment) -# if model_name == 'city': -# establishments = Establishment.objects.filter( -# address__city=instance) -# for establishment in establishments: -# registry.update(establishment) -# if model_name == 'address': -# establishments = Establishment.objects.filter( -# address=instance) -# for establishment in establishments: -# registry.update(establishment) -# -# if app_label == 'establishment': -# # todo: remove after migration -# from establishment import models as establishment_models -# if model_name == 'establishmenttype': -# if isinstance(instance, establishment_models.EstablishmentType): -# establishments = Establishment.objects.filter( -# establishment_type=instance) -# for establishment in establishments: -# registry.update(establishment) -# if model_name == 'establishmentsubtype': -# if isinstance(instance, establishment_models.EstablishmentSubType): -# establishments = Establishment.objects.filter( -# establishment_subtypes=instance) -# for establishment in establishments: -# registry.update(establishment) -# -# if app_label == 'tag': -# if model_name == 'tag': -# establishments = Establishment.objects.filter(tags=instance) -# for establishment in establishments: -# registry.update(establishment) -# -# -# @receiver(post_save) -# def update_news(sender, **kwargs): -# from news.models import News -# app_label = sender._meta.app_label -# model_name = sender._meta.model_name -# instance = kwargs['instance'] -# -# if app_label == 'location': -# if model_name == 'country': -# qs = News.objects.filter(country=instance) -# for news in qs: -# registry.update(news) -# -# if app_label == 'news': -# if model_name == 'newstype': -# qs = News.objects.filter(news_type=instance) -# for news in qs: -# registry.update(news) -# -# if app_label == 'tag': -# if model_name == 'tag': -# qs = News.objects.filter(tags=instance) -# for news in qs: -# registry.update(news) +@receiver(post_save) +def update_document(sender, **kwargs): + from establishment.models import Establishment + app_label = sender._meta.app_label + model_name = sender._meta.model_name + instance = kwargs['instance'] + + if app_label == 'location': + if model_name == 'country': + establishments = Establishment.objects.filter( + address__city__country=instance) + for establishment in establishments: + registry.update(establishment) + if model_name == 'city': + establishments = Establishment.objects.filter( + address__city=instance) + for establishment in establishments: + registry.update(establishment) + if model_name == 'address': + establishments = Establishment.objects.filter( + address=instance) + for establishment in establishments: + registry.update(establishment) + + if app_label == 'establishment': + # todo: remove after migration + from establishment import models as establishment_models + if model_name == 'establishmenttype': + if isinstance(instance, establishment_models.EstablishmentType): + establishments = Establishment.objects.filter( + establishment_type=instance) + for establishment in establishments: + registry.update(establishment) + if model_name == 'establishmentsubtype': + if isinstance(instance, establishment_models.EstablishmentSubType): + establishments = Establishment.objects.filter( + establishment_subtypes=instance) + for establishment in establishments: + registry.update(establishment) + + if app_label == 'tag': + if model_name == 'tag': + establishments = Establishment.objects.filter(tags=instance) + for establishment in establishments: + registry.update(establishment) + + +@receiver(post_save) +def update_news(sender, **kwargs): + from news.models import News + app_label = sender._meta.app_label + model_name = sender._meta.model_name + instance = kwargs['instance'] + + if app_label == 'location': + if model_name == 'country': + qs = News.objects.filter(country=instance) + for news in qs: + registry.update(news) + + if app_label == 'news': + if model_name == 'newstype': + qs = News.objects.filter(news_type=instance) + for news in qs: + registry.update(news) + + if app_label == 'tag': + if model_name == 'tag': + qs = News.objects.filter(tags=instance) + for news in qs: + registry.update(news) diff --git a/project/settings/development.py b/project/settings/development.py index 6c643301..a9cfe5be 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -26,8 +26,8 @@ ELASTICSEARCH_DSL = { ELASTICSEARCH_INDEX_NAMES = { - # 'search_indexes.documents.news': 'development_news', # temporarily disabled - # 'search_indexes.documents.establishment': 'development_establishment', + 'search_indexes.documents.news': 'development_news', # temporarily disabled + 'search_indexes.documents.establishment': 'development_establishment', } From 71f2f8a574d0aff9f786108a3922541c436698c8 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Tue, 29 Oct 2019 17:44:03 +0300 Subject: [PATCH 118/173] small fix for establishment model admin --- apps/establishment/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 0397c407..a66bc0e3 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -40,6 +40,7 @@ class ContactEmailInline(admin.TabularInline): class ReviewInline(GenericTabularInline): model = review_models.Review extra = 0 + readonly_fields = ('created_by', 'modified_by') class CommentInline(GenericTabularInline): @@ -55,7 +56,6 @@ class EstablishmentAdmin(BaseModelAdminMixin): AwardInline, ContactPhoneInline, ContactEmailInline, ReviewInline, CommentInline] raw_id_fields = ('address',) - fields = ['old_id', 'name'] @admin.register(models.Position) From 5984fdc7cd932332b789c4af5572b862db490e14 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Tue, 29 Oct 2019 17:57:11 +0300 Subject: [PATCH 119/173] fixed BaseModelAdminMixin --- apps/establishment/admin.py | 13 ++++++------- apps/news/admin.py | 2 +- apps/review/admin.py | 2 +- apps/utils/admin.py | 3 +-- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index a66bc0e3..9afa6331 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -11,12 +11,12 @@ from review import models as review_models @admin.register(models.EstablishmentType) -class EstablishmentTypeAdmin(BaseModelAdminMixin): +class EstablishmentTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin): """EstablishmentType admin.""" @admin.register(models.EstablishmentSubType) -class EstablishmentSubTypeAdmin(BaseModelAdminMixin): +class EstablishmentSubTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin): """EstablishmentSubType admin.""" @@ -37,10 +37,9 @@ class ContactEmailInline(admin.TabularInline): extra = 0 -class ReviewInline(GenericTabularInline): +class ReviewInline(BaseModelAdminMixin, GenericTabularInline): model = review_models.Review extra = 0 - readonly_fields = ('created_by', 'modified_by') class CommentInline(GenericTabularInline): @@ -49,7 +48,7 @@ class CommentInline(GenericTabularInline): @admin.register(models.Establishment) -class EstablishmentAdmin(BaseModelAdminMixin): +class EstablishmentAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Establishment admin.""" list_display = ['id', '__str__', 'image_tag', ] inlines = [ @@ -59,7 +58,7 @@ class EstablishmentAdmin(BaseModelAdminMixin): @admin.register(models.Position) -class PositionAdmin(BaseModelAdminMixin): +class PositionAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Position admin.""" @@ -70,7 +69,7 @@ class PlateInline(admin.TabularInline): @admin.register(models.Menu) -class MenuAdmin(BaseModelAdminMixin): +class MenuAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Menu admin.""" list_display = ['id', 'category_translated'] inlines = [ diff --git a/apps/news/admin.py b/apps/news/admin.py index a6ca3279..9cbf63cc 100644 --- a/apps/news/admin.py +++ b/apps/news/admin.py @@ -25,7 +25,7 @@ send_email_action.short_description = "Send the selected news by email" @admin.register(models.News) -class NewsAdmin(BaseModelAdminMixin): +class NewsAdmin(BaseModelAdminMixin, admin.ModelAdmin): """News admin.""" raw_id_fields = ('address',) actions = [send_email_action] diff --git a/apps/review/admin.py b/apps/review/admin.py index 03c2419a..2a7326ae 100644 --- a/apps/review/admin.py +++ b/apps/review/admin.py @@ -5,5 +5,5 @@ from utils.admin import BaseModelAdminMixin @admin.register(models.Review) -class ReviewAdminModel(BaseModelAdminMixin): +class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin): """Admin model for model Review.""" diff --git a/apps/utils/admin.py b/apps/utils/admin.py index 43680e20..fd5e353f 100644 --- a/apps/utils/admin.py +++ b/apps/utils/admin.py @@ -1,9 +1,8 @@ """Mixins for admin models.""" -from django.contrib import admin from django.db.models import ForeignKey -class BaseModelAdminMixin(admin.ModelAdmin): +class BaseModelAdminMixin: """ Class that overridden ModelAdmin and adds to readonly_fields attr persisted fields like created_by, modified_by. From 5675f23faee8e497e1a712f1d05a5312ada54f82 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Tue, 29 Oct 2019 17:57:11 +0300 Subject: [PATCH 120/173] fixed BaseModelAdminMixin --- apps/establishment/admin.py | 12 ++++++------ apps/news/admin.py | 2 +- apps/review/admin.py | 2 +- apps/utils/admin.py | 3 +-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/apps/establishment/admin.py b/apps/establishment/admin.py index 0397c407..8acadc02 100644 --- a/apps/establishment/admin.py +++ b/apps/establishment/admin.py @@ -11,12 +11,12 @@ from review import models as review_models @admin.register(models.EstablishmentType) -class EstablishmentTypeAdmin(BaseModelAdminMixin): +class EstablishmentTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin): """EstablishmentType admin.""" @admin.register(models.EstablishmentSubType) -class EstablishmentSubTypeAdmin(BaseModelAdminMixin): +class EstablishmentSubTypeAdmin(BaseModelAdminMixin, admin.ModelAdmin): """EstablishmentSubType admin.""" @@ -37,7 +37,7 @@ class ContactEmailInline(admin.TabularInline): extra = 0 -class ReviewInline(GenericTabularInline): +class ReviewInline(BaseModelAdminMixin, GenericTabularInline): model = review_models.Review extra = 0 @@ -48,7 +48,7 @@ class CommentInline(GenericTabularInline): @admin.register(models.Establishment) -class EstablishmentAdmin(BaseModelAdminMixin): +class EstablishmentAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Establishment admin.""" list_display = ['id', '__str__', 'image_tag', ] inlines = [ @@ -59,7 +59,7 @@ class EstablishmentAdmin(BaseModelAdminMixin): @admin.register(models.Position) -class PositionAdmin(BaseModelAdminMixin): +class PositionAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Position admin.""" @@ -70,7 +70,7 @@ class PlateInline(admin.TabularInline): @admin.register(models.Menu) -class MenuAdmin(BaseModelAdminMixin): +class MenuAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Menu admin.""" list_display = ['id', 'category_translated'] inlines = [ diff --git a/apps/news/admin.py b/apps/news/admin.py index 90527ea4..cc48a887 100644 --- a/apps/news/admin.py +++ b/apps/news/admin.py @@ -25,7 +25,7 @@ send_email_action.short_description = "Send the selected news by email" @admin.register(models.News) -class NewsAdmin(BaseModelAdminMixin): +class NewsAdmin(BaseModelAdminMixin, admin.ModelAdmin): """News admin.""" raw_id_fields = ('address',) actions = [send_email_action] diff --git a/apps/review/admin.py b/apps/review/admin.py index 03c2419a..2a7326ae 100644 --- a/apps/review/admin.py +++ b/apps/review/admin.py @@ -5,5 +5,5 @@ from utils.admin import BaseModelAdminMixin @admin.register(models.Review) -class ReviewAdminModel(BaseModelAdminMixin): +class ReviewAdminModel(BaseModelAdminMixin, admin.ModelAdmin): """Admin model for model Review.""" diff --git a/apps/utils/admin.py b/apps/utils/admin.py index 43680e20..fd5e353f 100644 --- a/apps/utils/admin.py +++ b/apps/utils/admin.py @@ -1,9 +1,8 @@ """Mixins for admin models.""" -from django.contrib import admin from django.db.models import ForeignKey -class BaseModelAdminMixin(admin.ModelAdmin): +class BaseModelAdminMixin: """ Class that overridden ModelAdmin and adds to readonly_fields attr persisted fields like created_by, modified_by. From 3f233fd2460e724677f340bed0fc5af693f33936 Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Tue, 29 Oct 2019 18:13:21 +0300 Subject: [PATCH 121/173] update establishment document index & establishment documentview --- .../search_indexes/documents/establishment.py | 19 +------------------ apps/search_indexes/serializers.py | 7 +++++++ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 88f7e44a..98138146 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -10,16 +10,6 @@ EstablishmentIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1) -# todo: check & refactor -class ObjectField(fields.ObjectField): - - def get_value_from_instance(self, *args, **kwargs): - value = super(ObjectField, self).get_value_from_instance(*args, **kwargs) - if value == {}: - return None - return value - - @EstablishmentIndex.doc_type class EstablishmentDocument(Document): """Establishment document.""" @@ -63,7 +53,7 @@ class EstablishmentDocument(Document): 'closed_at': fields.KeywordField(attr='closed_at_str'), } )) - address = ObjectField( + address = fields.ObjectField( properties={ 'id': fields.IntegerField(), 'street_name_1': fields.TextField( @@ -93,13 +83,6 @@ class EstablishmentDocument(Document): ), }, ) - # todo: need to fix - # collections = fields.ObjectField( - # properties={ - # 'id': fields.IntegerField(attr='collection.id'), - # 'collection_type': fields.IntegerField(attr='collection.collection_type'), - # }, - # multi=True) class Django: diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index cb1fe735..ee2e2ee8 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -13,6 +13,8 @@ class TagsDocumentSerializer(serializers.Serializer): label_translated = serializers.SerializerMethodField() def get_label_translated(self, obj): + if isinstance(obj, dict): + return get_translated_value(obj.get('label')) return get_translated_value(obj.label) @@ -29,6 +31,11 @@ class AddressDocumentSerializer(serializers.Serializer): geo_lon = serializers.FloatField(allow_null=True, source='coordinates.lon') geo_lat = serializers.FloatField(allow_null=True, source='coordinates.lat') + def to_representation(self, instance): + if len(instance) != 0: + return super().to_representation(instance) + return None + class ScheduleDocumentSerializer(serializers.Serializer): """Schedule serializer for ES Document""" From 9fecbfe98f7d34e271440c3d99e6da271035d5cc Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Tue, 29 Oct 2019 19:17:44 +0300 Subject: [PATCH 122/173] EstablishmentViewSet default ordering and filtration --- apps/search_indexes/documents/establishment.py | 3 ++- apps/search_indexes/serializers.py | 5 ++++- apps/search_indexes/views.py | 14 ++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 98138146..7eac2d6c 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -91,6 +91,7 @@ class EstablishmentDocument(Document): 'id', 'name', 'name_translated', + 'is_publish', 'price_level', 'toque_number', 'public_mark', @@ -98,4 +99,4 @@ class EstablishmentDocument(Document): ) def get_queryset(self): - return super().get_queryset().published().with_es_related() + return super().get_queryset().with_es_related() diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index ee2e2ee8..b9df01b7 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -1,5 +1,6 @@ """Search indexes serializers.""" from rest_framework import serializers +from elasticsearch_dsl import AttrDict from django_elasticsearch_dsl_drf.serializers import DocumentSerializer from news.serializers import NewsTypeSerializer from search_indexes.documents import EstablishmentDocument, NewsDocument @@ -31,8 +32,10 @@ class AddressDocumentSerializer(serializers.Serializer): geo_lon = serializers.FloatField(allow_null=True, source='coordinates.lon') geo_lat = serializers.FloatField(allow_null=True, source='coordinates.lat') + # todo: refator def to_representation(self, instance): - if len(instance) != 0: + if instance != AttrDict(d={}) or \ + (isinstance(instance, dict) and len(instance) != 0): return super().to_representation(instance) return None diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index 42f4a87e..72ce2efa 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -3,7 +3,8 @@ from rest_framework import permissions from django_elasticsearch_dsl_drf import constants from django_elasticsearch_dsl_drf.filter_backends import ( FilteringFilterBackend, - GeoSpatialFilteringFilterBackend + GeoSpatialFilteringFilterBackend, + DefaultOrderingFilterBackend, ) from django_elasticsearch_dsl_drf.viewsets import BaseDocumentViewSet from search_indexes import serializers, filters @@ -56,12 +57,20 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): pagination_class = ProjectMobilePagination permission_classes = (permissions.AllowAny,) serializer_class = serializers.EstablishmentDocumentSerializer - ordering = ('id',) + + def get_queryset(self): + qs = super(EstablishmentDocumentViewSet, self).get_queryset() + qs = qs.filter('match', is_publish=True) + if self.request.country_code: + qs = qs.filter('term', + **{'address.city.country.code': self.request.country_code}) + return qs filter_backends = [ FilteringFilterBackend, filters.CustomSearchFilterBackend, GeoSpatialFilteringFilterBackend, + DefaultOrderingFilterBackend, ] search_fields = { @@ -72,6 +81,7 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): translated_search_fields = ( 'description', ) + ordering = 'id' filter_fields = { 'slug': 'slug', 'tag': { From a0aa503cfa59211a1286d745f2a2c5bf9e99c024 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 123/173] Add index_name type field to geo --- apps/establishment/serializers/common.py | 25 ++++++++++++++++++++++++ apps/establishment/views/web.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/apps/establishment/serializers/common.py b/apps/establishment/serializers/common.py index 14be142a..12e217a9 100644 --- a/apps/establishment/serializers/common.py +++ b/apps/establishment/serializers/common.py @@ -118,6 +118,18 @@ class EstablishmentTypeBaseSerializer(serializers.ModelSerializer): 'use_subtypes': {'write_only': True}, } +class EstablishmentTypeGeoSerializer(EstablishmentTypeBaseSerializer): + """Serializer for EstablishmentType model w/ index_name.""" + + class Meta(EstablishmentTypeBaseSerializer.Meta): + fields = EstablishmentTypeBaseSerializer.Meta.fields + [ + 'index_name' + ] + extra_kwargs = { + **EstablishmentTypeBaseSerializer.Meta.extra_kwargs, + 'index_name': {'read_only': True}, + } + class EstablishmentSubTypeBaseSerializer(serializers.ModelSerializer): """Serializer for EstablishmentSubType models.""" @@ -185,6 +197,19 @@ class EstablishmentBaseSerializer(ProjectModelSerializer): ] +class EstablishmentGeoSerializer(EstablishmentBaseSerializer): + """Serializer for Geo view.""" + + type = EstablishmentTypeGeoSerializer(source='establishment_type', read_only=True) + + class Meta(EstablishmentBaseSerializer.Meta): + """Meta class.""" + + fields = EstablishmentBaseSerializer.Meta.fields + [ + 'type' + ] + + class EstablishmentDetailSerializer(EstablishmentBaseSerializer): """Serializer for Establishment model.""" diff --git a/apps/establishment/views/web.py b/apps/establishment/views/web.py index 0699d9d0..63033d4b 100644 --- a/apps/establishment/views/web.py +++ b/apps/establishment/views/web.py @@ -152,7 +152,7 @@ class EstablishmentFavoritesCreateDestroyView(generics.CreateAPIView, generics.D class EstablishmentNearestRetrieveView(EstablishmentListView, generics.ListAPIView): """Resource for getting list of nearest establishments.""" - serializer_class = serializers.EstablishmentBaseSerializer + serializer_class = serializers.EstablishmentGeoSerializer filter_class = filters.EstablishmentFilter def get_queryset(self): From 55bdc7771f62da0953f261d9d00a29f20135a4f5 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 29 Oct 2019 21:54:03 +0300 Subject: [PATCH 124/173] add image to establishment --- .../commands/add_establishment_image.py | 30 +++++++++++++ apps/transfer/models.py | 42 ++++++++++--------- 2 files changed, 52 insertions(+), 20 deletions(-) create mode 100644 apps/establishment/management/commands/add_establishment_image.py diff --git a/apps/establishment/management/commands/add_establishment_image.py b/apps/establishment/management/commands/add_establishment_image.py new file mode 100644 index 00000000..7806bfe4 --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_image.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment +from transfer.models import EstablishmentAssets + + +class Command(BaseCommand): + help = 'Add preview_image_url values from old db to new db' + + def handle(self, *args, **kwargs): + count = 0 + url = 'https://1dc3f33f6d-3.optimicdn.com/gaultmillau.com/' + raw_qs = EstablishmentAssets.objects.raw( + '''SELECT establishment_assets.id, establishment_id, attachment_suffix_url + FROM establishment_assets + WHERE attachment_file_name IS NOT NULL + AND attachment_suffix_url IS NOT NULL + AND scope = 'public' + AND type = 'Photo' + AND menu_id IS NULL + GROUP BY establishment_id;''' + ) + queryset = [vars(query) for query in raw_qs] + for obj in queryset: + establishment = Establishment.objects.filter(old_id=obj['establishment_id']).first() + if establishment: + establishment.preview_image_url = url + obj['attachment_suffix_url'] + establishment.save() + count += 1 + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/apps/transfer/models.py b/apps/transfer/models.py index c431c504..fbcea0d8 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -99,7 +99,7 @@ class Ezuser(MigrateMixin): password_hash = models.CharField(max_length=50, blank=True, null=True) password_hash_type = models.IntegerField() facebook_id = models.BigIntegerField() - #TODO: в legacy нету таблицы 'CadLevel' + # TODO: в legacy нету таблицы 'CadLevel' # level = models.ForeignKey('CadLevel', models.DO_NOTHING) points = models.IntegerField() @@ -478,25 +478,27 @@ class Dishes(MigrateMixin): db_table = 'dishes' -# class EstablishmentAssets(MigrateMixin): -# using = 'legacy' -# -# establishment = models.ForeignKey('Establishments', models.DO_NOTHING) -# account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) -# menu_id = models.IntegerField(blank=True, null=True) -# type = models.CharField(max_length=64) -# scope = models.CharField(max_length=32) -# created_at = models.DateTimeField() -# updated_at = models.DateTimeField() -# attachment_file_name = models.CharField(max_length=255, blank=True, null=True) -# attachment_content_type = models.CharField(max_length=255, blank=True, null=True) -# geometries = models.CharField(max_length=1024, blank=True, null=True) -# attachment_file_size = models.IntegerField(blank=True, null=True) -# attachment_updated_at = models.DateTimeField(blank=True, null=True) -# -# class Meta: -# managed = False -# db_table = 'establishment_assets' +class EstablishmentAssets(MigrateMixin): + using = 'legacy' + + establishment = models.ForeignKey('Establishments', models.DO_NOTHING) + account = models.ForeignKey(Accounts, models.DO_NOTHING, blank=True, null=True) + menu_id = models.IntegerField(blank=True, null=True) + type = models.CharField(max_length=64) + scope = models.CharField(max_length=32) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + attachment_suffix_url = models.TextField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'establishment_assets' + class EstablishmentBacklinks(MigrateMixin): using = 'legacy' From a3bf92177163c56ade4e3a1008516127e8b0a3b0 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Tue, 29 Oct 2019 21:54:59 +0300 Subject: [PATCH 125/173] fix timezone --- apps/establishment/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/models.py b/apps/establishment/models.py index 4eacdd0d..7aea9dc5 100644 --- a/apps/establishment/models.py +++ b/apps/establishment/models.py @@ -438,7 +438,7 @@ class Establishment(ProjectBaseMixin, URLImageMixin, TranslatedFieldsMixin): @property def works_now(self): """ Is establishment working now """ - now_at_est_tz = datetime.now(tz=ptz(self.tz)) + now_at_est_tz = datetime.now(tz=self.tz) current_week = now_at_est_tz.weekday() schedule_for_today = self.schedule.filter(weekday=current_week).first() if schedule_for_today is None or schedule_for_today.closed_at is None or schedule_for_today.opening_at is None: From e524483ed01abdd30bb5fa35028333111927d600 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 30 Oct 2019 08:03:30 +0300 Subject: [PATCH 126/173] fix ass_publish_data --- .../management/commands/add_publish_data.py | 1 + docker-compose.mysql.yml | 14 -------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/apps/establishment/management/commands/add_publish_data.py b/apps/establishment/management/commands/add_publish_data.py index 1c5aae3e..81a52890 100644 --- a/apps/establishment/management/commands/add_publish_data.py +++ b/apps/establishment/management/commands/add_publish_data.py @@ -15,4 +15,5 @@ class Command(BaseCommand): if obj: count += 1 obj.is_publish = item.state == 'published' + obj.save() self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index 3c891dce..213dabe8 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -52,17 +52,9 @@ services: # Redis redis: image: redis:2.8.23 - ports: - - "6379:6379" networks: - redis_network - # RabbitMQ - #rabbitmq: - # image: rabbitmq:latest - # ports: - # - "5672:5672" - # Celery worker: build: . @@ -78,7 +70,6 @@ services: - .:/code links: - db -# - rabbitmq - redis networks: - redis_network @@ -97,7 +88,6 @@ services: - .:/code links: - db -# - rabbitmq - redis networks: - redis_network @@ -116,7 +106,6 @@ services: depends_on: - mysql_db - db -# - rabbitmq - redis - worker - worker_beat @@ -135,13 +124,10 @@ services: volumes: gm-mysql_db: name: gm-mysql_db - gm-db: name: gm-db - gm-media: name: gm-media - gm-esdata: networks: From 9e6634f1a6e1534966def39c29d8e146693f8236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 30 Oct 2019 10:48:59 +0300 Subject: [PATCH 127/173] Fix fab --- fabfile.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fabfile.py b/fabfile.py index 9ad7f871..8bcff1c2 100644 --- a/fabfile.py +++ b/fabfile.py @@ -53,12 +53,13 @@ def collectstatic(): def deploy(branch=None): - fetch() - install_requirements() - migrate() - collectstatic() - touch() - kill_celery() + if env.roledefs[role]['branch'] !='develop': + fetch() + install_requirements() + migrate() + collectstatic() + touch() + kill_celery() def rev(): From 24ee90e8a09d8a28bd6b9d534c0e7111e770a6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 30 Oct 2019 10:56:48 +0300 Subject: [PATCH 128/173] Fix fab --- fabfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fabfile.py b/fabfile.py index 8bcff1c2..595a0903 100644 --- a/fabfile.py +++ b/fabfile.py @@ -53,6 +53,7 @@ def collectstatic(): def deploy(branch=None): + role = env.roles[0] if env.roledefs[role]['branch'] !='develop': fetch() install_requirements() From 156a5479aa08dfe0b4287ee5bd830e8e7fc4a508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 30 Oct 2019 11:42:23 +0300 Subject: [PATCH 129/173] Fix --- fabfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 595a0903..e8a52f85 100644 --- a/fabfile.py +++ b/fabfile.py @@ -54,7 +54,7 @@ def collectstatic(): def deploy(branch=None): role = env.roles[0] - if env.roledefs[role]['branch'] !='develop': + if env.roledefs[role]['branch'] != 'develop': fetch() install_requirements() migrate() From 00bcd6c2050abaefe8ce0df5c9cbbf3b9bc4437d Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Wed, 30 Oct 2019 11:59:19 +0300 Subject: [PATCH 130/173] added merge migrations --- .../migrations/0046_merge_20191030_0858.py | 14 ++++++++++++++ .../migrations/0019_merge_20191030_0858.py | 14 ++++++++++++++ apps/news/migrations/0032_merge_20191030_0858.py | 14 ++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 apps/establishment/migrations/0046_merge_20191030_0858.py create mode 100644 apps/location/migrations/0019_merge_20191030_0858.py create mode 100644 apps/news/migrations/0032_merge_20191030_0858.py diff --git a/apps/establishment/migrations/0046_merge_20191030_0858.py b/apps/establishment/migrations/0046_merge_20191030_0858.py new file mode 100644 index 00000000..39153492 --- /dev/null +++ b/apps/establishment/migrations/0046_merge_20191030_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0045_auto_20191029_0719'), + ('establishment', '0044_position_index_name'), + ] + + operations = [ + ] diff --git a/apps/location/migrations/0019_merge_20191030_0858.py b/apps/location/migrations/0019_merge_20191030_0858.py new file mode 100644 index 00000000..268dc08f --- /dev/null +++ b/apps/location/migrations/0019_merge_20191030_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0018_address_old_id'), + ('location', '0013_wineappellation_wineregion'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0032_merge_20191030_0858.py b/apps/news/migrations/0032_merge_20191030_0858.py new file mode 100644 index 00000000..2936caef --- /dev/null +++ b/apps/news/migrations/0032_merge_20191030_0858.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 08:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0029_auto_20191025_1241'), + ('news', '0031_merge_20191029_0858'), + ] + + operations = [ + ] From 7805a6305585d33e9ae10bdbd88b4b8e7e488a39 Mon Sep 17 00:00:00 2001 From: evgeniy-st Date: Wed, 30 Oct 2019 12:15:40 +0300 Subject: [PATCH 131/173] remove establishment document filtering by country code --- apps/search_indexes/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index 72ce2efa..2bcd2ea4 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -61,9 +61,6 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): def get_queryset(self): qs = super(EstablishmentDocumentViewSet, self).get_queryset() qs = qs.filter('match', is_publish=True) - if self.request.country_code: - qs = qs.filter('term', - **{'address.city.country.code': self.request.country_code}) return qs filter_backends = [ From 9310bff3198e66fd8870a7b681ba38c388b14ffd Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 29 Oct 2019 16:02:29 +0300 Subject: [PATCH 132/173] Add comment serializer --- .../comment/migrations/0004_comment_old_id.py | 18 ++++++ apps/comment/models.py | 2 + apps/comment/transfer_data.py | 22 +++++++ apps/transfer/serializers/comments.py | 64 +++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 apps/comment/migrations/0004_comment_old_id.py create mode 100644 apps/comment/transfer_data.py create mode 100644 apps/transfer/serializers/comments.py diff --git a/apps/comment/migrations/0004_comment_old_id.py b/apps/comment/migrations/0004_comment_old_id.py new file mode 100644 index 00000000..46dbb7d0 --- /dev/null +++ b/apps/comment/migrations/0004_comment_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-29 12:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('comment', '0003_auto_20191015_0704'), + ] + + operations = [ + migrations.AddField( + model_name='comment', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/comment/models.py b/apps/comment/models.py index 55c7802e..29021677 100644 --- a/apps/comment/models.py +++ b/apps/comment/models.py @@ -45,6 +45,8 @@ class Comment(ProjectBaseMixin): objects = CommentQuerySet.as_manager() country = models.ForeignKey(Country, verbose_name=_('Country'), on_delete=models.SET_NULL, null=True) + old_id = models.IntegerField(null=True, blank=True, default=None) + class Meta: """Meta class""" diff --git a/apps/comment/transfer_data.py b/apps/comment/transfer_data.py new file mode 100644 index 00000000..d159a045 --- /dev/null +++ b/apps/comment/transfer_data.py @@ -0,0 +1,22 @@ +from django.db.models import F +from pprint import pprint +from transfer.models import Comments, Accounts, Establishments +from transfer.serializers.comments import CommentSerializer + + +def transfer_comments(): + queryset = Comments.objects.filter(account__isnull=False, mark__isnull=False)\ + .only("id", "comment", "mark", "locale", "establishment_id", "account_id") + + serialized_data = CommentSerializer(data=list(queryset.values()), many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(serialized_data.errors) + + +data_types = { + "tmp": [ + transfer_comments + ] +} \ No newline at end of file diff --git a/apps/transfer/serializers/comments.py b/apps/transfer/serializers/comments.py new file mode 100644 index 00000000..584370e6 --- /dev/null +++ b/apps/transfer/serializers/comments.py @@ -0,0 +1,64 @@ +from rest_framework import serializers +from comment.models import Comment, User +from establishment.models import Establishment + + +class CommentSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + comment = serializers.CharField() + mark = serializers.DecimalField(max_digits=4, decimal_places=2) + locale = serializers.CharField() + account_id = serializers.IntegerField() + establishment_id = serializers.CharField() + + class Meta: + model = Comment + fields = ( + "id", + "comment", + "mark", + "locale", + "account_id", + "establishment_id" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_text(data) + data = self.set_mark(data) + data = self.set_establishment(data) + data = self.set_account(data) + return data + + def set_text(self, data): + data['text'] = data.pop('comment') + return data + + def set_mark(self, data): + if data['mark'] < 0: + data['mark'] = data['mark'] * -1 + return data + + def set_account(self, data): + try: + data['account'] = User.objects.filter(old_id=data['account_id']).first() + except User.DoesNotExist as e: + raise ValueError(f"User account not found with {data}: {e}") + + del(data['account_id']) + + return data + + def set_establishment(self, data): + try: + data['establishment'] = Establishment.objects.filter(old_id=data['account_id']).first() + except Establishment.DoesNotExist as e: + raise ValueError(f"Establishment not found with {data}: {e}") + + del(data['establishment_id']) + + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data From c564f4fd67477dc1ec127730ec2c71dea6f950b8 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Tue, 29 Oct 2019 20:46:20 +0300 Subject: [PATCH 133/173] Fix rewiewers --- apps/comment/transfer_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/comment/transfer_data.py b/apps/comment/transfer_data.py index d159a045..0e8dbb94 100644 --- a/apps/comment/transfer_data.py +++ b/apps/comment/transfer_data.py @@ -1,4 +1,3 @@ -from django.db.models import F from pprint import pprint from transfer.models import Comments, Accounts, Establishments from transfer.serializers.comments import CommentSerializer @@ -19,4 +18,4 @@ data_types = { "tmp": [ transfer_comments ] -} \ No newline at end of file +} From 86e0699c8c08686a08e2a1485ed501d22f50725d Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 30 Oct 2019 12:30:46 +0300 Subject: [PATCH 134/173] Add reviews transfer Add language transfer Fix comment transfer --- apps/comment/transfer_data.py | 4 +- apps/review/transfer_data.py | 75 +++++++ apps/transfer/management/commands/transfer.py | 1 + apps/transfer/serializers/reviews.py | 191 ++++++++++++++++++ .../migrations/0006_language_old_id.py | 18 ++ apps/translation/models.py | 2 + 6 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 apps/review/transfer_data.py create mode 100644 apps/transfer/serializers/reviews.py create mode 100644 apps/translation/migrations/0006_language_old_id.py diff --git a/apps/comment/transfer_data.py b/apps/comment/transfer_data.py index 0e8dbb94..3a291b86 100644 --- a/apps/comment/transfer_data.py +++ b/apps/comment/transfer_data.py @@ -1,5 +1,5 @@ from pprint import pprint -from transfer.models import Comments, Accounts, Establishments +from transfer.models import Comments from transfer.serializers.comments import CommentSerializer @@ -16,6 +16,6 @@ def transfer_comments(): data_types = { "tmp": [ - transfer_comments + # transfer_comments ] } diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py new file mode 100644 index 00000000..c877b34a --- /dev/null +++ b/apps/review/transfer_data.py @@ -0,0 +1,75 @@ +from transfer.models import Reviews, ReviewTexts +from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment +from pprint import pprint + + +def transfer_languages(): + queryset = ReviewTexts.objects.raw("""SELECT id, locale + FROM review_texts + WHERE CHAR_LENGTH(locale) = 5 + AND review_texts.locale IS NOT NULL GROUP BY locale""") + + queryset = [vars(query) for query in queryset] + + serialized_data = LanguageSerializer(data=queryset, many=True) + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"Language serializer errors: {serialized_data.errors}") + + +def transfer_reviews(): + queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, + reviews.reviewer_id, review_texts.text AS text, reviews.mark, + review_texts.created_at AS published, review_texts.locale AS locale + FROM reviews + LEFT OUTER JOIN review_texts + ON (reviews.id = review_texts.review_id) + WHERE reviews.reviewer_id > 0 + AND reviews.reviewer_id IS NOT NULL + AND review_texts.text IS NOT NULL + AND reviews.reviewer_id IN ( + SELECT accounts.id + FROM accounts + WHERE accounts.confirmed_at IS NOT NULL + AND NOT accounts.email IN ( + "cyril@tomatic.net", + "cyril2@tomatic.net", + "cyril2@tomatic.net", + "d.sadykova@id-east.ru", + "d.sadykova@octopod.ru", + "n.yurchenko@id-east.ru" + ) + ) ORDER BY review_texts.created_at DESC + """) + + queryset_result = [] + establishments_mark_list = {} + + for query in queryset: + query = vars(query) + if query['establishment_id'] not in establishments_mark_list.keys(): + establishments_mark_list[query['establishment_id']] = query['mark'] + del(query['mark']) + queryset_result.append(query) + + serialized_data = ReviewSerializer(data=queryset_result, many=True) + + if serialized_data.is_valid(): + serialized_data.save() + for establishment_id, mark in establishments_mark_list.items(): + establishment = Establishment.objects.get(old_id=establishment_id) + establishment.public_mark = mark + establishment.save() + else: + pprint(serialized_data.errors) + + +data_types = { + "mention": [ + transfer_languages + ], + "tmp": [ + transfer_reviews + ] +} diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 1c146018..ac89dfd8 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -18,6 +18,7 @@ class Command(BaseCommand): 'establishment', 'gallery', 'commercial', + 'mention', 'tmp' ] diff --git a/apps/transfer/serializers/reviews.py b/apps/transfer/serializers/reviews.py new file mode 100644 index 00000000..e6bd11ac --- /dev/null +++ b/apps/transfer/serializers/reviews.py @@ -0,0 +1,191 @@ +from rest_framework import serializers +from review.models import Review +from account.models import User +from translation.models import Language +from establishment.models import Establishment + + +class ReviewSerializer(serializers.ModelSerializer): + id = serializers.IntegerField() + reviewer_id = serializers.IntegerField() + vintage = serializers.IntegerField() + published = serializers.DateTimeField() + establishment_id = serializers.IntegerField() + text = serializers.CharField() + locale = serializers.CharField() + + class Meta: + model = Review + fields = ( + "id", "reviewer_id", "published", "vintage", "establishment_id", "text", "locale" + ) + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_reviewer(data) + data = self.set_establishment(data) + data = self.set_language(data) + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data + + def set_reviewer(self, data): + try: + data['reviewer'] = User.objects.get(old_id=data['reviewer_id']) + except User.DoesNotExist as e: + raise ValueError(f"Cannot find reviewer with {data}: {e}") + del(data['reviewer_id']) + return data + + def set_establishment(self, data): + try: + data['establishment'] = Establishment.objects.get(old_id=data.pop('old_id')) + except Establishment.DoesNotExist as e: + raise ValueError(f"Cannot find establishment with {data}: {e}") + return data + + def set_language(self, data): + try: + data['language'] = Language.objects.get(locale=data['locale']) + except Language.DoesNotExist as e: + raise ValueError(f"Cannot find language with {data}: {e}") + + del(data['locale']) + + return data + + +class LanguageSerializer(serializers.ModelSerializer): + + LANGUAGES = { + "be-BE": "Belarusian - Belarusia", + "el-GR": "Greek - Greek", + "he-IL": "Israel - Hebrew", + "hr-HR": "Croatia - Croatian", + "hu-HU": "Hangarian - Hungary", + "ja-JP": "Japanese - Japan", + "ka-GE": "Georgian - Georgia", + "pl-PL": "Polish - Poland", + "ro-RO": "Romainian - Romania", + "ru-RU": "Russian - Russia", + "sl-SI": "Slovenian - Slovenia", + "ar-DZ": "Arabic - Algeria", + "ar-BH": "Arabic - Bahrain", + "ar-EG": "Arabic - Egypt", + "ar-IQ": "Arabic - Iraq", + "ar-JO": "Arabic - Jordan", + "ar-KW": "Arabic - Kuwait", + "ar-LB": "Arabic - Lebanon", + "ar-LY": "Arabic - Libya", + "ar-MA": "Arabic - Morocco", + "ar-OM": "Arabic - Oman", + "ar-QA": "Arabic - Qatar", + "ar-SA": "Arabic - Saudi Arabia", + "ar-SY": "Arabic - Syria", + "ar-TN": "Arabic - Tunisia", + "ar-AE": "Arabic - United Arab Emirates", + "ar-YE": "Arabic - Yemen", + "az-AZ": "Azeri - Cyrillic", + "zh-CN": "Chinese - China", + "zh-HK": "Chinese - Hong Kong SAR", + "zh-MO": "Chinese - Macau SAR", + "zh-SG": "Chinese - Singapore", + "zh-TW": "Chinese - Taiwan", + "nl-BE": "Dutch - Belgium", + "nl-NL": "Dutch - Netherlands", + "en-AU": "English - Australia", + "en-BZ": "English - Belize", + "en-CA": "English - Canada", + "en-CB": "English - Caribbean", + "en-GB": "English - Great Britain", + "en-IN": "English - India", + "en-IE": "English - Ireland", + "en-JM": "English - Jamaica", + "en-NZ": "English - New Zealand", + "en-PH": "English - Phillippines", + "en-ZA": "English - Southern Africa", + "en-TT": "English - Trinidad", + "en-US": "English - United States", + "fr-BE": "French - Belgium", + "fr-CA": "French - Canada", + "fr-FR": "French - France", + "fr-LU": "French - Luxembourg", + "fr-CH": "French - Switzerland", + "fr-MA": "French - Morocco", + "gd-IE": "Gaelic - Ireland", + "de-AT": "German - Austria", + "de-DE": "German - Germany", + "de-LI": "German - Liechtenstein", + "de-LU": "German - Luxembourg", + "de-CH": "German - Switzerland", + "it-IT": "Italian - Italy", + "it-CH": "Italian - Switzerland", + "ms-BN": "Malay - Brunei", + "ms-MY": "Malay - Malaysia", + "no-NO": "Norwegian - Bokml", + "pt-BR": "Portuguese - Brazil", + "pt-PT": "Portuguese - Portugal", + "ro-MO": "Romanian - Moldova", + "ru-MO": "Russian - Moldova", + "sr-SP": "Serbian - Cyrillic", + "es-AR": "Spanish - Argentina", + "es-BO": "Spanish - Bolivia", + "es-CL": "Spanish - Chile", + "es-CO": "Spanish - Colombia", + "es-CR": "Spanish - Costa Rica", + "es-DO": "Spanish - Dominican Republic", + "es-EC": "Spanish - Ecuador", + "es-SV": "Spanish - El Salvador", + "es-GT": "Spanish - Guatemala", + "es-HN": "Spanish - Honduras", + "es-MX": "Spanish - Mexico", + "es-NI": "Spanish - Nicaragua", + "es-PA": "Spanish - Panama", + "es-PY": "Spanish - Paraguay", + "es-PE": "Spanish - Peru", + "es-PR": "Spanish - Puerto Rico", + "es-ES": "Spanish - Spain (Traditional)", + "es-UY": "Spanish - Uruguay", + "es-VE": "Spanish - Venezuela", + "sv-FI": "Swedish - Finland", + "sv-SE": "Swedish - Sweden", + "uz-UZ": "Uzbek - Cyrillic", + } + + id = serializers.CharField() + locale = serializers.CharField() + + class Meta: + model = Language + fields = ("locale", "id") + + def validate(self, data): + data = self.set_old_id(data) + data = self.set_locale(data) + return data + + def create(self, validated_data): + try: + locale = Language.objects.filter(locale=validated_data['locale']).first() + if locale is None: + raise Language.DoesNotExist + locale.old_id = validated_data['old_id'] + locale.save() + except Language.DoesNotExist: + locale = Language.objects.create(**validated_data) + + return locale + + def set_locale(self, data): + data['locale'] = data['locale'].replace("_", "-") + try: + data['title'] = self.LANGUAGES[data['locale']] + except Exception as e: + raise ValueError(f"{data}: {e}") + return data + + def set_old_id(self, data): + data['old_id'] = data.pop("id") + return data diff --git a/apps/translation/migrations/0006_language_old_id.py b/apps/translation/migrations/0006_language_old_id.py new file mode 100644 index 00000000..ea7d0ce8 --- /dev/null +++ b/apps/translation/migrations/0006_language_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-30 05:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('translation', '0005_auto_20191021_1201'), + ] + + operations = [ + migrations.AddField( + model_name='language', + name='old_id', + field=models.IntegerField(blank=True, default=None, null=True), + ), + ] diff --git a/apps/translation/models.py b/apps/translation/models.py index cb0729ea..20b00233 100644 --- a/apps/translation/models.py +++ b/apps/translation/models.py @@ -25,6 +25,8 @@ class Language(models.Model): locale = models.CharField(max_length=10, verbose_name=_('Locale identifier')) + old_id = models.IntegerField(null=True, blank=True, default=None) + objects = LanguageQuerySet.as_manager() class Meta: From 37c3a4393237027a9df6b7209296ee29cb473ba3 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 30 Oct 2019 12:31:24 +0300 Subject: [PATCH 135/173] Mv reviews to mention command --- apps/review/transfer_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index c877b34a..a24cb135 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -67,9 +67,7 @@ def transfer_reviews(): data_types = { "mention": [ - transfer_languages - ], - "tmp": [ + transfer_languages, transfer_reviews ] } From f690ad3b8feea618a9aec14a89f5ca52bff25484 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 30 Oct 2019 12:48:04 +0300 Subject: [PATCH 136/173] Fix key --- apps/review/transfer_data.py | 2 +- apps/transfer/management/commands/transfer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index a24cb135..03ace312 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -66,7 +66,7 @@ def transfer_reviews(): data_types = { - "mention": [ + "overlook": [ transfer_languages, transfer_reviews ] diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index ac89dfd8..20206b57 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -18,7 +18,7 @@ class Command(BaseCommand): 'establishment', 'gallery', 'commercial', - 'mention', + 'overlook', 'tmp' ] From a67df3208687adbe27a216d95474fdd6ca05da12 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Wed, 30 Oct 2019 12:55:44 +0300 Subject: [PATCH 137/173] Fix establishment_mark --- apps/review/transfer_data.py | 2 +- project/settings/base.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index 03ace312..80d67f0b 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -49,7 +49,7 @@ def transfer_reviews(): for query in queryset: query = vars(query) if query['establishment_id'] not in establishments_mark_list.keys(): - establishments_mark_list[query['establishment_id']] = query['mark'] + establishments_mark_list[query['establishment_id']] = int(query['mark']) del(query['mark']) queryset_result.append(query) diff --git a/project/settings/base.py b/project/settings/base.py index 2f05ca52..676a67d1 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,9 +159,9 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - # 'HOST': '172.17.0.1', + 'HOST': '172.17.0.1', # 'HOST': '172.23.0.1', - 'HOST': 'mysql_db', + # 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From c5218393894481ec4d6351d6cde847f43db3d763 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 30 Oct 2019 13:53:12 +0300 Subject: [PATCH 138/173] news refactor --- apps/news/transfer_data.py | 34 ++++++--- apps/transfer/models.py | 4 +- apps/transfer/serializers/news.py | 116 ++++++++++++++++-------------- docker-compose.mysql.yml | 2 +- 4 files changed, 91 insertions(+), 65 deletions(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 791b54ce..7524a6cc 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,28 +1,46 @@ from pprint import pprint -from django.db.models import Value, IntegerField, F +from django.db.models import Aggregate, CharField, Value +from django.db.models import IntegerField, F from news.models import NewsType from transfer.models import PageTexts from transfer.serializers.news import NewsSerializer -def transfer_news(): - news_type, _ = NewsType.objects.get_or_create(name="News") +class GroupConcat(Aggregate): + function = 'GROUP_CONCAT' + template = '%(function)s(%(expressions)s)' - queryset = PageTexts.objects.filter(page__type="News").annotate( + def __init__(self, expression, **extra): + output_field = extra.pop('output_field', CharField()) + super().__init__(expression, output_field=output_field, **extra) + + def as_postgresql(self, compiler, connection): + self.function = 'STRING_AGG' + return super().as_sql(compiler, connection) + + +def transfer_news(): + news_type, _ = NewsType.objects.get_or_create(name='News') + queryset = PageTexts.objects.filter( + page__type='News', + ).annotate( news_type=Value(news_type.id, output_field=IntegerField()), country_code=F('page__site__country_code_2'), - ) - queryset = queryset.annotate(template=F('page__template')) + news_title=F('page__root_title'), + image=F('page__attachment_suffix_url'), + template=F('page__template'), + tags=GroupConcat('page__tags__id'), + )[:100] serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + pprint(f'News serializer errors: {serialized_data.errors}') data_types = { - "news": [transfer_news] + 'news': [transfer_news] } diff --git a/apps/transfer/models.py b/apps/transfer/models.py index fbcea0d8..0f7d3348 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -761,6 +761,7 @@ class Pages(MigrateMixin): attachment_content_type = models.CharField(max_length=255, blank=True, null=True) attachment_file_size = models.IntegerField(blank=True, null=True) attachment_updated_at = models.DateTimeField(blank=True, null=True) + attachment_suffix_url = models.TextField(blank=True, null=True) geometries = models.CharField(max_length=1024, blank=True, null=True) scheduled_at = models.DateTimeField(blank=True, null=True) created_at = models.DateTimeField() @@ -785,7 +786,6 @@ class PageTexts(MigrateMixin): locale = models.CharField(max_length=255, blank=True, null=True) state = models.CharField(max_length=255, blank=True, null=True) page = models.ForeignKey(Pages, models.DO_NOTHING, blank=True, null=True) - # page_id = models.IntegerField(blank=True, null=True) created_at = models.DateTimeField() updated_at = models.DateTimeField() summary = models.TextField(blank=True, null=True) @@ -814,7 +814,7 @@ class PageMetadata(MigrateMixin): key = models.CharField(max_length=255, blank=True, null=True) value = models.CharField(max_length=255, blank=True, null=True) - page_id = models.IntegerField(blank=True, null=True) + page = models.ForeignKey('Pages', models.DO_NOTHING, blank=True, null=True, related_name='tags') created_at = models.DateTimeField() updated_at = models.DateTimeField() diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 82853cad..cafbe994 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -2,64 +2,67 @@ from rest_framework import serializers from location.models import Country from news.models import News +from tag.models import TagCategory from utils.legacy_parser import parse_legacy_news_content from utils.slug_generator import generate_unique_slug -class NewsSerializer(serializers.ModelSerializer): - locale = serializers.CharField() - slug = serializers.CharField() - body = serializers.CharField(allow_null=True) +class NewsSerializer(serializers.Serializer): + old_id = serializers.IntegerField() + news_type = serializers.IntegerField() + news_title = serializers.CharField() title = serializers.CharField() - template = serializers.CharField() + body = serializers.CharField(allow_null=True) + created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') + slug = serializers.CharField() state = serializers.CharField() + template = serializers.CharField() country_code = serializers.CharField(allow_null=True) - created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') - - class Meta: - model = News - fields = ( - 'old_id', - 'created_at', - 'state', - 'template', - 'title', - 'body', - 'slug', - 'news_type', - 'locale', - 'country_code', - ) - - def validate(self, data): - data.update({ - 'slug': generate_unique_slug(News, data['slug']), - 'state': self.get_state(data), - 'template': self.get_template(data), - 'title': self.get_title(data), - 'description': self.get_description(data), - 'subtitle': self.get_description(data), - 'country': self.get_country(data), - }) - data.pop('country_code') - data.pop('body') - data.pop('locale') - return data + locale = serializers.CharField() + image = serializers.CharField() + tags = serializers.CharField(allow_null=True) def create(self, validated_data): - return News.objects.create(**validated_data) - @staticmethod - def get_country(data): - return Country.objects.filter(code__iexact=data['country_code']).first() - - @staticmethod - def get_template(data): - templates = { - 'main': News.MAIN, - 'main.pdf.erb': News.MAIN_PDF_ERB, + payload = { + 'old_id': validated_data['old_id'], + 'news_type': validated_data['news_type'], + 'title': {validated_data['locale']: validated_data['news_title']}, + 'subtitle': {validated_data['locale']: validated_data['title']}, + 'description': self.get_description(validated_data), + 'start': validated_data['created_at'], + 'slug': generate_unique_slug(News, validated_data['slug']), + 'state': self.get_state(validated_data), + 'template': self.get_template(validated_data), + 'country': self.get_country(validated_data), } - return templates.get(data['template'], News.MAIN) + obj = News.objects.create(**payload) + + tags = self.get_tags(validated_data) + + return obj + + # TODO: теги + # TODO: галерея с картинкой + + @staticmethod + def get_tags(data): + if not data['tags']: + return None + tag_cat, _ = TagCategory.objects.get_or_create( + public=True, + label={'en-GB': 'tag'}, + index_name='tag', + country='tag', + ) + + + @staticmethod + def get_description(data): + if data['body']: + content = parse_legacy_news_content(data['body']) + return {data['locale']: content} + return None @staticmethod def get_state(data): @@ -73,12 +76,17 @@ class NewsSerializer(serializers.ModelSerializer): return states.get(data['state'], News.WAITING) @staticmethod - def get_title(data): - return {data['locale']: data['title']} + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + } + return templates.get(data['template'], News.MAIN) @staticmethod - def get_description(data): - content = None - if data['body']: - content = parse_legacy_news_content(data['body']) - return {data['locale']: content} + def get_country(data): + return Country.objects.filter(code__iexact=data['country_code']).first() + + @staticmethod + def get_title(data): + return {data['locale']: data['title']} diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index 213dabe8..4d4e553e 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -51,7 +51,7 @@ services: # Redis redis: - image: redis:2.8.23 + image: redis:alpine networks: - redis_network From 42c596b5920d384b8b61d13a1f0fedae44ebc84f Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Wed, 16 Oct 2019 14:11:07 +0300 Subject: [PATCH 139/173] Weekday for mobile devices --- apps/timetable/serialziers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/timetable/serialziers.py b/apps/timetable/serialziers.py index babe33c1..37725e1d 100644 --- a/apps/timetable/serialziers.py +++ b/apps/timetable/serialziers.py @@ -27,6 +27,7 @@ class ScheduleRUDSerializer(serializers.ModelSerializer): fields = [ 'id', 'weekday_display', + 'weekday', 'lunch_start', 'lunch_end', 'dinner_start', From 6fcfb263d3548d4475f88c9eecd412e88bf176a8 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Wed, 30 Oct 2019 14:23:40 +0300 Subject: [PATCH 140/173] fix ProductModelAdmin --- apps/product/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/product/admin.py b/apps/product/admin.py index b3dbc0cd..b77f2cbc 100644 --- a/apps/product/admin.py +++ b/apps/product/admin.py @@ -1,10 +1,11 @@ """Product admin conf.""" from django.contrib import admin +from utils.admin import BaseModelAdminMixin from .models import Product, ProductType, ProductSubType @admin.register(Product) -class ProductAdmin(admin.ModelAdmin): +class ProductAdmin(BaseModelAdminMixin, admin.ModelAdmin): """Admin page for model Product.""" From 8c3306628ea5cce3cf478fc187ce72fe7baa8bf1 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 30 Oct 2019 15:48:32 +0300 Subject: [PATCH 141/173] news transfer --- apps/news/transfer_data.py | 10 ++++-- apps/transfer/serializers/news.py | 57 +++++++++++++++++++++++-------- project/settings/base.py | 4 +-- project/settings/development.py | 4 +-- 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index 7524a6cc..bdd1c513 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -4,6 +4,7 @@ from django.db.models import Aggregate, CharField, Value from django.db.models import IntegerField, F from news.models import NewsType +from tag.models import TagCategory from transfer.models import PageTexts from transfer.serializers.news import NewsSerializer @@ -23,16 +24,21 @@ class GroupConcat(Aggregate): def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name='News') + tag_cat, _ = TagCategory.objects.get_or_create(index_name='tag', public=True) + news_type.tag_categories.add(tag_cat) + news_type.save() + queryset = PageTexts.objects.filter( page__type='News', ).annotate( - news_type=Value(news_type.id, output_field=IntegerField()), + tag_cat_id=Value(tag_cat.id, output_field=IntegerField()), + news_type_id=Value(news_type.id, output_field=IntegerField()), country_code=F('page__site__country_code_2'), news_title=F('page__root_title'), image=F('page__attachment_suffix_url'), template=F('page__template'), tags=GroupConcat('page__tags__id'), - )[:100] + ) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index cafbe994..4159be67 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,15 +1,18 @@ from rest_framework import serializers +from gallery.models import Image from location.models import Country -from news.models import News -from tag.models import TagCategory +from news.models import News, NewsGallery +from tag.models import Tag +from transfer.models import PageMetadata from utils.legacy_parser import parse_legacy_news_content from utils.slug_generator import generate_unique_slug class NewsSerializer(serializers.Serializer): - old_id = serializers.IntegerField() - news_type = serializers.IntegerField() + id = serializers.IntegerField() + tag_cat_id = serializers.IntegerField() + news_type_id = serializers.IntegerField() news_title = serializers.CharField() title = serializers.CharField() body = serializers.CharField(allow_null=True) @@ -25,8 +28,8 @@ class NewsSerializer(serializers.Serializer): def create(self, validated_data): payload = { - 'old_id': validated_data['old_id'], - 'news_type': validated_data['news_type'], + 'old_id': validated_data['id'], + 'news_type_id': validated_data['news_type_id'], 'title': {validated_data['locale']: validated_data['news_title']}, 'subtitle': {validated_data['locale']: validated_data['title']}, 'description': self.get_description(validated_data), @@ -39,23 +42,47 @@ class NewsSerializer(serializers.Serializer): obj = News.objects.create(**payload) tags = self.get_tags(validated_data) + for tag in tags: + obj.tags.add(tag) + obj.save() + self.make_gallery(validated_data, obj) return obj - # TODO: теги - # TODO: галерея с картинкой + @staticmethod + def make_gallery(data, obj): + if not data['image'] or data['image'] == 'default/missing.png': + return + + img = Image.objects.create( + image=data['image'], + title=data['news_title'], + ) + NewsGallery.objects.create( + news=obj, + image=img, + is_main=True, + ) @staticmethod def get_tags(data): + results = [] if not data['tags']: - return None - tag_cat, _ = TagCategory.objects.get_or_create( - public=True, - label={'en-GB': 'tag'}, - index_name='tag', - country='tag', - ) + return results + meta_ids = (int(_id) for _id in data['tags'].split(',')) + tags = PageMetadata.objects.filter( + id__in=meta_ids, + key='tag', + value__isnull=False, + ) + for old_tag in tags: + tag, _ = Tag.objects.get_or_create( + category_id=data['tag_cat_id'], + label={data['locale']: old_tag.value}, + ) + results.append(tag) + return results @staticmethod def get_description(data): diff --git a/project/settings/base.py b/project/settings/base.py index 3d3ec8ca..6979382e 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -329,8 +329,8 @@ REDOC_SETTINGS = { # RabbitMQ # BROKER_URL = 'amqp://rabbitmq:5672' # Redis -BROKER_URL = 'redis://localhost:6379/1' -# BROKER_URL = 'redis://redis:6379/1' +# BROKER_URL = 'redis://localhost:6379/1' +BROKER_URL = 'redis://redis:6379/1' CELERY_RESULT_BACKEND = BROKER_URL CELERY_BROKER_URL = BROKER_URL CELERY_ACCEPT_CONTENT = ['application/json'] diff --git a/project/settings/development.py b/project/settings/development.py index a9cfe5be..b9f03146 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,8 +19,8 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'localhost:9200' - # 'hosts': 'elasticsearch:9200' + # 'hosts': 'localhost:9200' + 'hosts': 'elasticsearch:9200' } } From b74b87d78da7c089a8fe8dfcaa5e69f8cb943905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 30 Oct 2019 17:23:45 +0300 Subject: [PATCH 142/173] Add products model in transfer --- apps/product/management/__init__.py | 0 apps/product/management/commands/__init__.py | 0 .../management/commands/add_product.py | 0 apps/product/old_models.py | 0 apps/transfer/models.py | 42 +++++++++++++++++++ 5 files changed, 42 insertions(+) create mode 100644 apps/product/management/__init__.py create mode 100644 apps/product/management/commands/__init__.py create mode 100644 apps/product/management/commands/add_product.py create mode 100644 apps/product/old_models.py diff --git a/apps/product/management/__init__.py b/apps/product/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/product/management/commands/__init__.py b/apps/product/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/product/management/commands/add_product.py b/apps/product/management/commands/add_product.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/product/old_models.py b/apps/product/old_models.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 0f7d3348..fecbc48b 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -841,3 +841,45 @@ class Ads(MigrateMixin): class Meta: managed = False db_table = 'ads' + +class Products(models.Model): + establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) + brand = models.CharField(max_length=255, blank=True, null=True) + name = models.CharField(max_length=255, blank=True, null=True) + vintage = models.CharField(max_length=255, blank=True, null=True) + type = models.CharField(max_length=255, blank=True, null=True) + unique_key = models.CharField(max_length=255, blank=True, null=True) + price = models.FloatField(blank=True, null=True) + average_price_in_shops = models.FloatField(blank=True, null=True) + fra_encima_id = models.IntegerField(blank=True, null=True) + wine_sub_region_id = models.IntegerField(blank=True, null=True) + classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) + wine_region = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) + wine_type = models.ForeignKey('WineTypes', models.DO_NOTHING, blank=True, null=True) + wine_color = models.ForeignKey('WineColors', models.DO_NOTHING, blank=True, null=True) + appellation = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + state = models.CharField(max_length=255, blank=True, null=True) + wine_style = models.ForeignKey('WineStyles', models.DO_NOTHING, blank=True, null=True) + village = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) + vineyard = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) + yard_classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) + wine_quality = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) + bottles_produced = models.CharField(max_length=3000, blank=True, null=True) + deu_import_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'products' + + +class WineTypes(models.Model): + name = models.CharField(max_length=255, blank=True, null=True) + fra_encima_id = models.IntegerField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'wine_types' From f6151c0a14a4fdfd124656b84564c066b0775098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 30 Oct 2019 17:24:40 +0300 Subject: [PATCH 143/173] delete okd_models from product --- apps/product/old_models.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 apps/product/old_models.py diff --git a/apps/product/old_models.py b/apps/product/old_models.py deleted file mode 100644 index e69de29b..00000000 From c1c9f99fb56f09dc83a1e439e56e973062f72433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Wed, 30 Oct 2019 18:04:30 +0300 Subject: [PATCH 144/173] Add product migrate data --- .../management/commands/add_product.py | 54 +++++++++++++++++++ apps/product/models.py | 18 +++++++ apps/transfer/models.py | 1 + 3 files changed, 73 insertions(+) diff --git a/apps/product/management/commands/add_product.py b/apps/product/management/commands/add_product.py index e69de29b..981487ec 100644 --- a/apps/product/management/commands/add_product.py +++ b/apps/product/management/commands/add_product.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand +from product.models import ProductType, ProductSubType + + +def add_type(): + product_type = ProductType.objects.create( + name={'"en-GB"': "Wine"}, + index_name=ProductType.WINE + ) + return product_type.save() + + +def add_subtype(id_type): + subtypes = ProductSubType.objects.bulk_create([ + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.EXTRA_BRUT}, + index_name=ProductSubType.EXTRA_BRUT), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.BRUT}, + index_name=ProductSubType.BRUT), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.BRUT_NATURE}, + index_name=ProductSubType.BRUT_NATURE), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.DEMI_SEC}, + index_name=ProductSubType.DEMI_SEC), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.EXTRA_DRY}, + index_name=ProductSubType.EXTRA_DRY), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.DOSAGE_ZERO}, + index_name=ProductSubType.DOSAGE_ZERO), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.SEC}, + index_name=ProductSubType.SEC), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.SEC}, + index_name=ProductSubType.SEC), + ProductSubType(product_type=id_type, + name={"en-GB", ProductSubType.MOELLEUX}, + index_name=ProductSubType.MOELLEUX), + ]) + + +class Command(BaseCommand): + help = 'Add product data' + + def handle(self, *args, **kwarg): + product_type = add_type() + add_subtype(product_type.id) + + + + diff --git a/apps/product/models.py b/apps/product/models.py index d4011fa0..332a92f6 100644 --- a/apps/product/models.py +++ b/apps/product/models.py @@ -46,10 +46,28 @@ class ProductSubType(TranslatedFieldsMixin, ProjectBaseMixin): # INDEX NAME CHOICES RUM = 'rum' OTHER = 'other' + EXTRA_BRUT = 'extra brut' + BRUT = 'brut' + BRUT_NATURE = 'brut nature' + DEMI_SEC = 'demi-sec' + EXTRA_DRY = 'Extra Dry' + DOSAGE_ZERO = 'dosage zero' + SEC = 'sec' + DOUX = 'doux' + MOELLEUX= 'moelleux' INDEX_NAME_TYPES = ( (RUM, _('Rum')), (OTHER, _('Other')), + (EXTRA_BRUT, _('extra brut')), + (BRUT, _('brut')), + (BRUT_NATURE, _('brut nature')), + (DEMI_SEC, _('demi-sec')), + (EXTRA_DRY, _('Extra Dry')), + (DOSAGE_ZERO, _('dosage zero')), + (SEC, _('sec')), + (DOUX, _('doux')), + (MOELLEUX, _('moelleux')) ) product_type = models.ForeignKey(ProductType, on_delete=models.CASCADE, diff --git a/apps/transfer/models.py b/apps/transfer/models.py index fecbc48b..1a978596 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -842,6 +842,7 @@ class Ads(MigrateMixin): managed = False db_table = 'ads' + class Products(models.Model): establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) brand = models.CharField(max_length=255, blank=True, null=True) From 58be2491a55151db539df2252940683af17bfee3 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Wed, 30 Oct 2019 18:47:13 +0300 Subject: [PATCH 145/173] create command for establishment tags --- .../migrations/0032_auto_20191030_1149.py | 18 ++++++ apps/news/models.py | 2 +- apps/tag/management/__init__.py | 0 apps/tag/management/commands/__init__.py | 0 apps/tag/management/commands/add_tags.py | 62 +++++++++++++++++++ .../tag/migrations/0006_auto_20191030_1151.py | 23 +++++++ .../tag/migrations/0007_auto_20191030_1514.py | 18 ++++++ apps/tag/models.py | 17 ++++- apps/transfer/models.py | 54 ++++++++++++++++ apps/transfer/serializers/news.py | 11 +++- 10 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 apps/news/migrations/0032_auto_20191030_1149.py create mode 100644 apps/tag/management/__init__.py create mode 100644 apps/tag/management/commands/__init__.py create mode 100644 apps/tag/management/commands/add_tags.py create mode 100644 apps/tag/migrations/0006_auto_20191030_1151.py create mode 100644 apps/tag/migrations/0007_auto_20191030_1514.py diff --git a/apps/news/migrations/0032_auto_20191030_1149.py b/apps/news/migrations/0032_auto_20191030_1149.py new file mode 100644 index 00000000..75408f46 --- /dev/null +++ b/apps/news/migrations/0032_auto_20191030_1149.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-30 11:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0031_merge_20191029_0858'), + ] + + operations = [ + migrations.AlterField( + model_name='news', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/news/models.py b/apps/news/models.py index f0caa13c..6508215d 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -126,7 +126,7 @@ class News(BaseAttributes, TranslatedFieldsMixin): (PUBLISHED_EXCLUSIVE, _('Published exclusive')), ) - old_id = models.PositiveIntegerField(_('odl id'), blank=True, null=True, default=None) + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) news_type = models.ForeignKey(NewsType, on_delete=models.PROTECT, verbose_name=_('news type')) title = TJSONField(blank=True, null=True, default=None, diff --git a/apps/tag/management/__init__.py b/apps/tag/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/tag/management/commands/__init__.py b/apps/tag/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py new file mode 100644 index 00000000..5fc721b7 --- /dev/null +++ b/apps/tag/management/commands/add_tags.py @@ -0,0 +1,62 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment, EstablishmentType +from transfer import models as legacy +from tag.models import Tag, TagCategory + + +class Command(BaseCommand): + help = 'Add tags values from old db to new db' + + def handle(self, *args, **kwargs): + + categories = legacy.KeyValueMetadata.objects.all() + existing_establishment = Establishment.objects.filter(old_id__isnull=False) + for category in categories: + ESTABLISHMENT = 1 + SHOP = 2 + RESTAURANT = 3 + WINEYARD = 4 + + MAPPER = { + RESTAURANT: EstablishmentType.RESTAURANT, + WINEYARD: EstablishmentType.PRODUCER, + } + + mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all() + for key, value in MAPPER.items(): + values_meta_id_list = mapper_values_meta.filter( + key_value_metadatum_establishment_id=key + ).values_list('key_value_metadatum_id') + + est_type, _ = EstablishmentType.objects.get_or_create(index_name=value) + + key_value_metadata = legacy.KeyValueMetadata.objects.filter(id__in=values_meta_id_list) + + # create TagCategory + for key_value in key_value_metadata: + tag_category, created = TagCategory.objects.get_or_create( + index_name=key_value.key_name, + ) + + if created: + tag_category.label = {'en-GB': key_value.key_name}, + tag_category.value_type = key_value.value_type + tag_category.save() + est_type.tag_categories.add( + tag_category + ) + + # create Tag + for tag in key_value.metadata_set.filter( + establishment__id__in=list(existing_establishment.values_list('old_id', flat=True))): + + new_tag, _ = Tag.objects.get_or_create( + label={'en-GB': tag.value}, + value=tag.value, + category=tag_category, + ) + est = existing_establishment.get(old_id=tag.establishment_id) + est.tags.add(new_tag) + est.save() + diff --git a/apps/tag/migrations/0006_auto_20191030_1151.py b/apps/tag/migrations/0006_auto_20191030_1151.py new file mode 100644 index 00000000..b11a4c20 --- /dev/null +++ b/apps/tag/migrations/0006_auto_20191030_1151.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.4 on 2019-10-30 11:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0005_tagcategory_name_indexing'), + ] + + operations = [ + migrations.AddField( + model_name='tag', + name='value', + field=models.CharField(blank=True, default=None, max_length=255, null=True, unique=True, verbose_name='indexing name'), + ), + migrations.AddField( + model_name='tagcategory', + name='value_type', + field=models.CharField(choices=[('string', 'string'), ('list', 'list'), ('integer', 'integer')], default='list', max_length=255, verbose_name='value type'), + ), + ] diff --git a/apps/tag/migrations/0007_auto_20191030_1514.py b/apps/tag/migrations/0007_auto_20191030_1514.py new file mode 100644 index 00000000..1bcb3deb --- /dev/null +++ b/apps/tag/migrations/0007_auto_20191030_1514.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-30 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0006_auto_20191030_1151'), + ] + + operations = [ + migrations.AlterField( + model_name='tag', + name='value', + field=models.CharField(blank=True, default=None, max_length=255, null=True, verbose_name='indexing name'), + ), + ] diff --git a/apps/tag/models.py b/apps/tag/models.py index 26079849..e3967f0f 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -19,6 +19,8 @@ class Tag(TranslatedFieldsMixin, models.Model): label = TJSONField(blank=True, null=True, default=None, verbose_name=_('label'), help_text='{"en-GB":"some text"}') + value = models.CharField(_('indexing name'), max_length=255, blank=True, + null=True, default=None) category = models.ForeignKey('TagCategory', on_delete=models.CASCADE, null=True, related_name='tags', verbose_name=_('Category')) @@ -72,6 +74,16 @@ class TagCategoryQuerySet(models.QuerySet): class TagCategory(TranslatedFieldsMixin, models.Model): """Tag base category model.""" + STRING = 'string' + LIST = 'list' + INTEGER = 'integer' + + VALUE_TYPE_CHOICES = ( + (STRING, _('string')), + (LIST, _('list')), + (INTEGER, _('integer')), + ) + label = TJSONField(blank=True, null=True, default=None, verbose_name=_('label'), help_text='{"en-GB":"some text"}') @@ -80,7 +92,10 @@ class TagCategory(TranslatedFieldsMixin, models.Model): default=None) public = models.BooleanField(default=False) index_name = models.CharField(max_length=255, blank=True, null=True, - verbose_name=_('indexing name'), unique=True) + verbose_name=_('indexing name'), unique=True) + + value_type = models.CharField(_('value type'), max_length=255, + choices=VALUE_TYPE_CHOICES, default=LIST, ) objects = TagCategoryQuerySet.as_manager() diff --git a/apps/transfer/models.py b/apps/transfer/models.py index fbcea0d8..99dfd0ac 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -841,3 +841,57 @@ class Ads(MigrateMixin): class Meta: managed = False db_table = 'ads' + + +class KeyValueMetadata(MigrateMixin): + using = 'legacy' + + key_name = models.CharField(max_length=255, blank=True, null=True) + value_type = models.CharField(max_length=255, blank=True, null=True) + value_list = models.TextField(blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + public = models.IntegerField(blank=True, null=True) + site_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'key_value_metadata' + + +class Metadata(MigrateMixin): + using = 'legacy' + + key = models.CharField(max_length=255, blank=True, null=True) + value = models.CharField(max_length=255, blank=True, null=True) + establishment = models.ForeignKey('transfer.Establishments', models.DO_NOTHING, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + key_value_metadatum = models.ForeignKey('transfer.KeyValueMetadata', models.DO_NOTHING, blank=True, null=True) + + class Meta: + managed = False + db_table = 'metadata' + + +class KeyValueMetadatumEstablishments(MigrateMixin): + using = 'legacy' + + name = models.CharField(max_length=255, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'key_value_metadatum_establishments' + + +class KeyValueMetadatumKeyValueMetadatumEstablishments(MigrateMixin): + using = 'legacy' + + key_value_metadatum_id = models.IntegerField(blank=True, null=True) + key_value_metadatum_establishment_id = models.IntegerField(blank=True, null=True) + + class Meta: + managed = False + db_table = 'key_value_metadatum_key_value_metadatum_establishments' diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 82853cad..6842f248 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -15,6 +15,7 @@ class NewsSerializer(serializers.ModelSerializer): state = serializers.CharField() country_code = serializers.CharField(allow_null=True) created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') + summary = serializers.CharField(allow_null=True, allow_blank=True) class Meta: model = News @@ -29,6 +30,7 @@ class NewsSerializer(serializers.ModelSerializer): 'news_type', 'locale', 'country_code', + 'summary' ) def validate(self, data): @@ -38,7 +40,7 @@ class NewsSerializer(serializers.ModelSerializer): 'template': self.get_template(data), 'title': self.get_title(data), 'description': self.get_description(data), - 'subtitle': self.get_description(data), + 'subtitle': self.get_subtitle(data), 'country': self.get_country(data), }) data.pop('country_code') @@ -82,3 +84,10 @@ class NewsSerializer(serializers.ModelSerializer): if data['body']: content = parse_legacy_news_content(data['body']) return {data['locale']: content} + + @staticmethod + def get_subtitle(data): + if 'summary' in data: + content = data.pop('summary') + if content: + return {data['locale']: content} From 523e3a1ca0cf5b3198fbd96ed0a0df59bc3fd507 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Wed, 30 Oct 2019 18:59:16 +0300 Subject: [PATCH 146/173] merge work in subtitle --- apps/transfer/serializers/news.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 49b64476..311cee6a 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -28,7 +28,7 @@ class NewsSerializer(serializers.Serializer): 'old_id': validated_data['old_id'], 'news_type': validated_data['news_type'], 'title': {validated_data['locale']: validated_data['news_title']}, - 'subtitle': {validated_data['locale']: validated_data['title']}, + 'subtitle': self.get_subtitle(validated_data), 'description': self.get_description(validated_data), 'start': validated_data['created_at'], 'slug': generate_unique_slug(News, validated_data['slug']), @@ -91,16 +91,10 @@ class NewsSerializer(serializers.Serializer): def get_title(data): return {data['locale']: data['title']} - @staticmethod - def get_description(data): - content = None - if data['body']: - content = parse_legacy_news_content(data['body']) - return {data['locale']: content} - @staticmethod def get_subtitle(data): if 'summary' in data: content = data.pop('summary') - if content: - return {data['locale']: content} + if not content: + content = {data['locale']: data['title']} + return {data['locale']: content} From 887bc92b850c8afce9383cf92a5cb38cb704f779 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Wed, 30 Oct 2019 19:17:50 +0300 Subject: [PATCH 147/173] fix merge --- project/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/settings/base.py b/project/settings/base.py index 3a23122a..5a52f212 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -75,7 +75,7 @@ PROJECT_APPS = [ 'favorites.apps.FavoritesConfig', 'rating.apps.RatingConfig', 'transfer.apps.TransferConfig', - 'tag.apps.TagConfig' + 'tag.apps.TagConfig', 'product.apps.ProductConfig', ] From d31bb9b495629ecb780ffdc7079e3839d494469d Mon Sep 17 00:00:00 2001 From: "a.feteleu" Date: Wed, 30 Oct 2019 19:19:40 +0300 Subject: [PATCH 148/173] merge migrations --- .../migrations/0046_merge_20191030_1618.py | 14 ++++++++++++++ .../migrations/0019_merge_20191030_1618.py | 14 ++++++++++++++ apps/news/migrations/0033_merge_20191030_1618.py | 14 ++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 apps/establishment/migrations/0046_merge_20191030_1618.py create mode 100644 apps/location/migrations/0019_merge_20191030_1618.py create mode 100644 apps/news/migrations/0033_merge_20191030_1618.py diff --git a/apps/establishment/migrations/0046_merge_20191030_1618.py b/apps/establishment/migrations/0046_merge_20191030_1618.py new file mode 100644 index 00000000..853b25eb --- /dev/null +++ b/apps/establishment/migrations/0046_merge_20191030_1618.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 16:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0045_auto_20191029_0719'), + ('establishment', '0044_position_index_name'), + ] + + operations = [ + ] diff --git a/apps/location/migrations/0019_merge_20191030_1618.py b/apps/location/migrations/0019_merge_20191030_1618.py new file mode 100644 index 00000000..74d64c63 --- /dev/null +++ b/apps/location/migrations/0019_merge_20191030_1618.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 16:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0018_address_old_id'), + ('location', '0013_wineappellation_wineregion'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0033_merge_20191030_1618.py b/apps/news/migrations/0033_merge_20191030_1618.py new file mode 100644 index 00000000..05811754 --- /dev/null +++ b/apps/news/migrations/0033_merge_20191030_1618.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 16:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0032_auto_20191030_1149'), + ('news', '0029_auto_20191025_1241'), + ] + + operations = [ + ] From aa2759d7e063e924fc1c6800a692be6f6e732acc Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Wed, 30 Oct 2019 19:51:33 +0300 Subject: [PATCH 149/173] fix estab tags --- apps/tag/management/commands/add_tags.py | 79 ++++++++++++------------ 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index 5fc721b7..bf6d4f3c 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -10,53 +10,52 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): - categories = legacy.KeyValueMetadata.objects.all() existing_establishment = Establishment.objects.filter(old_id__isnull=False) - for category in categories: - ESTABLISHMENT = 1 - SHOP = 2 - RESTAURANT = 3 - WINEYARD = 4 + ESTABLISHMENT = 1 + SHOP = 2 + RESTAURANT = 3 + WINEYARD = 4 - MAPPER = { - RESTAURANT: EstablishmentType.RESTAURANT, - WINEYARD: EstablishmentType.PRODUCER, - } + MAPPER = { + RESTAURANT: EstablishmentType.RESTAURANT, + WINEYARD: EstablishmentType.PRODUCER, + } - mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all() - for key, value in MAPPER.items(): - values_meta_id_list = mapper_values_meta.filter( - key_value_metadatum_establishment_id=key - ).values_list('key_value_metadatum_id') + mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all() + for key, value in MAPPER.items(): + values_meta_id_list = mapper_values_meta.filter( + key_value_metadatum_establishment_id=key + ).values_list('key_value_metadatum_id') - est_type, _ = EstablishmentType.objects.get_or_create(index_name=value) + est_type, _ = EstablishmentType.objects.get_or_create(index_name=value) - key_value_metadata = legacy.KeyValueMetadata.objects.filter(id__in=values_meta_id_list) + key_value_metadata = legacy.KeyValueMetadata.objects.filter( + id__in=values_meta_id_list) - # create TagCategory - for key_value in key_value_metadata: - tag_category, created = TagCategory.objects.get_or_create( - index_name=key_value.key_name, + # create TagCategory + for key_value in key_value_metadata: + tag_category, created = TagCategory.objects.get_or_create( + index_name=key_value.key_name, + ) + + if created: + tag_category.label = {'en-GB': key_value.key_name}, + tag_category.value_type = key_value.value_type + tag_category.save() + est_type.tag_categories.add( + tag_category ) - if created: - tag_category.label = {'en-GB': key_value.key_name}, - tag_category.value_type = key_value.value_type - tag_category.save() - est_type.tag_categories.add( - tag_category - ) + # create Tag + for tag in key_value.metadata_set.filter( + establishment__id__in=list(existing_establishment.values_list('old_id', flat=True))): - # create Tag - for tag in key_value.metadata_set.filter( - establishment__id__in=list(existing_establishment.values_list('old_id', flat=True))): - - new_tag, _ = Tag.objects.get_or_create( - label={'en-GB': tag.value}, - value=tag.value, - category=tag_category, - ) - est = existing_establishment.get(old_id=tag.establishment_id) - est.tags.add(new_tag) - est.save() + new_tag, _ = Tag.objects.get_or_create( + label={'en-GB': tag.value}, + value=tag.value, + category=tag_category, + ) + est = existing_establishment.get(old_id=tag.establishment_id) + est.tags.add(new_tag) + est.save() From 776b500d195b0dad9aef888a9ea497d455eeb290 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Wed, 30 Oct 2019 19:59:12 +0300 Subject: [PATCH 150/173] modified NewsImageSerializer, in future needs refactoring --- apps/news/serializers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/news/serializers.py b/apps/news/serializers.py index 1389d20e..ee6d5d33 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -100,7 +100,9 @@ class NewsImageSerializer(serializers.ModelSerializer): orientation_display = serializers.CharField(source='get_orientation_display', read_only=True) - original_url = serializers.URLField(source='image.url') + # todo: refactor + # original_url = serializers.URLField(source='image.url') + original_url = serializers.SerializerMethodField() auto_crop_images = CropImageSerializer(source='image', allow_null=True) class Meta: @@ -116,6 +118,10 @@ class NewsImageSerializer(serializers.ModelSerializer): 'orientation': {'write_only': True} } + def get_original_url(self, obj): + """Get absolute image url.""" + return obj.image.__str__() + class NewsTypeSerializer(serializers.ModelSerializer): """News type serializer.""" From 414ad72ed3c3397522c6d69c286a3d0ce1674bc3 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Wed, 30 Oct 2019 20:01:19 +0300 Subject: [PATCH 151/173] Revert "modified NewsImageSerializer, in future needs refactoring" This reverts commit 776b500d --- apps/news/serializers.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/apps/news/serializers.py b/apps/news/serializers.py index ee6d5d33..1389d20e 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -100,9 +100,7 @@ class NewsImageSerializer(serializers.ModelSerializer): orientation_display = serializers.CharField(source='get_orientation_display', read_only=True) - # todo: refactor - # original_url = serializers.URLField(source='image.url') - original_url = serializers.SerializerMethodField() + original_url = serializers.URLField(source='image.url') auto_crop_images = CropImageSerializer(source='image', allow_null=True) class Meta: @@ -118,10 +116,6 @@ class NewsImageSerializer(serializers.ModelSerializer): 'orientation': {'write_only': True} } - def get_original_url(self, obj): - """Get absolute image url.""" - return obj.image.__str__() - class NewsTypeSerializer(serializers.ModelSerializer): """News type serializer.""" From 7e503f617e6e5e5fb5b2c799ce353d704d328496 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 30 Oct 2019 20:15:19 +0300 Subject: [PATCH 152/173] merge migrations --- .../migrations/0047_merge_20191030_1714.py | 14 ++++ .../migrations/0020_merge_20191030_1714.py | 14 ++++ .../migrations/0034_merge_20191030_1714.py | 14 ++++ apps/transfer/models.py | 82 +++++++++---------- 4 files changed, 83 insertions(+), 41 deletions(-) create mode 100644 apps/establishment/migrations/0047_merge_20191030_1714.py create mode 100644 apps/location/migrations/0020_merge_20191030_1714.py create mode 100644 apps/news/migrations/0034_merge_20191030_1714.py diff --git a/apps/establishment/migrations/0047_merge_20191030_1714.py b/apps/establishment/migrations/0047_merge_20191030_1714.py new file mode 100644 index 00000000..845a472e --- /dev/null +++ b/apps/establishment/migrations/0047_merge_20191030_1714.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 17:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('establishment', '0046_merge_20191030_1618'), + ('establishment', '0046_merge_20191030_0858'), + ] + + operations = [ + ] diff --git a/apps/location/migrations/0020_merge_20191030_1714.py b/apps/location/migrations/0020_merge_20191030_1714.py new file mode 100644 index 00000000..f0b28e5a --- /dev/null +++ b/apps/location/migrations/0020_merge_20191030_1714.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 17:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('location', '0019_merge_20191030_0858'), + ('location', '0019_merge_20191030_1618'), + ] + + operations = [ + ] diff --git a/apps/news/migrations/0034_merge_20191030_1714.py b/apps/news/migrations/0034_merge_20191030_1714.py new file mode 100644 index 00000000..5ac187ab --- /dev/null +++ b/apps/news/migrations/0034_merge_20191030_1714.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-30 17:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0033_merge_20191030_1618'), + ('news', '0032_merge_20191030_0858'), + ] + + operations = [ + ] diff --git a/apps/transfer/models.py b/apps/transfer/models.py index 4033f3b7..ccda8cd6 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -897,44 +897,44 @@ class KeyValueMetadatumKeyValueMetadatumEstablishments(MigrateMixin): db_table = 'key_value_metadatum_key_value_metadatum_establishments' -class Products(models.Model): - establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) - brand = models.CharField(max_length=255, blank=True, null=True) - name = models.CharField(max_length=255, blank=True, null=True) - vintage = models.CharField(max_length=255, blank=True, null=True) - type = models.CharField(max_length=255, blank=True, null=True) - unique_key = models.CharField(max_length=255, blank=True, null=True) - price = models.FloatField(blank=True, null=True) - average_price_in_shops = models.FloatField(blank=True, null=True) - fra_encima_id = models.IntegerField(blank=True, null=True) - wine_sub_region_id = models.IntegerField(blank=True, null=True) - classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) - wine_region = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) - wine_type = models.ForeignKey('WineTypes', models.DO_NOTHING, blank=True, null=True) - wine_color = models.ForeignKey('WineColors', models.DO_NOTHING, blank=True, null=True) - appellation = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) - created_at = models.DateTimeField() - updated_at = models.DateTimeField() - state = models.CharField(max_length=255, blank=True, null=True) - wine_style = models.ForeignKey('WineStyles', models.DO_NOTHING, blank=True, null=True) - village = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) - vineyard = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) - yard_classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) - wine_quality = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) - bottles_produced = models.CharField(max_length=3000, blank=True, null=True) - deu_import_id = models.IntegerField(blank=True, null=True) - - class Meta: - managed = False - db_table = 'products' - - -class WineTypes(models.Model): - name = models.CharField(max_length=255, blank=True, null=True) - fra_encima_id = models.IntegerField(blank=True, null=True) - created_at = models.DateTimeField() - updated_at = models.DateTimeField() - - class Meta: - managed = False - db_table = 'wine_types' +# class Products(models.Model): +# establishment = models.ForeignKey('Establishments', models.DO_NOTHING, blank=True, null=True) +# brand = models.CharField(max_length=255, blank=True, null=True) +# name = models.CharField(max_length=255, blank=True, null=True) +# vintage = models.CharField(max_length=255, blank=True, null=True) +# type = models.CharField(max_length=255, blank=True, null=True) +# unique_key = models.CharField(max_length=255, blank=True, null=True) +# price = models.FloatField(blank=True, null=True) +# average_price_in_shops = models.FloatField(blank=True, null=True) +# fra_encima_id = models.IntegerField(blank=True, null=True) +# wine_sub_region_id = models.IntegerField(blank=True, null=True) +# classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# wine_region = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# wine_type = models.ForeignKey('WineTypes', models.DO_NOTHING, blank=True, null=True) +# wine_color = models.ForeignKey('WineColors', models.DO_NOTHING, blank=True, null=True) +# appellation = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# state = models.CharField(max_length=255, blank=True, null=True) +# wine_style = models.ForeignKey('WineStyles', models.DO_NOTHING, blank=True, null=True) +# village = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# vineyard = models.ForeignKey('WineLocations', models.DO_NOTHING, blank=True, null=True) +# yard_classification = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# wine_quality = models.ForeignKey('WineClassifications', models.DO_NOTHING, blank=True, null=True) +# bottles_produced = models.CharField(max_length=3000, blank=True, null=True) +# deu_import_id = models.IntegerField(blank=True, null=True) +# +# class Meta: +# managed = False +# db_table = 'products' +# +# +# class WineTypes(models.Model): +# name = models.CharField(max_length=255, blank=True, null=True) +# fra_encima_id = models.IntegerField(blank=True, null=True) +# created_at = models.DateTimeField() +# updated_at = models.DateTimeField() +# +# class Meta: +# managed = False +# db_table = 'wine_types' From e75229b4d3e1d613adb946e351afca7b2e5ce22c Mon Sep 17 00:00:00 2001 From: "a.feteleu" Date: Wed, 30 Oct 2019 20:35:15 +0300 Subject: [PATCH 153/173] fix elastic settings --- project/settings/development.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/settings/development.py b/project/settings/development.py index b9f03146..a9cfe5be 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,8 +19,8 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - # 'hosts': 'localhost:9200' - 'hosts': 'elasticsearch:9200' + 'hosts': 'localhost:9200' + # 'hosts': 'elasticsearch:9200' } } From e74c32f0cb8e24b082786c671c0e279df72e12b4 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 30 Oct 2019 23:15:40 +0300 Subject: [PATCH 154/173] fix summary --- apps/transfer/serializers/news.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 34052e0a..021aefe0 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -15,6 +15,7 @@ class NewsSerializer(serializers.Serializer): news_type_id = serializers.IntegerField() news_title = serializers.CharField() title = serializers.CharField() + summary = serializers.CharField(allow_null=True) body = serializers.CharField(allow_null=True) created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') slug = serializers.CharField() From 1bbe481a390c6bca898a6d2fc973f6c615f44994 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Thu, 31 Oct 2019 00:16:33 +0300 Subject: [PATCH 155/173] add settings for phone --- project/settings/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/settings/base.py b/project/settings/base.py index 18f835bd..5b096020 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -485,3 +485,6 @@ STATICFILES_DIRS = ( # MEDIA MEDIA_LOCATION = 'media' + +PHONENUMBER_DB_FORMAT = 'NATIONAL' +PHONENUMBER_DEFAULT_REGION = "FR" From e653e6c7da388e8d29da4695fee6e24dc2e50579 Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Thu, 31 Oct 2019 00:53:01 +0300 Subject: [PATCH 156/173] fix tag command --- apps/tag/management/commands/add_tags.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index bf6d4f3c..e1e97f33 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -39,7 +39,11 @@ class Command(BaseCommand): ) if created: - tag_category.label = {'en-GB': key_value.key_name}, + tag_category.label = { + 'en-GB': key_value.key_name, + 'fr-FR': key_value.key_name, + 'ru-RU': key_value.key_name, + }, tag_category.value_type = key_value.value_type tag_category.save() est_type.tag_categories.add( From a4c9c8f0cc1a4f6a1a6bb7119de13ff981051e08 Mon Sep 17 00:00:00 2001 From: "a.feteleu" Date: Thu, 31 Oct 2019 02:16:29 +0300 Subject: [PATCH 157/173] fix tags command --- apps/tag/management/commands/add_tags.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index e1e97f33..89ee89a1 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -55,7 +55,11 @@ class Command(BaseCommand): establishment__id__in=list(existing_establishment.values_list('old_id', flat=True))): new_tag, _ = Tag.objects.get_or_create( - label={'en-GB': tag.value}, + label={ + 'en-GB': tag.value, + 'fr-FR': tag.value, + 'ru-RU': tag.value, + }, value=tag.value, category=tag_category, ) From b146e6cd2441f57bc8f512f90a607779a7e12796 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 07:00:44 +0300 Subject: [PATCH 158/173] Add establishment address transfer --- apps/establishment/transfer_data.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 4670e6cf..a5a5bc49 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -4,6 +4,8 @@ from django.db.models import Q from transfer.models import Establishments from transfer.serializers.establishment import EstablishmentSerializer +from establishment.models import Establishment +from location.models import Address def transfer_establishment(): @@ -77,6 +79,30 @@ def transfer_establishment(): pprint(f"Establishment serializer errors: {serialized_data.errors}") +def transfer_establishment_addresses(): + old_establishments = Establishments.objects.only("id", "location_id").exclude( + Q(type='Wineyard') | + Q(location__timezone__isnull=True) + ) + + for old_establishment in old_establishments: + try: + establishment = Establishment.objects.get(old_id=old_establishment.id) + except Establishment.DoesNotExist: + continue + + try: + location = Address.objects.get(old_id=old_establishment.location_id) + except Establishment.DoesNotExist: + continue + + establishment.address = location + establishment.save() + + data_types = { - "establishment": [transfer_establishment] + "establishment": [ + transfer_establishment, + transfer_establishment_addresses + ] } From 968ae4246beefdded8b818db6f47d9e02d22591b Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 07:37:09 +0300 Subject: [PATCH 159/173] Fix mark --- apps/review/transfer_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index 80d67f0b..9bc9e5fb 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -28,6 +28,7 @@ def transfer_reviews(): WHERE reviews.reviewer_id > 0 AND reviews.reviewer_id IS NOT NULL AND review_texts.text IS NOT NULL + AND reviews.mark IS NOT NULL AND reviews.reviewer_id IN ( SELECT accounts.id FROM accounts @@ -58,7 +59,10 @@ def transfer_reviews(): if serialized_data.is_valid(): serialized_data.save() for establishment_id, mark in establishments_mark_list.items(): - establishment = Establishment.objects.get(old_id=establishment_id) + try: + establishment = Establishment.objects.get(old_id=establishment_id) + except Establishment.DoesNotExist: + continue establishment.public_mark = mark establishment.save() else: From d654b093ef8874c9f615ba81bb130c67cdf33504 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 07:44:53 +0300 Subject: [PATCH 160/173] Fix --- apps/establishment/transfer_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 632fb1fd..317942ea 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -128,6 +128,6 @@ data_types = { "establishment": [ transfer_establishment, transfer_establishment_addresses - ] + ], "menu": [transfer_menu], } From c08e202f64dfc22152dcb62193d59302dd5e669e Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 07:46:29 +0300 Subject: [PATCH 161/173] Fix db --- apps/establishment/transfer_data.py | 1 + project/settings/base.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 317942ea..000df561 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -129,5 +129,6 @@ data_types = { transfer_establishment, transfer_establishment_addresses ], + "menu": [transfer_menu], } diff --git a/project/settings/base.py b/project/settings/base.py index d2cf7b49..5b096020 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,9 +159,9 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.17.0.1', + # 'HOST': '172.17.0.1', # 'HOST': '172.23.0.1', - # 'HOST': 'mysql_db', + 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From 08ca1dc35641978fa8e5ffabd89a2cf58a8e9668 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 07:47:45 +0300 Subject: [PATCH 162/173] Move addresses to new key --- apps/establishment/transfer_data.py | 2 ++ apps/transfer/management/commands/transfer.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 000df561..6e345e27 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -127,6 +127,8 @@ def transfer_establishment_addresses(): data_types = { "establishment": [ transfer_establishment, + ], + "location_establishment": [ transfer_establishment_addresses ], diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 8af8208d..f3abbfab 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -20,7 +20,8 @@ class Command(BaseCommand): 'commercial', 'overlook', 'tmp', - 'menu' + 'menu', + 'location_establishment' ] def handle(self, *args, **options): From ea5140cc9c23aafc79c9eff8ede7794a5659da78 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Oct 2019 08:11:14 +0300 Subject: [PATCH 163/173] fix summary field in news --- apps/transfer/serializers/news.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 021aefe0..4dc7d913 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -15,7 +15,7 @@ class NewsSerializer(serializers.Serializer): news_type_id = serializers.IntegerField() news_title = serializers.CharField() title = serializers.CharField() - summary = serializers.CharField(allow_null=True) + summary = serializers.CharField(allow_null=True, allow_blank=True) body = serializers.CharField(allow_null=True) created_at = serializers.DateTimeField(format='%m-%d-%Y %H:%M:%S') slug = serializers.CharField() @@ -121,8 +121,8 @@ class NewsSerializer(serializers.Serializer): @staticmethod def get_subtitle(data): - if 'summary' in data: - content = data.pop('summary') - if not content: - content = {data['locale']: data['title']} - return {data['locale']: content} + if data.get('summary'): + content = {data['locale']: data['summary']} + else: + content = {data['locale']: data['title']} + return content From 9cdb3cca2719aa2f27a543fdee9e8feed3551457 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Oct 2019 08:17:08 +0300 Subject: [PATCH 164/173] rm works_now from elastic --- apps/search_indexes/documents/establishment.py | 2 +- apps/search_indexes/serializers.py | 2 +- apps/search_indexes/views.py | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index 7eac2d6c..e4e0c937 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -37,7 +37,7 @@ class EstablishmentDocument(Document): works_noon = fields.ListField(fields.IntegerField( attr='works_noon' )) - works_now = fields.BooleanField(attr='works_now') + # works_now = fields.BooleanField(attr='works_now') tags = fields.ObjectField( properties={ 'id': fields.IntegerField(attr='id'), diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index b9df01b7..a356f99b 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -107,7 +107,7 @@ class EstablishmentDocumentSerializer(DocumentSerializer): 'schedule', 'works_noon', 'works_evening', - 'works_now', + # 'works_now', # 'collections', # 'establishment_type', # 'establishment_subtypes', diff --git a/apps/search_indexes/views.py b/apps/search_indexes/views.py index 2bcd2ea4..1cbf9d05 100644 --- a/apps/search_indexes/views.py +++ b/apps/search_indexes/views.py @@ -152,12 +152,12 @@ class EstablishmentDocumentViewSet(BaseDocumentViewSet): constants.LOOKUP_QUERY_IN, ], }, - 'works_now': { - 'field': 'works_now', - 'lookups': [ - constants.LOOKUP_FILTER_TERM, - ] - }, + # 'works_now': { + # 'field': 'works_now', + # 'lookups': [ + # constants.LOOKUP_FILTER_TERM, + # ] + # }, } geo_spatial_filter_fields = { From f259319ade299874b786c81538325e3c23a9f097 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 08:42:52 +0300 Subject: [PATCH 165/173] Fix exception --- apps/establishment/transfer_data.py | 2 +- apps/transfer/serializers/establishment.py | 2 ++ celerybeat-schedule | Bin 12845 -> 16384 bytes project/settings/base.py | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 6e345e27..2b45c388 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -117,7 +117,7 @@ def transfer_establishment_addresses(): try: location = Address.objects.get(old_id=old_establishment.location_id) - except Establishment.DoesNotExist: + except Address.DoesNotExist: continue establishment.address = location diff --git a/apps/transfer/serializers/establishment.py b/apps/transfer/serializers/establishment.py index c9979430..7c8ce97c 100644 --- a/apps/transfer/serializers/establishment.py +++ b/apps/transfer/serializers/establishment.py @@ -7,6 +7,7 @@ from location.models import Address from timetable.models import Timetable from utils.legacy_parser import parse_legacy_schedule_content from utils.slug_generator import generate_unique_slug +from pytz import timezone as ptz class EstablishmentSerializer(serializers.ModelSerializer): @@ -58,6 +59,7 @@ class EstablishmentSerializer(serializers.ModelSerializer): 'establishment_type_id': self.get_type(data), 'is_publish': data.get('state') == 'published', }) + data['tz'] = ptz(data['tz']) data.pop('location') data.pop('type') data.pop('state') diff --git a/celerybeat-schedule b/celerybeat-schedule index e1a56a152b845b38540fb26180e00295cb3cb2b1..42475a623430ec365e9741260cf2c026a509b8e1 100644 GIT binary patch delta 481 zcmZ3R($K&-L5G!@0SG2DDu_$*f>1MI)8JL!a80)m^eF7TAXpSoSp=;Q)*sGQD$m!19NR5Lxdqv zBDJ_AF)1grI3pJ*rB{+zoL#I}l$unUnUfNaA{U>Tmy(&BT0Gg$P+@Yjfh-4?Jd3#u z=h-)tR~T5-ds{~^0!`B@AAurGu XAg}P(77COU3I-Jl73*y@+{yz0WLSbc delta 243 zcmZo@U|gFrL1$x0FaN{_kIjq%|M?kNCmM=RmQYZT`py6WW`+>@GB1SIHJVr`wK+#& z2`?Xm0Yr!$NU Date: Thu, 31 Oct 2019 08:49:17 +0300 Subject: [PATCH 166/173] Add MultipkleObjects exception --- apps/establishment/transfer_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/establishment/transfer_data.py b/apps/establishment/transfer_data.py index 2b45c388..8366ed94 100644 --- a/apps/establishment/transfer_data.py +++ b/apps/establishment/transfer_data.py @@ -117,6 +117,8 @@ def transfer_establishment_addresses(): try: location = Address.objects.get(old_id=old_establishment.location_id) + except Address.MultipleObjectsReturned: + location = Address.objects.filter(old_id=old_establishment.location_id).first() except Address.DoesNotExist: continue From 79fda63d7c214e78ea5ce5876885108c97a2c2a5 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 08:51:13 +0300 Subject: [PATCH 167/173] Fix db --- project/settings/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/settings/base.py b/project/settings/base.py index d2cf7b49..5b096020 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,9 +159,9 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.17.0.1', + # 'HOST': '172.17.0.1', # 'HOST': '172.23.0.1', - # 'HOST': 'mysql_db', + 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From fea93f50f7886ad54b14e2f95fb643884e479cba Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Oct 2019 09:40:50 +0300 Subject: [PATCH 168/173] fix image_url for establishment --- .../management/commands/add_establishment_image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/establishment/management/commands/add_establishment_image.py b/apps/establishment/management/commands/add_establishment_image.py index 7806bfe4..3e88a398 100644 --- a/apps/establishment/management/commands/add_establishment_image.py +++ b/apps/establishment/management/commands/add_establishment_image.py @@ -24,7 +24,9 @@ class Command(BaseCommand): for obj in queryset: establishment = Establishment.objects.filter(old_id=obj['establishment_id']).first() if establishment: - establishment.preview_image_url = url + obj['attachment_suffix_url'] + img = url + obj['attachment_suffix_url'] + establishment.preview_image_url = img + establishment.image_url = img establishment.save() count += 1 self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) From c8554772f1d586d338f914f71ca242364c771531 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 10:04:49 +0300 Subject: [PATCH 169/173] Fix review published_at Add old_id Add description to establishment --- apps/review/migrations/0005_review_old_id.py | 18 ++++++++++ apps/review/models.py | 3 ++ apps/review/transfer_data.py | 36 ++++++++++++++++---- apps/transfer/serializers/reviews.py | 28 +++++++++++++-- 4 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 apps/review/migrations/0005_review_old_id.py diff --git a/apps/review/migrations/0005_review_old_id.py b/apps/review/migrations/0005_review_old_id.py new file mode 100644 index 00000000..c0e92f04 --- /dev/null +++ b/apps/review/migrations/0005_review_old_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-31 06:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('review', '0004_review_country'), + ] + + operations = [ + migrations.AddField( + model_name='review', + name='old_id', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='old id'), + ), + ] diff --git a/apps/review/models.py b/apps/review/models.py index 4c7f3385..194c6d58 100644 --- a/apps/review/models.py +++ b/apps/review/models.py @@ -68,6 +68,9 @@ class Review(BaseAttributes, TranslatedFieldsMixin): country = models.ForeignKey('location.Country', on_delete=models.CASCADE, related_name='country', verbose_name=_('Country'), null=True) + + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) + objects = ReviewQuerySet.as_manager() class Meta: diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index 9bc9e5fb..2859c446 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -1,6 +1,7 @@ from transfer.models import Reviews, ReviewTexts from transfer.serializers.reviews import LanguageSerializer, ReviewSerializer, Establishment from pprint import pprint +import json def transfer_languages(): @@ -20,14 +21,16 @@ def transfer_languages(): def transfer_reviews(): queryset = Reviews.objects.raw("""SELECT reviews.id, reviews.vintage, reviews.establishment_id, - reviews.reviewer_id, review_texts.text AS text, reviews.mark, - review_texts.created_at AS published, review_texts.locale AS locale + reviews.reviewer_id, review_texts.text AS text, reviews.mark, reviews.published_at, + review_texts.created_at AS published, review_texts.locale AS locale, + reviews.aasm_state FROM reviews LEFT OUTER JOIN review_texts ON (reviews.id = review_texts.review_id) WHERE reviews.reviewer_id > 0 AND reviews.reviewer_id IS NOT NULL AND review_texts.text IS NOT NULL + AND review_texts.locale IS NOT NULL AND reviews.mark IS NOT NULL AND reviews.reviewer_id IN ( SELECT accounts.id @@ -40,8 +43,19 @@ def transfer_reviews(): "d.sadykova@id-east.ru", "d.sadykova@octopod.ru", "n.yurchenko@id-east.ru" - ) - ) ORDER BY review_texts.created_at DESC + )) + AND reviews.establishment_id IN ( + SELECT establishments.id + FROM establishments + INNER JOIN locations + ON (establishments.location_id = locations.id) + INNER JOIN cities + ON (locations.city_id = cities.id) + WHERE UPPER(cities.name) LIKE UPPER("%%paris%%") + AND NOT establishments.type = "Wineyard" + AND establishments.type IS NOT NULL AND locations.timezone IS NOT NULL + ) + ORDER BY review_texts.created_at DESC """) queryset_result = [] @@ -50,7 +64,13 @@ def transfer_reviews(): for query in queryset: query = vars(query) if query['establishment_id'] not in establishments_mark_list.keys(): - establishments_mark_list[query['establishment_id']] = int(query['mark']) + if "aasm_state" in query and query['aasm_state'] is not None and query['aasm_state'] == "published": + establishments_mark_list[query['establishment_id']] = [ + int(query['mark']), + json.dumps({query['locale']: query['text']}) + ] + else: + establishments_mark_list[query['establishment_id']] = int(query['mark']) del(query['mark']) queryset_result.append(query) @@ -58,11 +78,15 @@ def transfer_reviews(): if serialized_data.is_valid(): serialized_data.save() + for establishment_id, mark in establishments_mark_list.items(): try: establishment = Establishment.objects.get(old_id=establishment_id) except Establishment.DoesNotExist: continue + if isinstance(mark, list): + mark, review_text = mark + establishment.public_mark = mark establishment.save() else: @@ -71,7 +95,7 @@ def transfer_reviews(): data_types = { "overlook": [ - transfer_languages, + # transfer_languages, transfer_reviews ] } diff --git a/apps/transfer/serializers/reviews.py b/apps/transfer/serializers/reviews.py index e6bd11ac..024ea033 100644 --- a/apps/transfer/serializers/reviews.py +++ b/apps/transfer/serializers/reviews.py @@ -10,23 +10,35 @@ class ReviewSerializer(serializers.ModelSerializer): reviewer_id = serializers.IntegerField() vintage = serializers.IntegerField() published = serializers.DateTimeField() + published_at = serializers.DateTimeField(allow_null=True) establishment_id = serializers.IntegerField() text = serializers.CharField() locale = serializers.CharField() + aasm_state = serializers.CharField(allow_null=True) class Meta: model = Review fields = ( - "id", "reviewer_id", "published", "vintage", "establishment_id", "text", "locale" + "id", "reviewer_id", "published", "vintage", + "establishment_id", "text", "locale", + "published_at", "aasm_state" ) def validate(self, data): data = self.set_old_id(data) + data = self.set_published_at(data) data = self.set_reviewer(data) data = self.set_establishment(data) data = self.set_language(data) + data = self.set_published(data) return data + def create(self, validated_data): + try: + return Review.objects.create(**validated_data) + except Exception as e: + raise ValueError(f"Error creating review with {validated_data}: {e}") + def set_old_id(self, data): data['old_id'] = data.pop("id") return data @@ -41,9 +53,9 @@ class ReviewSerializer(serializers.ModelSerializer): def set_establishment(self, data): try: - data['establishment'] = Establishment.objects.get(old_id=data.pop('old_id')) + data['content_object'] = Establishment.objects.get(old_id=data.pop('establishment_id')) except Establishment.DoesNotExist as e: - raise ValueError(f"Cannot find establishment with {data}: {e}") + raise ValueError(f"Cannot find review establishment with {data}: {e}") return data def set_language(self, data): @@ -56,6 +68,16 @@ class ReviewSerializer(serializers.ModelSerializer): return data + def set_published(self, data): + data['published_at'] = data.pop("published") + return data + + def set_published_at(self, data): + if "aasm_state" in data and data['aasm_state'] is not None and data['aasm_state'] == "published": + data['status'] = Review.READY + del(data['aasm_state']) + return data + class LanguageSerializer(serializers.ModelSerializer): From 20785770e1fdccad5f6a34a53f80eafc210c0b06 Mon Sep 17 00:00:00 2001 From: littlewolf Date: Thu, 31 Oct 2019 10:05:39 +0300 Subject: [PATCH 170/173] Add transfer_language --- apps/review/transfer_data.py | 2 +- project/settings/base.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/review/transfer_data.py b/apps/review/transfer_data.py index 2859c446..b3a0e833 100644 --- a/apps/review/transfer_data.py +++ b/apps/review/transfer_data.py @@ -95,7 +95,7 @@ def transfer_reviews(): data_types = { "overlook": [ - # transfer_languages, + transfer_languages, transfer_reviews ] } diff --git a/project/settings/base.py b/project/settings/base.py index 5b096020..d2cf7b49 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,9 +159,9 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - # 'HOST': '172.17.0.1', + 'HOST': '172.17.0.1', # 'HOST': '172.23.0.1', - 'HOST': 'mysql_db', + # 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', From 6fe8252025fc1ad6de91e68b8f04359ccadf6905 Mon Sep 17 00:00:00 2001 From: Anatoly Date: Thu, 31 Oct 2019 11:04:53 +0300 Subject: [PATCH 171/173] need refactoring, put temporary solution TagBaseSerializer TagCategoryBaseSerializer --- apps/tag/serializers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/tag/serializers.py b/apps/tag/serializers.py index 790c8926..c09f42f1 100644 --- a/apps/tag/serializers.py +++ b/apps/tag/serializers.py @@ -12,7 +12,9 @@ from utils.serializers import TranslatedField class TagBaseSerializer(serializers.ModelSerializer): """Serializer for model Tag.""" - label_translated = TranslatedField() + # todo: refactor this + # label_translated = TranslatedField() + label_translated = serializers.CharField(source='value') class Meta: """Meta class.""" @@ -39,7 +41,9 @@ class TagBackOfficeSerializer(TagBaseSerializer): class TagCategoryBaseSerializer(serializers.ModelSerializer): """Serializer for model TagCategory.""" - label_translated = TranslatedField() + # todo: refactor this + # label_translated = TranslatedField() + label_translated = serializers.CharField(source='index_name') tags = TagBaseSerializer(many=True, read_only=True) class Meta: From 7be3c2b93faf6ea2d658612c0bba674900676296 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Oct 2019 12:02:52 +0300 Subject: [PATCH 172/173] add_establishment_description command --- .../commands/add_establishment_description.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 apps/establishment/management/commands/add_establishment_description.py diff --git a/apps/establishment/management/commands/add_establishment_description.py b/apps/establishment/management/commands/add_establishment_description.py new file mode 100644 index 00000000..1748d81d --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_description.py @@ -0,0 +1,31 @@ +from django.core.management.base import BaseCommand + +from establishment.models import Establishment +from transfer.models import Descriptions + + +class Command(BaseCommand): + help = 'Add description values from old db to new db' + + def handle(self, *args, **kwargs): + count = 0 + + queryset = Descriptions.objects.all() + for obj in queryset: + try: + establishment = Establishment.objects.get(old_id=obj.establishment.id) + except Establishment.DoesNotExist: + continue + except Establishment.MultipleObjectsReturned: + establishment = Establishment.objects.filter(old_id=obj.establishment.id).first() + else: + description = establishment.description + description.update({ + obj.locale: obj.text + }) + establishment.description = description + establishment.save() + count += 1 + break + + self.stdout.write(self.style.WARNING(f'Updated {count} objects.')) From 0d399cc44edae29cdeec6d53daf5b4e8cf0536a8 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Oct 2019 12:29:40 +0300 Subject: [PATCH 173/173] add_establishment_mark command --- .../commands/add_establishment_mark.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 apps/establishment/management/commands/add_establishment_mark.py diff --git a/apps/establishment/management/commands/add_establishment_mark.py b/apps/establishment/management/commands/add_establishment_mark.py new file mode 100644 index 00000000..1588268f --- /dev/null +++ b/apps/establishment/management/commands/add_establishment_mark.py @@ -0,0 +1,46 @@ +from pprint import pprint + +from django.core.management.base import BaseCommand +from django.db.models import Q + +from establishment.models import Establishment +from transfer.models import Reviews + + +class Command(BaseCommand): + help = 'Add description values from old db to new db' + + def handle(self, *args, **kwargs): + count = 0 + valid_data = {} + + queryset = Reviews.objects.exclude( + Q(establishment_id__isnull=True) | + Q(mark__isnull=True) + ).filter(aasm_state='published').values_list('establishment_id', 'mark', 'updated_at') + + print(queryset.count()) + + for es_id, new_mark, new_date in queryset: + try: + mark, date = valid_data[es_id] + except KeyError: + valid_data[es_id] = (new_mark, new_date) + else: + if new_date > date: + valid_data[es_id] = (new_mark, new_date) + + for key, value in valid_data.items(): + try: + establishment = Establishment.objects.get(old_id=key) + except Establishment.DoesNotExist: + continue + except Establishment.MultipleObjectsReturned: + establishment = Establishment.objects.filter(old_id=key).first() + else: + establishment.public_mark = int(value[0]) + establishment.save() + count += 1 + break + + self.stdout.write(self.style.WARNING(f'Updated {count} objects.'))