diff --git a/apps/advertisement/migrations/0004_auto_20191025_0903.py b/apps/advertisement/migrations/0004_auto_20191025_0903.py new file mode 100644 index 00000000..f633d67c --- /dev/null +++ b/apps/advertisement/migrations/0004_auto_20191025_0903.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('advertisement', '0003_auto_20190919_1344'), + ] + + operations = [ + migrations.AlterField( + model_name='advertisement', + name='block_level', + field=models.CharField(blank=True, max_length=10, null=True, verbose_name='Block level'), + ), + ] diff --git a/apps/advertisement/models.py b/apps/advertisement/models.py index fdc25988..62b41428 100644 --- a/apps/advertisement/models.py +++ b/apps/advertisement/models.py @@ -15,7 +15,7 @@ class Advertisement(ImageMixin, ProjectBaseMixin, PlatformMixin): url = models.URLField(verbose_name=_('Ad URL')) width = models.PositiveIntegerField(verbose_name=_('Block width')) height = models.PositiveIntegerField(verbose_name=_('Block height')) - block_level = models.CharField(verbose_name=_('Block level'), max_length=10) + block_level = models.CharField(verbose_name=_('Block level'), max_length=10, blank=True, null=True) target_languages = models.ManyToManyField(Language) class Meta: diff --git a/apps/advertisement/transfer_data.py b/apps/advertisement/transfer_data.py new file mode 100644 index 00000000..3af16286 --- /dev/null +++ b/apps/advertisement/transfer_data.py @@ -0,0 +1,20 @@ +from pprint import pprint +from django.db.models import Value, IntegerField, F +from transfer.models import Ads +from transfer.serializers.advertisement import AdvertisementSerializer + + +def transfer_advertisement(): + queryset = Ads.objects.filter(href__isnull=False) + + serialized_data = AdvertisementSerializer(data=list(queryset.values()), many=True) + + if serialized_data.is_valid(): + serialized_data.save() + else: + pprint(f"News serializer errors: {serialized_data.errors}") + + +data_types = { + "commercial": [transfer_advertisement] +} \ No newline at end of file diff --git a/apps/gallery/migrations/0004_merge_20191025_0906.py b/apps/gallery/migrations/0004_merge_20191025_0906.py new file mode 100644 index 00000000..4372fa95 --- /dev/null +++ b/apps/gallery/migrations/0004_merge_20191025_0906.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('gallery', '0002_auto_20191023_1207'), + ('gallery', '0003_auto_20191003_1228'), + ] + + operations = [ + ] diff --git a/apps/gallery/models.py b/apps/gallery/models.py index bf5bc51c..baed48fc 100644 --- a/apps/gallery/models.py +++ b/apps/gallery/models.py @@ -27,6 +27,7 @@ class Image(ProjectBaseMixin, SORLImageMixin, PlatformMixin): blank=True, null=True, default=None, verbose_name=_('image orientation')) title = models.CharField(_('title'), max_length=255, default='') + objects = ImageQuerySet.as_manager() class Meta: diff --git a/apps/news/migrations/0029_merge_20191025_0906.py b/apps/news/migrations/0029_merge_20191025_0906.py new file mode 100644 index 00000000..e5967a8d --- /dev/null +++ b/apps/news/migrations/0029_merge_20191025_0906.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.4 on 2019-10-25 09:06 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0028_auto_20191024_1649'), + ('news', '0021_auto_20191021_1120'), + ] + + operations = [ + ] diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index deee2fe5..d46c7613 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,16 +1,18 @@ -from transfer.serializers.news import NewsSerializer -from transfer.models import PageTexts -from news.models import NewsType -from django.db.models import Value, IntegerField, F from pprint import pprint +from django.db.models import Value, IntegerField, F + +from news.models import NewsType +from transfer.models import PageTexts +from transfer.serializers.news import NewsSerializer + def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name="News") - queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), - playlist=Value(1, output_field=IntegerField())) - queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) + queryset = PageTexts.objects.filter(page__type="News").annotate( + news_type=Value(news_type.id, output_field=IntegerField()), + ) queryset = queryset.annotate(template=F('page__template')) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) diff --git a/apps/transfer/management/commands/transfer.py b/apps/transfer/management/commands/transfer.py index 96b4155a..79d291d1 100644 --- a/apps/transfer/management/commands/transfer.py +++ b/apps/transfer/management/commands/transfer.py @@ -16,7 +16,7 @@ class Command(BaseCommand): 'recipe', 'partner', 'gallery', - + 'commercial', 'tmp' ] diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f469f698..f6dd2dbc 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -694,6 +694,7 @@ class ReviewTexts(MigrateMixin): managed = False db_table = 'review_texts' + class Comments(MigrateMixin): using = 'legacy' @@ -787,3 +788,23 @@ class PageMetadata(MigrateMixin): class Meta: managed = False db_table = 'page_metadata' + + +class Ads(MigrateMixin): + using = 'legacy' + + site_id = models.IntegerField(blank=True, null=True) + href = models.CharField(max_length=255, blank=True, null=True) + start_at = models.DateTimeField(blank=True, null=True) + expire_at = models.DateTimeField(blank=True, null=True) + attachment_file_name = models.CharField(max_length=255, blank=True, null=True) + attachment_content_type = models.CharField(max_length=255, blank=True, null=True) + attachment_file_size = models.IntegerField(blank=True, null=True) + attachment_updated_at = models.DateTimeField(blank=True, null=True) + geometries = models.CharField(max_length=1024, blank=True, null=True) + created_at = models.DateTimeField() + updated_at = models.DateTimeField() + + class Meta: + managed = False + db_table = 'ads' \ No newline at end of file diff --git a/apps/transfer/serializers/advertisement.py b/apps/transfer/serializers/advertisement.py new file mode 100644 index 00000000..2e3a488f --- /dev/null +++ b/apps/transfer/serializers/advertisement.py @@ -0,0 +1,39 @@ +from rest_framework import serializers +from advertisement.models import Advertisement +import yaml + + +class AdvertisementSerializer(serializers.ModelSerializer): + href = serializers.CharField() + geometries = serializers.CharField(max_length=1024) + + class Meta: + model = Advertisement + fields = ( + "href", + "geometries" + ) + + def validate(self, data): + data["url"] = data["href"] + data["width"] = self.get_width(data["geometries"]) + data["height"] = self.get_height(data["geometries"]) + data.pop("href") + data.pop("geometries") + return data + + def create(self, validated_data): + return Advertisement.objects.create(**validated_data) + + def get_width(self, data): + data = self.parse_geometries(data) + return int(float(data["width"])) + + def get_height(self, data): + data = self.parse_geometries(data) + return int(float(data["height"])) + + def parse_geometries(self, geo_str): + clear_str = "!ruby/object:Paperclip::Geometry" + content_dict = yaml.safe_load(geo_str.replace(clear_str, '')) + return content_dict[':original'] \ No newline at end of file diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 2d85bea1..f47b4940 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,123 +1,70 @@ from rest_framework import serializers + from news.models import News -from uuid import uuid4 -import datetime -from django.conf import settings +from utils.legacy_parser import parse_legacy_content class NewsSerializer(serializers.ModelSerializer): - # image_url = serializers.CharField() - # preview_image_url = serializers.CharField() - attachment_file_name = serializers.CharField(allow_null=True) - # body = serializers.CharField(allow_null=True, source="description") + locale = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() template = serializers.CharField() state = serializers.CharField() - created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') + created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') class Meta: model = News fields = ( - "created_at", - "state", - "template", - # "image_url", - # "preview_image_url", - "attachment_file_name", - "title", - "body", - "slug", - "news_type", - "playlist" + 'created_at', + 'state', + 'template', + 'title', + 'body', + 'slug', + 'news_type', + 'locale', ) def validate(self, data): - data["state"] = self.get_state(data) - data["template"] = self.get_template(data) - data["title"] = self.get_title(data) - data["description"] = self.get_description(data) - data["image_url"] = self.get_image_url(data) - data["preview_image_url"] = self.get_preview_image_url(data) - data.pop("attachment_file_name") - data.pop("body") + data.update({ + 'state': self.get_state(data), + 'template': self.get_template(data), + 'title': self.get_title(data), + 'description': self.get_description(data), + }) + data.pop('body') + data.pop('locale') return data def create(self, validated_data): - # validated_data['state'] = self.get_state(validated_data) return News.objects.create(**validated_data) - def get_template(self, obj): - if obj["template"] == "main": - return News.MAIN - elif obj["template"] == "main.pdf.erb": - return News.MAIN_PDF_ERB - else: - return News.NEWSPAPER + @staticmethod + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + } + return templates.get(data['template'], News.MAIN) - # if obj.template == "main": - # return News.MAIN - # elif obj.template == "main.pdf.erb": - # return News.MAIN_PDF_ERB - # else: - # return News.NEWSPAPER + @staticmethod + def get_state(data): + states = { + 'new': News.WAITING, + 'published': News.PUBLISHED, + 'hidden': News.HIDDEN, + 'published_exclusive': News.PUBLISHED_EXCLUSIVE, + 'scheduled_exclusively': News.WAITING, + } + return states.get(data['state'], News.WAITING) - def get_state(self, obj): - if obj["state"] == "published": - return News.PUBLISHED - elif obj["state"] == "hidden": - return News.HIDDEN - elif obj["state"] == "published_exclusive": - return News.PUBLISHED_EXCLUSIVE - else: - return News.WAITING + @staticmethod + def get_title(data): + return {data['locale']: data['title']} - # if obj.state == "published": - # return News.PUBLISHED - # elif obj.state == "hidden": - # return News.HIDDEN - # elif obj.state == "published_exclusive": - # return News.PUBLISHED_EXCLUSIVE - # else: - # return News.WAITING - - # django.db.utils.IntegrityError: "start" NOT NULL - # def get_start(self, obj): - # # return obj.created_at - # return datetime.datetime.now().isoformat() - - def get_title(self, obj): - tit = obj.get("title") - return {"en-GB": tit} - # return f'{"en-GB": {obj.title}}' - - def get_description(self, obj): - # return {"en-GB": {obj["body"]}} - desc = obj.get("body") - return {"en-GB": desc} - # if not hasattr(obj, "body"): - # return "" - # else: - # return obj.body - - def get_image_url(self, obj): - if not obj.get("attachment_file_name"): - return "" - - if len(obj.get("attachment_file_name"))>200: - return obj.get("attachment_file_name")[:200] - return obj.get("attachment_file_name") - # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: - # return "" - # return obj.attachment_file_name - - def get_preview_image_url(self, obj): - if not obj.get("attachment_file_name"): - return "" - - if len(obj.get("attachment_file_name"))>200: - return obj.get("attachment_file_name")[:200] - return obj.get("attachment_file_name") - # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: - # return "" - # return obj.attachment_file_name + @staticmethod + def get_description(data): + content = None + if data['body']: + content = parse_legacy_content(data['body']) + return {data['locale']: content} diff --git a/apps/transfer/serializers/recipe.py b/apps/transfer/serializers/recipe.py index 890d2b42..3948bbdd 100644 --- a/apps/transfer/serializers/recipe.py +++ b/apps/transfer/serializers/recipe.py @@ -1,8 +1,10 @@ from rest_framework import serializers from recipe.models import Recipe +from utils.legacy_parser import parse_legacy_content class RecipeSerializer(serializers.ModelSerializer): + locale = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() state = serializers.CharField() @@ -14,7 +16,8 @@ class RecipeSerializer(serializers.ModelSerializer): "body", "title", "state", - "created_at" + "created_at", + 'locale', ) def validate(self, data): @@ -22,6 +25,7 @@ class RecipeSerializer(serializers.ModelSerializer): data["title"] = self.get_title(data) data["description"] = self.get_description(data) data.pop("body") + data.pop("locale") return data def create(self, validated_data): @@ -38,9 +42,14 @@ class RecipeSerializer(serializers.ModelSerializer): return Recipe.WAITING def get_title(self, obj): - tit = obj.get("title") - return {"en-GB": tit} + # tit = obj.get("title") + # return {"en-GB": tit} + return {obj['locale']: obj['title']} def get_description(self, obj): - desc = obj.get("body") - return {"en-GB": desc} + # desc = obj.get("body") + # return {"en-GB": desc} + content = None + if obj['body']: + content = parse_legacy_content(obj['body']) + return {obj['locale']: content} diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py new file mode 100644 index 00000000..b69e2604 --- /dev/null +++ b/apps/utils/legacy_parser.py @@ -0,0 +1,12 @@ +import yaml + + +def parse_legacy_content(legacy_content): + clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' + content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) + result = '' + try: + result = content_dict['news_content']['value'] + except KeyError: + pass + return result diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index ac4619dd..cabb3808 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -14,6 +14,8 @@ services: volumes: - .:/code - gm-mysql_db:/var/lib/mysql + networks: + - mysql_network # PostgreSQL database db: @@ -29,6 +31,8 @@ services: - "5436:5432" volumes: - gm-db:/var/lib/postgresql/data/ + networks: + - database_network elasticsearch: image: elasticsearch:7.3.1 @@ -42,6 +46,8 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - discovery.type=single-node - xpack.security.enabled=false + networks: + - elasticsearch_network # Redis redis: @@ -113,6 +119,10 @@ services: - gm-media:/media-data ports: - "8000:8000" + networks: + - database_network + - mysql_network + - elasticsearch_network volumes: gm-mysql_db: @@ -124,4 +134,12 @@ volumes: gm-media: name: gm-media - gm-esdata: \ No newline at end of file + gm-esdata: + +networks: + database_network: + driver: bridge + mysql_network: + driver: bridge + elasticsearch_network: + driver: bridge diff --git a/project/settings/base.py b/project/settings/base.py index 99a76f79..676a67d1 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -159,8 +159,9 @@ DATABASES = { }, 'legacy': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '172.20.0.1', - # 'HOST': '127.0.0.1', + 'HOST': '172.17.0.1', + # 'HOST': '172.23.0.1', + # 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', diff --git a/project/settings/development.py b/project/settings/development.py index 9f950abf..73e0095d 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -20,6 +20,7 @@ DOMAIN_URI = 'gm.id-east.ru' ELASTICSEARCH_DSL = { 'default': { 'hosts': 'localhost:9200' + # 'hosts': 'elasticsearch:9200' } } diff --git a/requirements/base.txt b/requirements/base.txt index b9fbcdfc..3a45dedc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,7 +9,7 @@ fcm-django django-easy-select2 bootstrap-admin drf-yasg==1.16.0 -timezonefinder==4.1.0 +timezonefinder PySocks!=1.5.7,>=1.5.6; djangorestframework==3.9.4 @@ -41,9 +41,11 @@ django-storages==1.7.2 sorl-thumbnail==12.5.0 + +mysqlclient==1.4.4 +PyYAML==5.1.2 + # temp solution redis==3.2.0 amqp>=2.4.0 celery==4.3.0rc2 - -mysqlclient==1.4.4 \ No newline at end of file