diff --git a/apps/news/transfer_data.py b/apps/news/transfer_data.py index deee2fe5..9d13e40e 100644 --- a/apps/news/transfer_data.py +++ b/apps/news/transfer_data.py @@ -1,23 +1,23 @@ -from transfer.serializers.news import NewsSerializer -from transfer.models import PageTexts -from news.models import NewsType from django.db.models import Value, IntegerField, F -from pprint import pprint + +from news.models import NewsType +from transfer.models import PageTexts +from transfer.serializers.news import NewsSerializer def transfer_news(): news_type, _ = NewsType.objects.get_or_create(name="News") - queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()), - playlist=Value(1, output_field=IntegerField())) - queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name')) + queryset = PageTexts.objects.filter(page__type="News").annotate( + news_type=Value(news_type.id, output_field=IntegerField()), + ) queryset = queryset.annotate(template=F('page__template')) serialized_data = NewsSerializer(data=list(queryset.values()), many=True) if serialized_data.is_valid(): serialized_data.save() else: - pprint(f"News serializer errors: {serialized_data.errors}") + print(f"News serializer errors: {serialized_data.errors}") data_types = { diff --git a/apps/transfer/models.py b/apps/transfer/models.py index f469f698..f2a634bf 100644 --- a/apps/transfer/models.py +++ b/apps/transfer/models.py @@ -694,6 +694,7 @@ class ReviewTexts(MigrateMixin): managed = False db_table = 'review_texts' + class Comments(MigrateMixin): using = 'legacy' diff --git a/apps/transfer/serializers/news.py b/apps/transfer/serializers/news.py index 2d85bea1..6cddb8dc 100644 --- a/apps/transfer/serializers/news.py +++ b/apps/transfer/serializers/news.py @@ -1,123 +1,68 @@ from rest_framework import serializers + from news.models import News -from uuid import uuid4 -import datetime -from django.conf import settings +from utils.legacy_parser import parse_legacy_content class NewsSerializer(serializers.ModelSerializer): - # image_url = serializers.CharField() - # preview_image_url = serializers.CharField() - attachment_file_name = serializers.CharField(allow_null=True) - # body = serializers.CharField(allow_null=True, source="description") + locale = serializers.CharField() body = serializers.CharField(allow_null=True) title = serializers.CharField() template = serializers.CharField() state = serializers.CharField() - created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S') + created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S') class Meta: model = News fields = ( - "created_at", - "state", - "template", - # "image_url", - # "preview_image_url", - "attachment_file_name", - "title", - "body", - "slug", - "news_type", - "playlist" + 'created_at', + 'state', + 'template', + 'title', + 'body', + 'slug', + 'news_type', + 'locale', ) def validate(self, data): - data["state"] = self.get_state(data) - data["template"] = self.get_template(data) - data["title"] = self.get_title(data) - data["description"] = self.get_description(data) - data["image_url"] = self.get_image_url(data) - data["preview_image_url"] = self.get_preview_image_url(data) - data.pop("attachment_file_name") - data.pop("body") + data.update({ + 'state': self.get_state(data), + 'template': self.get_template(data), + 'title': self.get_title(data), + 'description': self.get_description(data), + }) + data.pop('body') + data.pop('locale') return data def create(self, validated_data): - # validated_data['state'] = self.get_state(validated_data) return News.objects.create(**validated_data) - def get_template(self, obj): - if obj["template"] == "main": - return News.MAIN - elif obj["template"] == "main.pdf.erb": - return News.MAIN_PDF_ERB - else: - return News.NEWSPAPER + @staticmethod + def get_template(data): + templates = { + 'main': News.MAIN, + 'main.pdf.erb': News.MAIN_PDF_ERB, + } + return templates.get(data['template'], News.MAIN) - # if obj.template == "main": - # return News.MAIN - # elif obj.template == "main.pdf.erb": - # return News.MAIN_PDF_ERB - # else: - # return News.NEWSPAPER + @staticmethod + def get_state(data): + states = { + 'new': News.PUBLISHED, + 'published': News.PUBLISHED, + 'hidden': News.HIDDEN, + } + return states.get(data['state'], News.WAITING) - def get_state(self, obj): - if obj["state"] == "published": - return News.PUBLISHED - elif obj["state"] == "hidden": - return News.HIDDEN - elif obj["state"] == "published_exclusive": - return News.PUBLISHED_EXCLUSIVE - else: - return News.WAITING + @staticmethod + def get_title(data): + return {data['locale']: data['title']} - # if obj.state == "published": - # return News.PUBLISHED - # elif obj.state == "hidden": - # return News.HIDDEN - # elif obj.state == "published_exclusive": - # return News.PUBLISHED_EXCLUSIVE - # else: - # return News.WAITING - - # django.db.utils.IntegrityError: "start" NOT NULL - # def get_start(self, obj): - # # return obj.created_at - # return datetime.datetime.now().isoformat() - - def get_title(self, obj): - tit = obj.get("title") - return {"en-GB": tit} - # return f'{"en-GB": {obj.title}}' - - def get_description(self, obj): - # return {"en-GB": {obj["body"]}} - desc = obj.get("body") - return {"en-GB": desc} - # if not hasattr(obj, "body"): - # return "" - # else: - # return obj.body - - def get_image_url(self, obj): - if not obj.get("attachment_file_name"): - return "" - - if len(obj.get("attachment_file_name"))>200: - return obj.get("attachment_file_name")[:200] - return obj.get("attachment_file_name") - # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: - # return "" - # return obj.attachment_file_name - - def get_preview_image_url(self, obj): - if not obj.get("attachment_file_name"): - return "" - - if len(obj.get("attachment_file_name"))>200: - return obj.get("attachment_file_name")[:200] - return obj.get("attachment_file_name") - # if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None: - # return "" - # return obj.attachment_file_name + @staticmethod + def get_description(data): + content = None + if data['body']: + content = parse_legacy_content(data['body']) + return {data['locale']: content} diff --git a/apps/utils/legacy_parser.py b/apps/utils/legacy_parser.py new file mode 100644 index 00000000..a0e526f2 --- /dev/null +++ b/apps/utils/legacy_parser.py @@ -0,0 +1,12 @@ +import yaml + + +def parse_legacy_content(legacy_content): + clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess' + content_dict = yaml.safe_load(legacy_content.replace(clear_str, '')) + result = '' + try: + result = content_dict['news_content']['value'] + except IndexError: + pass + return result diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml index ac4619dd..cabb3808 100644 --- a/docker-compose.mysql.yml +++ b/docker-compose.mysql.yml @@ -14,6 +14,8 @@ services: volumes: - .:/code - gm-mysql_db:/var/lib/mysql + networks: + - mysql_network # PostgreSQL database db: @@ -29,6 +31,8 @@ services: - "5436:5432" volumes: - gm-db:/var/lib/postgresql/data/ + networks: + - database_network elasticsearch: image: elasticsearch:7.3.1 @@ -42,6 +46,8 @@ services: - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - discovery.type=single-node - xpack.security.enabled=false + networks: + - elasticsearch_network # Redis redis: @@ -113,6 +119,10 @@ services: - gm-media:/media-data ports: - "8000:8000" + networks: + - database_network + - mysql_network + - elasticsearch_network volumes: gm-mysql_db: @@ -124,4 +134,12 @@ volumes: gm-media: name: gm-media - gm-esdata: \ No newline at end of file + gm-esdata: + +networks: + database_network: + driver: bridge + mysql_network: + driver: bridge + elasticsearch_network: + driver: bridge diff --git a/project/settings/base.py b/project/settings/base.py index f5cce87d..f2001779 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -160,7 +160,7 @@ DATABASES = { 'legacy': { 'ENGINE': 'django.db.backends.mysql', # 'HOST': '172.23.0.1', - 'HOST': '127.0.0.1', + 'HOST': 'mysql_db', 'PORT': 3306, 'NAME': 'dev', 'USER': 'dev', diff --git a/project/settings/development.py b/project/settings/development.py index 59691818..3e9968f4 100644 --- a/project/settings/development.py +++ b/project/settings/development.py @@ -19,7 +19,7 @@ DOMAIN_URI = 'gm.id-east.ru' # ELASTICSEARCH SETTINGS ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'localhost:9200' + 'hosts': 'elasticsearch:9200' } } diff --git a/requirements/base.txt b/requirements/base.txt index a4349bea..3a45dedc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -43,6 +43,7 @@ sorl-thumbnail==12.5.0 mysqlclient==1.4.4 +PyYAML==5.1.2 # temp solution redis==3.2.0