phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

migrate from legacy news

Browse Source
This commit is contained in:
alex 2019-10-24 14:01:38 +03:00
parent 0adac058c5
commit 3ce7e186f7
8 changed files with 87 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
apps/news/transfer_data.py
View File

@ -1,23 +1,23 @@
from transfer.serializers.news import NewsSerializer
from transfer.models import PageTexts
from news.models import NewsType
from django.db.models import Value, IntegerField, F
from pprint import pprint
from news.models import NewsType
from transfer.models import PageTexts
from transfer.serializers.news import NewsSerializer
def transfer_news():
news_type, _ = NewsType.objects.get_or_create(name="News")
queryset = PageTexts.objects.filter(page__type="News").annotate(news_type=Value(news_type.id, output_field=IntegerField()),
playlist=Value(1, output_field=IntegerField()))
queryset = queryset.annotate(attachment_file_name=F('page__attachment_file_name'))
queryset = PageTexts.objects.filter(page__type="News").annotate(
news_type=Value(news_type.id, output_field=IntegerField()),
)
queryset = queryset.annotate(template=F('page__template'))
serialized_data = NewsSerializer(data=list(queryset.values()), many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"News serializer errors: {serialized_data.errors}")
print(f"News serializer errors: {serialized_data.errors}")
data_types = {

1
apps/transfer/models.py
View File

@ -694,6 +694,7 @@ class ReviewTexts(MigrateMixin):
managed = False
db_table = 'review_texts'
class Comments(MigrateMixin):
using = 'legacy'

143
apps/transfer/serializers/news.py
View File

@ -1,123 +1,68 @@
from rest_framework import serializers
from news.models import News
from uuid import uuid4
import datetime
from django.conf import settings
from utils.legacy_parser import parse_legacy_content
class NewsSerializer(serializers.ModelSerializer):
# image_url = serializers.CharField()
# preview_image_url = serializers.CharField()
attachment_file_name = serializers.CharField(allow_null=True)
# body = serializers.CharField(allow_null=True, source="description")
locale = serializers.CharField()
body = serializers.CharField(allow_null=True)
title = serializers.CharField()
template = serializers.CharField()
state = serializers.CharField()
created_at = serializers.DateTimeField(source="start", format='%m-%d-%Y %H:%M:%S')
created_at = serializers.DateTimeField(source='start', format='%m-%d-%Y %H:%M:%S')
class Meta:
model = News
fields = (
"created_at",
"state",
"template",
# "image_url",
# "preview_image_url",
"attachment_file_name",
"title",
"body",
"slug",
"news_type",
"playlist"
'created_at',
'state',
'template',
'title',
'body',
'slug',
'news_type',
'locale',
)
def validate(self, data):
data["state"] = self.get_state(data)
data["template"] = self.get_template(data)
data["title"] = self.get_title(data)
data["description"] = self.get_description(data)
data["image_url"] = self.get_image_url(data)
data["preview_image_url"] = self.get_preview_image_url(data)
data.pop("attachment_file_name")
data.pop("body")
data.update({
'state': self.get_state(data),
'template': self.get_template(data),
'title': self.get_title(data),
'description': self.get_description(data),
})
data.pop('body')
data.pop('locale')
return data
def create(self, validated_data):
# validated_data['state'] = self.get_state(validated_data)
return News.objects.create(**validated_data)
def get_template(self, obj):
if obj["template"] == "main":
return News.MAIN
elif obj["template"] == "main.pdf.erb":
return News.MAIN_PDF_ERB
else:
return News.NEWSPAPER
@staticmethod
def get_template(data):
templates = {
'main': News.MAIN,
'main.pdf.erb': News.MAIN_PDF_ERB,
}
return templates.get(data['template'], News.MAIN)
# if obj.template == "main":
# return News.MAIN
# elif obj.template == "main.pdf.erb":
# return News.MAIN_PDF_ERB
# else:
# return News.NEWSPAPER
@staticmethod
def get_state(data):
states = {
'new': News.PUBLISHED,
'published': News.PUBLISHED,
'hidden': News.HIDDEN,
}
return states.get(data['state'], News.WAITING)
def get_state(self, obj):
if obj["state"] == "published":
return News.PUBLISHED
elif obj["state"] == "hidden":
return News.HIDDEN
elif obj["state"] == "published_exclusive":
return News.PUBLISHED_EXCLUSIVE
else:
return News.WAITING
@staticmethod
def get_title(data):
return {data['locale']: data['title']}
# if obj.state == "published":
# return News.PUBLISHED
# elif obj.state == "hidden":
# return News.HIDDEN
# elif obj.state == "published_exclusive":
# return News.PUBLISHED_EXCLUSIVE
# else:
# return News.WAITING
# django.db.utils.IntegrityError: "start" NOT NULL
# def get_start(self, obj):
# # return obj.created_at
# return datetime.datetime.now().isoformat()
def get_title(self, obj):
tit = obj.get("title")
return {"en-GB": tit}
# return f'{"en-GB": {obj.title}}'
def get_description(self, obj):
# return {"en-GB": {obj["body"]}}
desc = obj.get("body")
return {"en-GB": desc}
# if not hasattr(obj, "body"):
# return ""
# else:
# return obj.body
def get_image_url(self, obj):
if not obj.get("attachment_file_name"):
return ""
if len(obj.get("attachment_file_name"))>200:
return obj.get("attachment_file_name")[:200]
return obj.get("attachment_file_name")
# if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None:
# return ""
# return obj.attachment_file_name
def get_preview_image_url(self, obj):
if not obj.get("attachment_file_name"):
return ""
if len(obj.get("attachment_file_name"))>200:
return obj.get("attachment_file_name")[:200]
return obj.get("attachment_file_name")
# if not hasattr(obj, "attachment_file_name") or obj.attachment_file_name is None:
# return ""
# return obj.attachment_file_name
@staticmethod
def get_description(data):
content = None
if data['body']:
content = parse_legacy_content(data['body'])
return {data['locale']: content}

12
apps/utils/legacy_parser.py Normal file
View File

@ -0,0 +1,12 @@
import yaml
def parse_legacy_content(legacy_content):
clear_str = '!ruby/hash:ActiveSupport::HashWithIndifferentAccess'
content_dict = yaml.safe_load(legacy_content.replace(clear_str, ''))
result = ''
try:
result = content_dict['news_content']['value']
except IndexError:
pass
return result

20
docker-compose.mysql.yml
View File

@ -14,6 +14,8 @@ services:
volumes:
- .:/code
- gm-mysql_db:/var/lib/mysql
networks:
- mysql_network
# PostgreSQL database
db:
@ -29,6 +31,8 @@ services:
- "5436:5432"
volumes:
- gm-db:/var/lib/postgresql/data/
networks:
- database_network
elasticsearch:
image: elasticsearch:7.3.1
@ -42,6 +46,8 @@ services:
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- discovery.type=single-node
- xpack.security.enabled=false
networks:
- elasticsearch_network
# Redis
redis:
@ -113,6 +119,10 @@ services:
- gm-media:/media-data
ports:
- "8000:8000"
networks:
- database_network
- mysql_network
- elasticsearch_network
volumes:
gm-mysql_db:
@ -124,4 +134,12 @@ volumes:
gm-media:
name: gm-media
gm-esdata:
gm-esdata:
networks:
database_network:
driver: bridge
mysql_network:
driver: bridge
elasticsearch_network:
driver: bridge

2
project/settings/base.py
View File

@ -160,7 +160,7 @@ DATABASES = {
'legacy': {
'ENGINE': 'django.db.backends.mysql',
# 'HOST': '172.23.0.1',
'HOST': '127.0.0.1',
'HOST': 'mysql_db',
'PORT': 3306,
'NAME': 'dev',
'USER': 'dev',

2
project/settings/development.py
View File

@ -19,7 +19,7 @@ DOMAIN_URI = 'gm.id-east.ru'
# ELASTICSEARCH SETTINGS
ELASTICSEARCH_DSL = {
'default': {
'hosts': 'localhost:9200'
'hosts': 'elasticsearch:9200'
}
}

1
requirements/base.txt
View File

@ -43,6 +43,7 @@ sorl-thumbnail==12.5.0
mysqlclient==1.4.4
PyYAML==5.1.2
# temp solution
redis==3.2.0