From 4010c9fedea6ed20d0976f0f4b45dcb9cc337dc0 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 19:16:19 +0300 Subject: [PATCH 1/8] News multilang slugs (model && views) --- _dockerfiles/db/Dockerfile | 1 + _dockerfiles/db/hstore.sql | 1 + apps/news/migrations/0039_news_slugs.py | 27 +++++++++++++++++++++++++ apps/news/models.py | 4 ++++ apps/news/views.py | 6 +++++- apps/utils/views.py | 3 +++ project/settings/base.py | 1 + 7 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 _dockerfiles/db/hstore.sql create mode 100644 apps/news/migrations/0039_news_slugs.py diff --git a/_dockerfiles/db/Dockerfile b/_dockerfiles/db/Dockerfile index c3e35955..1a9d28c6 100644 --- a/_dockerfiles/db/Dockerfile +++ b/_dockerfiles/db/Dockerfile @@ -1,3 +1,4 @@ FROM mdillon/postgis:10 RUN localedef -i ru_RU -c -f UTF-8 -A /usr/share/locale/locale.alias ru_RU.UTF-8 ENV LANG ru_RU.utf8 +COPY hstore.sql /docker-entrypoint-initdb.d \ No newline at end of file diff --git a/_dockerfiles/db/hstore.sql b/_dockerfiles/db/hstore.sql new file mode 100644 index 00000000..97962703 --- /dev/null +++ b/_dockerfiles/db/hstore.sql @@ -0,0 +1 @@ +create extension hstore; \ No newline at end of file diff --git a/apps/news/migrations/0039_news_slugs.py b/apps/news/migrations/0039_news_slugs.py new file mode 100644 index 00000000..e8b996c8 --- /dev/null +++ b/apps/news/migrations/0039_news_slugs.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2.7 on 2019-12-10 13:49 + +import django.contrib.postgres.fields.hstore +from django.db import migrations + +def migrate_slugs(apps, schemaeditor): + News = apps.get_model('news', 'News') + for news in News.objects.all(): + if news.slug: + news.slugs = {'en-GB': news.slug} + news.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0038_news_backoffice_title'), + ] + + operations = [ + migrations.AddField( + model_name='news', + name='slugs', + field=django.contrib.postgres.fields.hstore.HStoreField(blank=True, default=None, help_text='{"en-GB":"some slug"}', null=True, verbose_name='Slugs for current news obj'), + ), + migrations.RunPython(migrate_slugs, migrations.RunPython.noop) + ] diff --git a/apps/news/models.py b/apps/news/models.py index a2db35b4..465ce8ee 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -12,6 +12,7 @@ from utils.models import (BaseAttributes, TJSONField, TranslatedFieldsMixin, Has FavoritesMixin) from utils.querysets import TranslationQuerysetMixin from django.conf import settings +from django.contrib.postgres.fields import HStoreField class Agenda(ProjectBaseMixin, TranslatedFieldsMixin): @@ -182,6 +183,9 @@ class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixi verbose_name=_('End')) slug = models.SlugField(unique=True, max_length=255, verbose_name=_('News slug')) + slugs = HStoreField(null=True, blank=True, default=None, + verbose_name=_('Slugs for current news obj'), + help_text='{"en-GB":"some slug"}') state = models.PositiveSmallIntegerField(default=WAITING, choices=STATE_CHOICES, verbose_name=_('State')) is_highlighted = models.BooleanField(default=False, diff --git a/apps/news/views.py b/apps/news/views.py index a4a5c33a..54868e52 100644 --- a/apps/news/views.py +++ b/apps/news/views.py @@ -31,6 +31,10 @@ class NewsMixinView: qs = qs.by_country_code(country_code) return qs + def get_object(self): + return self.get_queryset() \ + .filter(slugs__values__contains=[self.kwargs['slug']]).first() + class NewsListView(NewsMixinView, generics.ListAPIView): """News list view.""" @@ -46,7 +50,7 @@ class NewsListView(NewsMixinView, generics.ListAPIView): class NewsDetailView(NewsMixinView, generics.RetrieveAPIView): """News detail view.""" - lookup_field = 'slug' + lookup_field = None serializer_class = serializers.NewsDetailWebSerializer def get_queryset(self): diff --git a/apps/utils/views.py b/apps/utils/views.py index 9ac8ca60..379e501b 100644 --- a/apps/utils/views.py +++ b/apps/utils/views.py @@ -9,6 +9,7 @@ from rest_framework.response import Response from gallery.tasks import delete_image from search_indexes.documents import es_update +from news.models import News # JWT @@ -124,6 +125,8 @@ class BaseCreateDestroyMixinView(generics.CreateAPIView, generics.DestroyAPIView lookup_field = 'slug' def get_base_object(self): + if isinstance(self._model, News): + get_object_or_404(self._model, slugs__values__contains=[self.kwargs['slug']]) return get_object_or_404(self._model, slug=self.kwargs['slug']) def es_update_base_object(self): diff --git a/project/settings/base.py b/project/settings/base.py index 5a48c261..31b7b8f7 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -48,6 +48,7 @@ CONTRIB_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.gis', + 'django.contrib.postgres', ] From c1bb7c9b79f1a28b79e60788efcf4060a19afa0e Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 19:23:40 +0300 Subject: [PATCH 2/8] News multilang slug (ES) --- apps/news/migrations/0040_remove_news_slug.py | 17 +++++++++++++++++ apps/news/models.py | 2 -- apps/news/serializers.py | 2 +- apps/search_indexes/documents/establishment.py | 2 +- apps/search_indexes/documents/news.py | 2 +- apps/search_indexes/serializers.py | 2 +- 6 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 apps/news/migrations/0040_remove_news_slug.py diff --git a/apps/news/migrations/0040_remove_news_slug.py b/apps/news/migrations/0040_remove_news_slug.py new file mode 100644 index 00000000..f4ef00bb --- /dev/null +++ b/apps/news/migrations/0040_remove_news_slug.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.7 on 2019-12-10 16:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('news', '0039_news_slugs'), + ] + + operations = [ + migrations.RemoveField( + model_name='news', + name='slug', + ), + ] diff --git a/apps/news/models.py b/apps/news/models.py index 465ce8ee..7931b17b 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -181,8 +181,6 @@ class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixi verbose_name=_('Start')) end = models.DateTimeField(blank=True, null=True, default=None, verbose_name=_('End')) - slug = models.SlugField(unique=True, max_length=255, - verbose_name=_('News slug')) slugs = HStoreField(null=True, blank=True, default=None, verbose_name=_('Slugs for current news obj'), help_text='{"en-GB":"some slug"}') diff --git a/apps/news/serializers.py b/apps/news/serializers.py index c14e28fe..3e5d3ecb 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -80,7 +80,7 @@ class NewsBaseSerializer(ProjectModelSerializer): 'is_highlighted', 'news_type', 'tags', - 'slug', + 'slugs', 'view_counter', ) diff --git a/apps/search_indexes/documents/establishment.py b/apps/search_indexes/documents/establishment.py index c6b68ed4..8ae26097 100644 --- a/apps/search_indexes/documents/establishment.py +++ b/apps/search_indexes/documents/establishment.py @@ -7,7 +7,7 @@ from establishment import models EstablishmentIndex = Index(settings.ELASTICSEARCH_INDEX_NAMES.get(__name__, 'establishment')) -EstablishmentIndex.settings(number_of_shards=1, number_of_replicas=1) +EstablishmentIndex.settings(number_of_shards=5, number_of_replicas=2) @EstablishmentIndex.doc_type diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py index 2aab01c8..f48494fd 100644 --- a/apps/search_indexes/documents/news.py +++ b/apps/search_indexes/documents/news.py @@ -17,6 +17,7 @@ class NewsDocument(Document): 'name': fields.KeywordField()}) title = fields.ObjectField(attr='title_indexing', properties=OBJECT_FIELD_PROPERTIES) + slugs = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) backoffice_title = fields.TextField(analyzer='english') subtitle = fields.ObjectField(attr='subtitle_indexing', properties=OBJECT_FIELD_PROPERTIES) @@ -50,7 +51,6 @@ class NewsDocument(Document): fields = ( 'id', 'end', - 'slug', 'state', 'is_highlighted', 'template', diff --git a/apps/search_indexes/serializers.py b/apps/search_indexes/serializers.py index 3b5561fa..81a31afa 100644 --- a/apps/search_indexes/serializers.py +++ b/apps/search_indexes/serializers.py @@ -221,7 +221,7 @@ class NewsDocumentSerializer(InFavoritesMixin, DocumentSerializer): 'news_type', 'tags', 'start', - 'slug', + 'slugs', ) @staticmethod From c90f8302ee2f397b488929e498f01f449a0f6b94 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 19:29:11 +0300 Subject: [PATCH 3/8] fix news indexing --- apps/news/models.py | 2 +- apps/search_indexes/documents/news.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/news/models.py b/apps/news/models.py index 7931b17b..f54378b6 100644 --- a/apps/news/models.py +++ b/apps/news/models.py @@ -232,7 +232,7 @@ class News(GalleryModelMixin, BaseAttributes, TranslatedFieldsMixin, HasTagsMixi @property def web_url(self): - return reverse('web:news:rud', kwargs={'slug': self.slug}) + return reverse('web:news:rud', kwargs={'slug': next(iter(self.slugs.values()))}) def should_read(self, user): return self.__class__.objects.should_read(self, user)[:3] diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py index f48494fd..05e8c2bb 100644 --- a/apps/search_indexes/documents/news.py +++ b/apps/search_indexes/documents/news.py @@ -17,7 +17,7 @@ class NewsDocument(Document): 'name': fields.KeywordField()}) title = fields.ObjectField(attr='title_indexing', properties=OBJECT_FIELD_PROPERTIES) - slugs = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) + slugs = fields.ObjectField() backoffice_title = fields.TextField(analyzer='english') subtitle = fields.ObjectField(attr='subtitle_indexing', properties=OBJECT_FIELD_PROPERTIES) From 77af35f543bcb5151fa4aa74a8c3a5af1b86c0e8 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 19:33:15 +0300 Subject: [PATCH 4/8] many slugs es news fix --- apps/search_indexes/documents/news.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py index 05e8c2bb..ec89f9ee 100644 --- a/apps/search_indexes/documents/news.py +++ b/apps/search_indexes/documents/news.py @@ -17,7 +17,7 @@ class NewsDocument(Document): 'name': fields.KeywordField()}) title = fields.ObjectField(attr='title_indexing', properties=OBJECT_FIELD_PROPERTIES) - slugs = fields.ObjectField() + slugs = fields.ListField(fields.ObjectField()) backoffice_title = fields.TextField(analyzer='english') subtitle = fields.ObjectField(attr='subtitle_indexing', properties=OBJECT_FIELD_PROPERTIES) From 401abc568cf0d6b003cb8f52d6782efad725192a Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 20:04:23 +0300 Subject: [PATCH 5/8] news unique slug creation --- apps/news/serializers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/news/serializers.py b/apps/news/serializers.py index 3e5d3ecb..12b1faf6 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -177,6 +177,12 @@ class NewsBackOfficeBaseSerializer(NewsBaseSerializer): 'backoffice_title': {'allow_null': False}, } + def validate(self, attrs): + slugs = attrs.get('slugs', {}) + if models.News.objects.filter(slugs__values__contains=[slugs.values()]).exists(): + raise serializers.ValidationError({'slugs': _('News with this slug already exists.')}) + return attrs + class NewsBackOfficeDetailSerializer(NewsBackOfficeBaseSerializer, NewsDetailSerializer): From 377d8196dc6ee3009b031bec0e4c84710b0b7033 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 20:58:08 +0300 Subject: [PATCH 6/8] fix slugs serialization --- apps/search_indexes/documents/news.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/search_indexes/documents/news.py b/apps/search_indexes/documents/news.py index ec89f9ee..62e3e984 100644 --- a/apps/search_indexes/documents/news.py +++ b/apps/search_indexes/documents/news.py @@ -17,7 +17,7 @@ class NewsDocument(Document): 'name': fields.KeywordField()}) title = fields.ObjectField(attr='title_indexing', properties=OBJECT_FIELD_PROPERTIES) - slugs = fields.ListField(fields.ObjectField()) + slugs = fields.ObjectField(properties=OBJECT_FIELD_PROPERTIES) backoffice_title = fields.TextField(analyzer='english') subtitle = fields.ObjectField(attr='subtitle_indexing', properties=OBJECT_FIELD_PROPERTIES) @@ -45,6 +45,10 @@ class NewsDocument(Document): multi=True) favorites_for_users = fields.ListField(field=fields.IntegerField()) start = fields.DateField(attr='start') + + def prepare_slugs(self, instance): + return {locale: instance.slugs.get(locale) for locale in OBJECT_FIELD_PROPERTIES} + class Django: model = models.News From ecef1a217a59196d166361f76bb340b1af3b2a8f Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 22:10:37 +0300 Subject: [PATCH 7/8] fix some test --- apps/favorites/tests.py | 3 ++- apps/news/migrations/0039_news_slugs.py | 2 ++ apps/news/serializers.py | 2 +- apps/news/tests.py | 13 +++++++------ apps/utils/tests/tests_translated.py | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/favorites/tests.py b/apps/favorites/tests.py index 5f0a2fcd..22953133 100644 --- a/apps/favorites/tests.py +++ b/apps/favorites/tests.py @@ -42,8 +42,9 @@ class BaseTestCase(APITestCase): start=datetime.fromisoformat("2020-12-03 12:00:00"), end=datetime.fromisoformat("2020-12-03 12:00:00"), state=News.PUBLISHED, - slug='test-news' + slugs={'en-GB': 'test-news'} ) + self.slug = next(iter(self.test_news.slugs.values())) self.test_content_type = ContentType.objects.get( app_label="news", model="news") diff --git a/apps/news/migrations/0039_news_slugs.py b/apps/news/migrations/0039_news_slugs.py index e8b996c8..cc9a3194 100644 --- a/apps/news/migrations/0039_news_slugs.py +++ b/apps/news/migrations/0039_news_slugs.py @@ -2,6 +2,7 @@ import django.contrib.postgres.fields.hstore from django.db import migrations +from django.contrib.postgres.operations import HStoreExtension def migrate_slugs(apps, schemaeditor): News = apps.get_model('news', 'News') @@ -18,6 +19,7 @@ class Migration(migrations.Migration): ] operations = [ + HStoreExtension(), migrations.AddField( model_name='news', name='slugs', diff --git a/apps/news/serializers.py b/apps/news/serializers.py index 12b1faf6..da3ea2df 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -179,7 +179,7 @@ class NewsBackOfficeBaseSerializer(NewsBaseSerializer): def validate(self, attrs): slugs = attrs.get('slugs', {}) - if models.News.objects.filter(slugs__values__contains=[slugs.values()]).exists(): + if models.News.objects.filter(slugs__values__contains=list(slugs.values())).exists(): raise serializers.ValidationError({'slugs': _('News with this slug already exists.')}) return attrs diff --git a/apps/news/tests.py b/apps/news/tests.py index 40f47312..42c4a694 100644 --- a/apps/news/tests.py +++ b/apps/news/tests.py @@ -66,10 +66,11 @@ class BaseTestCase(APITestCase): start=datetime.now() + timedelta(hours=-2), end=datetime.now() + timedelta(hours=2), state=News.PUBLISHED, - slug='test-news-slug', + slugs={'en-GB': 'test-news-slug'}, country=self.country_ru, site=self.site_ru ) + self.slug = next(iter(self.test_news.slugs.values())) class NewsTestCase(BaseTestCase): @@ -84,7 +85,7 @@ class NewsTestCase(BaseTestCase): "start": datetime.now() + timedelta(hours=-2), "end": datetime.now() + timedelta(hours=2), "state": News.PUBLISHED, - "slug": 'test-news-slug_post', + "slugs": {'en-GB': 'test-news-slug_post'}, "country_id": self.country_ru.id, "site_id": self.site_ru.id } @@ -97,7 +98,7 @@ class NewsTestCase(BaseTestCase): response = self.client.get(reverse('web:news:list')) self.assertEqual(response.status_code, status.HTTP_200_OK) - response = self.client.get(f"/api/web/news/slug/{self.test_news.slug}/") + response = self.client.get(f"/api/web/news/slug/{self.slug}/") self.assertEqual(response.status_code, status.HTTP_200_OK) response = self.client.get("/api/web/news/types/") @@ -117,7 +118,7 @@ class NewsTestCase(BaseTestCase): data = { 'id': self.test_news.id, 'description': {"ru-RU": "Description test news!"}, - 'slug': self.test_news.slug, + 'slugs': self.test_news.slugs, 'start': self.test_news.start, 'news_type_id': self.test_news.news_type_id, 'country_id': self.country_ru.id, @@ -133,10 +134,10 @@ class NewsTestCase(BaseTestCase): "object_id": self.test_news.id } - response = self.client.post(f'/api/web/news/slug/{self.test_news.slug}/favorites/', data=data) + response = self.client.post(f'/api/web/news/slug/{self.slug}/favorites/', data=data) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - response = self.client.delete(f'/api/web/news/slug/{self.test_news.slug}/favorites/', format='json') + response = self.client.delete(f'/api/web/news/slug/{self.slug}/favorites/', format='json') self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) diff --git a/apps/utils/tests/tests_translated.py b/apps/utils/tests/tests_translated.py index 6249ebd1..6ddcd1e7 100644 --- a/apps/utils/tests/tests_translated.py +++ b/apps/utils/tests/tests_translated.py @@ -56,17 +56,18 @@ class TranslateFieldTests(BaseTestCase): start=datetime.now(pytz.utc) + timedelta(hours=-13), end=datetime.now(pytz.utc) + timedelta(hours=13), news_type=self.news_type, - slug='test', + slugs={'en-GB': 'test'}, state=News.PUBLISHED, country=self.country_ru, ) + self.slug = next(iter(self.news_item.slugs.values())) self.news_item.save() def test_model_field(self): self.assertTrue(hasattr(self.news_item, "title_translated")) def test_read_locale(self): - response = self.client.get(f"/api/web/news/slug/{self.news_item.slug}/", format='json') + response = self.client.get(f"/api/web/news/slug/{self.slug}/", format='json') self.assertEqual(response.status_code, status.HTTP_200_OK) news_data = response.json() self.assertIn("title_translated", news_data) From 043fcb262b19efa0e8987929f5b366e11f8ddc54 Mon Sep 17 00:00:00 2001 From: Kuroshini Date: Tue, 10 Dec 2019 23:30:07 +0300 Subject: [PATCH 8/8] fix test #2 --- apps/news/serializers.py | 6 ++++-- apps/utils/views.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/news/serializers.py b/apps/news/serializers.py index da3ea2df..846bc31a 100644 --- a/apps/news/serializers.py +++ b/apps/news/serializers.py @@ -179,7 +179,9 @@ class NewsBackOfficeBaseSerializer(NewsBaseSerializer): def validate(self, attrs): slugs = attrs.get('slugs', {}) - if models.News.objects.filter(slugs__values__contains=list(slugs.values())).exists(): + if models.News.objects.filter( + slugs__values__contains=list(slugs.values()) + ).exclude(id=attrs.get('id', 0)).exists(): raise serializers.ValidationError({'slugs': _('News with this slug already exists.')}) return attrs @@ -262,7 +264,7 @@ class NewsFavoritesCreateSerializer(FavoritesCreateSerializer): def validate(self, attrs): """Overridden validate method""" # Check establishment object - news_qs = models.News.objects.filter(slug=self.slug) + news_qs = models.News.objects.filter(slugs__values__contains=[self.slug]) # Check establishment obj by slug from lookup_kwarg if not news_qs.exists(): diff --git a/apps/utils/views.py b/apps/utils/views.py index 379e501b..77f22032 100644 --- a/apps/utils/views.py +++ b/apps/utils/views.py @@ -125,8 +125,8 @@ class BaseCreateDestroyMixinView(generics.CreateAPIView, generics.DestroyAPIView lookup_field = 'slug' def get_base_object(self): - if isinstance(self._model, News): - get_object_or_404(self._model, slugs__values__contains=[self.kwargs['slug']]) + if 'slugs' in [f.name for f in self._model._meta.get_fields()]: # slugs instead of `slug` + return get_object_or_404(self._model, slugs__values__contains=[self.kwargs['slug']]) return get_object_or_404(self._model, slug=self.kwargs['slug']) def es_update_base_object(self):