From 45b0b93068b580dcb6f9ce1416a5ac2a4d4566ce Mon Sep 17 00:00:00 2001 From: Dmitriy Kuzmenko Date: Sat, 16 Nov 2019 22:49:38 +0300 Subject: [PATCH] add new migrations for tags --- apps/news/views.py | 5 +- apps/search_indexes/__init__.py | 2 +- apps/search_indexes/signals.py | 134 +++++++++---------- apps/tag/management/commands/add_tags.py | 162 +++++++++++++---------- apps/tag/models.py | 1 + project/settings/base.py | 6 +- project/settings/local.py | 10 +- 7 files changed, 171 insertions(+), 149 deletions(-) diff --git a/apps/news/views.py b/apps/news/views.py index f7d18101..d0528132 100644 --- a/apps/news/views.py +++ b/apps/news/views.py @@ -1,11 +1,8 @@ """News app views.""" from django.conf import settings -from django.db.transaction import on_commit from django.shortcuts import get_object_or_404 -from rest_framework import generics, permissions, status -from rest_framework.response import Response +from rest_framework import generics, permissions -from gallery.tasks import delete_image from news import filters, models, serializers from rating.tasks import add_rating from utils.permissions import IsCountryAdmin, IsContentPageManager diff --git a/apps/search_indexes/__init__.py b/apps/search_indexes/__init__.py index 6c812219..c45cbc38 100644 --- a/apps/search_indexes/__init__.py +++ b/apps/search_indexes/__init__.py @@ -1 +1 @@ -from search_indexes.signals import update_document +# from search_indexes.signals import update_document diff --git a/apps/search_indexes/signals.py b/apps/search_indexes/signals.py index 5bee4e17..1c7dbfe0 100644 --- a/apps/search_indexes/signals.py +++ b/apps/search_indexes/signals.py @@ -1,67 +1,67 @@ -"""Search indexes app signals.""" -from django.db.models.signals import post_save -from django.dispatch import receiver -from django_elasticsearch_dsl.registries import registry - - -@receiver(post_save) -def update_document(sender, **kwargs): - from establishment.models import Establishment - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - - app_label_model_name_to_filter = { - ('location','country'): 'address__city__country', - ('location','city'): 'address__city', - ('location', 'address'): 'address', - # todo: remove after migration - ('establishment', 'establishmenttype'): 'establishment_type', - ('establishment', 'establishmentsubtype'): 'establishment_subtypes', - ('tag', 'tag'): 'tags', - } - filter_name = app_label_model_name_to_filter.get((app_label, model_name)) - if filter_name: - qs = Establishment.objects.filter(**{filter_name: instance}) - for product in qs: - registry.update(product) - - -@receiver(post_save) -def update_news(sender, **kwargs): - from news.models import News - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - app_label_model_name_to_filter = { - ('location','country'): 'country', - ('news','newstype'): 'news_type', - ('tag', 'tag'): 'tags', - } - filter_name = app_label_model_name_to_filter.get((app_label, model_name)) - if filter_name: - qs = News.objects.filter(**{filter_name: instance}) - for product in qs: - registry.update(product) - - -@receiver(post_save) -def update_product(sender, **kwargs): - from product.models import Product - app_label = sender._meta.app_label - model_name = sender._meta.model_name - instance = kwargs['instance'] - app_label_model_name_to_filter = { - ('product','productstandard'): 'standards', - ('product', 'producttype'): 'product_type', - ('tag','tag'): 'tags', - ('location', 'wineregion'): 'wine_region', - ('location', 'winesubregion'): 'wine_sub_region', - ('location', 'winevillage'): 'wine_village', - ('establishment', 'establishment'): 'establishment', - } - filter_name = app_label_model_name_to_filter.get((app_label, model_name)) - if filter_name: - qs = Product.objects.filter(**{filter_name: instance}) - for product in qs: - registry.update(product) +# """Search indexes app signals.""" +# from django.db.models.signals import post_save +# from django.dispatch import receiver +# from django_elasticsearch_dsl.registries import registry +# +# +# @receiver(post_save) +# def update_document(sender, **kwargs): +# from establishment.models import Establishment +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# +# app_label_model_name_to_filter = { +# ('location','country'): 'address__city__country', +# ('location','city'): 'address__city', +# ('location', 'address'): 'address', +# # todo: remove after migration +# ('establishment', 'establishmenttype'): 'establishment_type', +# ('establishment', 'establishmentsubtype'): 'establishment_subtypes', +# ('tag', 'tag'): 'tags', +# } +# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) +# if filter_name: +# qs = Establishment.objects.filter(**{filter_name: instance}) +# for product in qs: +# registry.update(product) +# +# +# @receiver(post_save) +# def update_news(sender, **kwargs): +# from news.models import News +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# app_label_model_name_to_filter = { +# ('location','country'): 'country', +# ('news','newstype'): 'news_type', +# ('tag', 'tag'): 'tags', +# } +# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) +# if filter_name: +# qs = News.objects.filter(**{filter_name: instance}) +# for product in qs: +# registry.update(product) +# +# +# @receiver(post_save) +# def update_product(sender, **kwargs): +# from product.models import Product +# app_label = sender._meta.app_label +# model_name = sender._meta.model_name +# instance = kwargs['instance'] +# app_label_model_name_to_filter = { +# ('product','productstandard'): 'standards', +# ('product', 'producttype'): 'product_type', +# ('tag','tag'): 'tags', +# ('location', 'wineregion'): 'wine_region', +# ('location', 'winesubregion'): 'wine_sub_region', +# ('location', 'winevillage'): 'wine_village', +# ('establishment', 'establishment'): 'establishment', +# } +# filter_name = app_label_model_name_to_filter.get((app_label, model_name)) +# if filter_name: +# qs = Product.objects.filter(**{filter_name: instance}) +# for product in qs: +# registry.update(product) diff --git a/apps/tag/management/commands/add_tags.py b/apps/tag/management/commands/add_tags.py index 0fba46f8..b0c54e5c 100644 --- a/apps/tag/management/commands/add_tags.py +++ b/apps/tag/management/commands/add_tags.py @@ -3,87 +3,111 @@ from django.core.management.base import BaseCommand from establishment.models import Establishment, EstablishmentType from transfer import models as legacy from tag.models import Tag, TagCategory +from tqdm import tqdm +from django.db import connections +from collections import namedtuple + + +def namedtuplefetchall(cursor): + "Return all rows from a cursor as a namedtuple" + desc = cursor.description + nt_result = namedtuple('Result', [col[0] for col in desc]) + return [nt_result(*row) for row in cursor.fetchall()] + + +def metadata_category_sql(): + with connections['legacy'].cursor() as cursor: + cursor.execute( + '''SELECT + `key`, + establishments.type, + key_value_metadata.`value_type`, + public, + key_value_metadata.id as 'old_id' + FROM metadata + LEFT JOIN establishments + ON metadata.establishment_id=establishments.id + LEFT JOIN key_value_metadata + ON metadata.key=key_value_metadata.key_name + GROUP BY + establishments.type, + `key`, + key_value_metadata.`value_type`, + public, old_id;''' + ) + return namedtuplefetchall(cursor) + + +def metadata_tags_sql(): + with connections['legacy'].cursor() as cursor: + cursor.execute( + """ + SELECT + value, + `key` as category, + establishment_id +FROM metadata +WHERE establishment_id is not null""" + ) + return namedtuplefetchall(cursor) class Command(BaseCommand): help = 'Add tags values from old db to new db' - def handle(self, *args, **kwargs): + def get_type(self, meta): + meta_type = meta.value_type + if not meta.value_type: + if meta.key == 'wineyard_visits': + meta_type = 'list' + elif meta.key in ['private_room', 'outside_sits']: + meta_type = 'bool' + return meta_type + def handle(self, *args, **kwargs): existing_establishment = Establishment.objects.filter( old_id__isnull=False ) - ESTABLISHMENT = 1 - SHOP = 2 - RESTAURANT = 3 - WINEYARD = 4 - MAPPER = { - RESTAURANT: EstablishmentType.RESTAURANT, - WINEYARD: EstablishmentType.PRODUCER, - SHOP: EstablishmentType.ARTISAN + 'Restaurant': EstablishmentType.RESTAURANT, + 'Wineyard': EstablishmentType.PRODUCER, + 'Shop': EstablishmentType.ARTISAN } + # remove old black category + for establishment_tag in tqdm(EstablishmentType.objects.all()): + establishment_tag.tag_categories.remove(*list( + establishment_tag.tag_categories.exclude( + tags__establishments__isnull=False).distinct())) - mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all() - for key, value in MAPPER.items(): - values_meta_id_list = mapper_values_meta.filter( - key_value_metadatum_establishment_id=key - ).values_list('key_value_metadatum_id') + # created Tag Category + for meta in tqdm(metadata_category_sql()): + category, _ = TagCategory.objects.update_or_create( + index_name=meta.key, + defaults={ + "public": True if meta.public == 1 else False, + "value_type": self.get_type(meta), + "label": {"en-GB": meta.key} + } + ) - est_type, _ = EstablishmentType.objects.get_or_create(index_name=value) + # add to EstablishmentType + est_type = EstablishmentType.objects.get(index_name=MAPPER[meta.type]) + if category not in est_type.tag_categories.all(): + est_type.tag_categories.add(category) - key_value_metadata = legacy.KeyValueMetadata.objects.filter( - id__in=values_meta_id_list) + count = 0 + for meta_tag in tqdm(metadata_tags_sql()): - # create TagCategory - for key_value in key_value_metadata: - tag_category, created = TagCategory.objects.get_or_create( - index_name=key_value.key_name, - ) - - if created: - tag_category.label = { - 'en-GB': key_value.key_name, - 'fr-FR': key_value.key_name, - 'ru-RU': key_value.key_name, - } - tag_category.value_type = key_value.value_type - tag_category.save() - est_type.tag_categories.add( - tag_category - ) - - # create Tag - for tag in key_value.metadata_set.filter( - establishment__id__in=list( - existing_establishment.values_list('old_id', flat=True) - )): - - new_tag, created = Tag.objects.get_or_create( - value=tag.value, - category=tag_category, - ) - if created: - - sp = tag.value.split('_') - value = ' '.join([sp[0].capitalize()] + sp[1:]) - - trans = { - 'en-GB': value, - 'fr-FR': value, - 'ru-RU': value, - } - - aliases = legacy.MetadatumAliases.objects.filter(value=tag.value) - - for alias in aliases: - trans[alias.locale] = alias.meta_alias - - new_tag.label = trans - new_tag.save() - - est = existing_establishment.filter( - old_id=tag.establishment_id).first() - if est: - est.tags.add(new_tag) - est.save() + tag, _ = Tag.objects.update_or_create( + category=TagCategory.objects.get(index_name=meta_tag.category), + value=meta_tag.value, + defaults={ + "label": {"en-GB": meta_tag.value} + } + ) + establishment = existing_establishment.filter(old_id=meta_tag.establishment_id).first() + if establishment: + if tag not in establishment.tags.all(): + establishment.tags.add(tag) + count += 1 + self.stdout.write(self.style.WARNING(f'Created {count} tags to Establishment')) diff --git a/apps/tag/models.py b/apps/tag/models.py index a35425e8..29f18a23 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -37,6 +37,7 @@ class Tag(TranslatedFieldsMixin, models.Model): chosen_tag_settings = models.ManyToManyField(Country, through='ChosenTagSettings') priority = models.PositiveIntegerField(null=True, default=0) + # It does not make sense since in the old base another structure with duplicates old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) old_id_meta_product = models.PositiveIntegerField(_('old id metadata product'), diff --git a/project/settings/base.py b/project/settings/base.py index 70427ef3..4a2d0584 100644 --- a/project/settings/base.py +++ b/project/settings/base.py @@ -66,7 +66,7 @@ PROJECT_APPS = [ 'partner.apps.PartnerConfig', 'product.apps.ProductConfig', 'recipe.apps.RecipeConfig', - 'search_indexes.apps.SearchIndexesConfig', + # 'search_indexes.apps.SearchIndexesConfig', 'translation.apps.TranslationConfig', 'configuration.apps.ConfigurationConfig', 'timetable.apps.TimetableConfig', @@ -79,8 +79,8 @@ PROJECT_APPS = [ EXTERNAL_APPS = [ 'corsheaders', - 'django_elasticsearch_dsl', - 'django_elasticsearch_dsl_drf', + # 'django_elasticsearch_dsl', + # 'django_elasticsearch_dsl_drf', 'django_filters', 'drf_yasg', 'fcm_django', diff --git a/project/settings/local.py b/project/settings/local.py index f9a096fe..605a98c9 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -80,11 +80,11 @@ LOGGING = { 'py.warnings': { 'handlers': ['console'], }, - 'django.db.backends': { - 'handlers': ['console', ], - 'level': 'DEBUG', - 'propagate': False, - }, + # 'django.db.backends': { + # 'handlers': ['console', ], + # 'level': 'DEBUG', + # 'propagate': False, + # }, } }