add new migrations for tags

apps/news/views.py

@@ -1,11 +1,8 @@
 """News app views."""
 from django.conf import settings
-from django.db.transaction import on_commit
 from django.shortcuts import get_object_or_404
-from rest_framework import generics, permissions, status
+from rest_framework import generics, permissions
-from rest_framework.response import Response
 
-from gallery.tasks import delete_image
 from news import filters, models, serializers
 from rating.tasks import add_rating
 from utils.permissions import IsCountryAdmin, IsContentPageManager

apps/search_indexes/__init__.py

@@ -1 +1 @@
-from search_indexes.signals import update_document
+# from search_indexes.signals import update_document

apps/search_indexes/signals.py

@@ -1,67 +1,67 @@
-"""Search indexes app signals."""
+# """Search indexes app signals."""
-from django.db.models.signals import post_save
+# from django.db.models.signals import post_save
-from django.dispatch import receiver
+# from django.dispatch import receiver
-from django_elasticsearch_dsl.registries import registry
+# from django_elasticsearch_dsl.registries import registry
-
+#
-
+#
-@receiver(post_save)
+# @receiver(post_save)
-def update_document(sender, **kwargs):
+# def update_document(sender, **kwargs):
-    from establishment.models import Establishment
+#     from establishment.models import Establishment
-    app_label = sender._meta.app_label
+#     app_label = sender._meta.app_label
-    model_name = sender._meta.model_name
+#     model_name = sender._meta.model_name
-    instance = kwargs['instance']
+#     instance = kwargs['instance']
-
+#
-    app_label_model_name_to_filter = {
+#     app_label_model_name_to_filter = {
-        ('location','country'): 'address__city__country',
+#         ('location','country'): 'address__city__country',
-        ('location','city'): 'address__city',
+#         ('location','city'): 'address__city',
-        ('location', 'address'): 'address',
+#         ('location', 'address'): 'address',
-        # todo: remove after migration
+#         # todo: remove after migration
-        ('establishment', 'establishmenttype'): 'establishment_type',
+#         ('establishment', 'establishmenttype'): 'establishment_type',
-        ('establishment', 'establishmentsubtype'): 'establishment_subtypes',
+#         ('establishment', 'establishmentsubtype'): 'establishment_subtypes',
-        ('tag', 'tag'): 'tags',
+#         ('tag', 'tag'): 'tags',
-    }
+#     }
-    filter_name = app_label_model_name_to_filter.get((app_label, model_name))
+#     filter_name = app_label_model_name_to_filter.get((app_label, model_name))
-    if filter_name:
+#     if filter_name:
-        qs = Establishment.objects.filter(**{filter_name: instance})
+#         qs = Establishment.objects.filter(**{filter_name: instance})
-        for product in qs:
+#         for product in qs:
-            registry.update(product)
+#             registry.update(product)
-
+#
-
+#
-@receiver(post_save)
+# @receiver(post_save)
-def update_news(sender, **kwargs):
+# def update_news(sender, **kwargs):
-    from news.models import News
+#     from news.models import News
-    app_label = sender._meta.app_label
+#     app_label = sender._meta.app_label
-    model_name = sender._meta.model_name
+#     model_name = sender._meta.model_name
-    instance = kwargs['instance']
+#     instance = kwargs['instance']
-    app_label_model_name_to_filter = {
+#     app_label_model_name_to_filter = {
-        ('location','country'): 'country',
+#         ('location','country'): 'country',
-        ('news','newstype'): 'news_type',
+#         ('news','newstype'): 'news_type',
-        ('tag', 'tag'): 'tags',
+#         ('tag', 'tag'): 'tags',
-    }
+#     }
-    filter_name = app_label_model_name_to_filter.get((app_label, model_name))
+#     filter_name = app_label_model_name_to_filter.get((app_label, model_name))
-    if filter_name:
+#     if filter_name:
-        qs = News.objects.filter(**{filter_name: instance})
+#         qs = News.objects.filter(**{filter_name: instance})
-        for product in qs:
+#         for product in qs:
-            registry.update(product)
+#             registry.update(product)
-
+#
-
+#
-@receiver(post_save)
+# @receiver(post_save)
-def update_product(sender, **kwargs):
+# def update_product(sender, **kwargs):
-    from product.models import Product
+#     from product.models import Product
-    app_label = sender._meta.app_label
+#     app_label = sender._meta.app_label
-    model_name = sender._meta.model_name
+#     model_name = sender._meta.model_name
-    instance = kwargs['instance']
+#     instance = kwargs['instance']
-    app_label_model_name_to_filter = {
+#     app_label_model_name_to_filter = {
-        ('product','productstandard'): 'standards',
+#         ('product','productstandard'): 'standards',
-        ('product', 'producttype'): 'product_type',
+#         ('product', 'producttype'): 'product_type',
-        ('tag','tag'): 'tags',
+#         ('tag','tag'): 'tags',
-        ('location', 'wineregion'): 'wine_region',
+#         ('location', 'wineregion'): 'wine_region',
-        ('location', 'winesubregion'): 'wine_sub_region',
+#         ('location', 'winesubregion'): 'wine_sub_region',
-        ('location', 'winevillage'): 'wine_village',
+#         ('location', 'winevillage'): 'wine_village',
-        ('establishment', 'establishment'): 'establishment',
+#         ('establishment', 'establishment'): 'establishment',
-    }
+#     }
-    filter_name = app_label_model_name_to_filter.get((app_label, model_name))
+#     filter_name = app_label_model_name_to_filter.get((app_label, model_name))
-    if filter_name:
+#     if filter_name:
-        qs = Product.objects.filter(**{filter_name: instance})
+#         qs = Product.objects.filter(**{filter_name: instance})
-        for product in qs:
+#         for product in qs:
-            registry.update(product)
+#             registry.update(product)

apps/tag/management/commands/add_tags.py

@@ -3,87 +3,111 @@ from django.core.management.base import BaseCommand
 from establishment.models import Establishment, EstablishmentType
 from transfer import models as legacy
 from tag.models import Tag, TagCategory
+from tqdm import tqdm
+from django.db import connections
+from collections import namedtuple
+
+
+def namedtuplefetchall(cursor):
+    "Return all rows from a cursor as a namedtuple"
+    desc = cursor.description
+    nt_result = namedtuple('Result', [col[0] for col in desc])
+    return [nt_result(*row) for row in cursor.fetchall()]
+
+
+def metadata_category_sql():
+    with connections['legacy'].cursor() as cursor:
+        cursor.execute(
+            '''SELECT
+                                `key`,
+                                establishments.type,
+                                key_value_metadata.`value_type`,
+                                public,
+                                key_value_metadata.id as 'old_id'
+                            FROM metadata
+                                LEFT JOIN establishments
+                                          ON metadata.establishment_id=establishments.id
+                                LEFT JOIN key_value_metadata
+                                          ON metadata.key=key_value_metadata.key_name
+                            GROUP BY
+                                establishments.type,
+                                `key`,
+                                key_value_metadata.`value_type`,
+                                public, old_id;'''
+        )
+        return namedtuplefetchall(cursor)
+
+
+def metadata_tags_sql():
+    with connections['legacy'].cursor() as cursor:
+        cursor.execute(
+            """
+            SELECT
+    value,
+    `key` as category,
+    establishment_id
+FROM metadata
+WHERE establishment_id is not null"""
+        )
+        return namedtuplefetchall(cursor)
 
 
 class Command(BaseCommand):
     help = 'Add tags values from old db to new db'
 
-    def handle(self, *args, **kwargs):
+    def get_type(self, meta):
+        meta_type = meta.value_type
+        if not meta.value_type:
+            if meta.key == 'wineyard_visits':
+                meta_type = 'list'
+            elif meta.key in ['private_room', 'outside_sits']:
+                meta_type = 'bool'
+        return meta_type
 
+    def handle(self, *args, **kwargs):
         existing_establishment = Establishment.objects.filter(
             old_id__isnull=False
         )
-        ESTABLISHMENT = 1
-        SHOP = 2
-        RESTAURANT = 3
-        WINEYARD = 4
-
         MAPPER = {
-            RESTAURANT: EstablishmentType.RESTAURANT,
+            'Restaurant': EstablishmentType.RESTAURANT,
-            WINEYARD: EstablishmentType.PRODUCER,
+            'Wineyard': EstablishmentType.PRODUCER,
-            SHOP: EstablishmentType.ARTISAN
+            'Shop': EstablishmentType.ARTISAN
         }
+        # remove old black category
+        for establishment_tag in tqdm(EstablishmentType.objects.all()):
+                establishment_tag.tag_categories.remove(*list(
+                    establishment_tag.tag_categories.exclude(
+                        tags__establishments__isnull=False).distinct()))
 
-        mapper_values_meta = legacy.KeyValueMetadatumKeyValueMetadatumEstablishments.objects.all()
+        # created Tag Category
-        for key, value in MAPPER.items():
+        for meta in tqdm(metadata_category_sql()):
-            values_meta_id_list = mapper_values_meta.filter(
+            category, _ = TagCategory.objects.update_or_create(
-                key_value_metadatum_establishment_id=key
+                index_name=meta.key,
-            ).values_list('key_value_metadatum_id')
+                defaults={
+                    "public": True if meta.public == 1 else False,
+                    "value_type": self.get_type(meta),
+                    "label": {"en-GB": meta.key}
+                }
+            )
 
-            est_type, _ = EstablishmentType.objects.get_or_create(index_name=value)
+            # add to EstablishmentType
+            est_type = EstablishmentType.objects.get(index_name=MAPPER[meta.type])
+            if category not in est_type.tag_categories.all():
+                est_type.tag_categories.add(category)
 
-            key_value_metadata = legacy.KeyValueMetadata.objects.filter(
+        count = 0
-                id__in=values_meta_id_list)
+        for meta_tag in tqdm(metadata_tags_sql()):
 
-            # create TagCategory
+            tag, _ = Tag.objects.update_or_create(
-            for key_value in key_value_metadata:
+                category=TagCategory.objects.get(index_name=meta_tag.category),
-                tag_category, created = TagCategory.objects.get_or_create(
+                value=meta_tag.value,
-                    index_name=key_value.key_name,
+                defaults={
-                )
+                    "label": {"en-GB": meta_tag.value}
-
+                }
-                if created:
+            )
-                    tag_category.label = {
+            establishment = existing_establishment.filter(old_id=meta_tag.establishment_id).first()
-                                             'en-GB': key_value.key_name,
+            if establishment:
-                                             'fr-FR': key_value.key_name,
+                if tag not in establishment.tags.all():
-                                             'ru-RU': key_value.key_name,
+                    establishment.tags.add(tag)
-                                         }
+                    count += 1
-                    tag_category.value_type = key_value.value_type
+        self.stdout.write(self.style.WARNING(f'Created {count} tags to Establishment'))
-                    tag_category.save()
-                    est_type.tag_categories.add(
-                        tag_category
-                    )
-
-                # create Tag
-                for tag in key_value.metadata_set.filter(
-                        establishment__id__in=list(
-                            existing_establishment.values_list('old_id', flat=True)
-                        )):
-
-                    new_tag, created = Tag.objects.get_or_create(
-                        value=tag.value,
-                        category=tag_category,
-                    )
-                    if created:
-
-                        sp = tag.value.split('_')
-                        value = ' '.join([sp[0].capitalize()] + sp[1:])
-
-                        trans = {
-                               'en-GB': value,
-                               'fr-FR': value,
-                               'ru-RU': value,
-                               }
-
-                        aliases = legacy.MetadatumAliases.objects.filter(value=tag.value)
-
-                        for alias in aliases:
-                            trans[alias.locale] = alias.meta_alias
-
-                        new_tag.label = trans
-                        new_tag.save()
-
-                    est = existing_establishment.filter(
-                        old_id=tag.establishment_id).first()
-                    if est:
-                        est.tags.add(new_tag)
-                        est.save()

apps/tag/models.py

@@ -37,6 +37,7 @@ class Tag(TranslatedFieldsMixin, models.Model):
     chosen_tag_settings = models.ManyToManyField(Country, through='ChosenTagSettings')
     priority = models.PositiveIntegerField(null=True, default=0)
 
+    # It does not make sense since in the old base another structure with duplicates
     old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None)
 
     old_id_meta_product = models.PositiveIntegerField(_('old id metadata product'),

project/settings/base.py

@@ -66,7 +66,7 @@ PROJECT_APPS = [
     'partner.apps.PartnerConfig',
     'product.apps.ProductConfig',
     'recipe.apps.RecipeConfig',
-    'search_indexes.apps.SearchIndexesConfig',
+    # 'search_indexes.apps.SearchIndexesConfig',
     'translation.apps.TranslationConfig',
     'configuration.apps.ConfigurationConfig',
     'timetable.apps.TimetableConfig',
@@ -79,8 +79,8 @@ PROJECT_APPS = [
 
 EXTERNAL_APPS = [
     'corsheaders',
-    'django_elasticsearch_dsl',
+    # 'django_elasticsearch_dsl',
-    'django_elasticsearch_dsl_drf',
+    # 'django_elasticsearch_dsl_drf',
     'django_filters',
     'drf_yasg',
     'fcm_django',

project/settings/local.py

@@ -80,11 +80,11 @@ LOGGING = {
         'py.warnings': {
             'handlers': ['console'],
         },
-        'django.db.backends': {
+        # 'django.db.backends': {
-            'handlers': ['console', ],
+        #     'handlers': ['console', ],
-            'level': 'DEBUG',
+        #     'level': 'DEBUG',
-            'propagate': False,
+        #     'propagate': False,
-        },
+        # },
     }
 }