From 7d9b6d1f30ce2d694385447381a307a384dad2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Tue, 12 Nov 2019 15:01:12 +0300 Subject: [PATCH 1/4] Add category tag for products --- .../account/management/commands/add_social.py | 1 - .../management/commands/add_product_tag.py | 38 +++++++++++++++++++ apps/tag/models.py | 2 + requirements/base.txt | 3 ++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 apps/product/management/commands/add_product_tag.py diff --git a/apps/account/management/commands/add_social.py b/apps/account/management/commands/add_social.py index ecf86d60..c9b056f5 100644 --- a/apps/account/management/commands/add_social.py +++ b/apps/account/management/commands/add_social.py @@ -1,6 +1,5 @@ from django.core.management.base import BaseCommand from django.db import connections -from django.db.models import Q from social_django.models import UserSocialAuth from establishment.management.commands.add_position import namedtuplefetchall from account.models import User diff --git a/apps/product/management/commands/add_product_tag.py b/apps/product/management/commands/add_product_tag.py new file mode 100644 index 00000000..9e63d213 --- /dev/null +++ b/apps/product/management/commands/add_product_tag.py @@ -0,0 +1,38 @@ +from django.core.management.base import BaseCommand +from django.db import connections +from social_django.models import UserSocialAuth +from establishment.management.commands.add_position import namedtuplefetchall +from account.models import User +from tag.models import Tag, TagCategory +from tqdm import tqdm + +class Command(BaseCommand): + help = '''Add add product tags networks from old db to new db. + Run after add_product!!!''' + + def category_sql(self): + with connections['legacy'].cursor() as cursor: + cursor.execute(''' + select DISTINCT + id, + key_name as category, + value_type as value_type + from product_key_value_metadata + ''') + return namedtuplefetchall(cursor) + + def add_category_tag(self): + objects = [] + for c in tqdm(self.category_sql()): + cat = TagCategory.objects.filter(index_name=c.category, + value_type=c.value_type, + label={'en-GB':c.category}) + if not cat.exists(): + objects.append(TagCategory(index_name=c.category, + value_type=c.value_type, + label={'en-GB': c.category})) + TagCategory.objects.bulk_create(objects) + self.stdout.write(self.style.WARNING(f'Created tag category.')) + + def handle(self, *args, **kwargs): + self.add_category_tag() \ No newline at end of file diff --git a/apps/tag/models.py b/apps/tag/models.py index 5965b4fb..59e2339c 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -107,11 +107,13 @@ class TagCategory(TranslatedFieldsMixin, models.Model): STRING = 'string' LIST = 'list' INTEGER = 'integer' + FLOAT = 'float' VALUE_TYPE_CHOICES = ( (STRING, _('string')), (LIST, _('list')), (INTEGER, _('integer')), + (FLOAT, _('float')) ) label = TJSONField(blank=True, null=True, default=None, diff --git a/requirements/base.txt b/requirements/base.txt index a1408fc5..d7050a29 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -21,6 +21,9 @@ pycountry==19.8.18 django-phonenumber-field[phonenumbers]==2.1.0 django-timezone-field==3.1 +# custom +tqdm==4.38.0 + # auth socials django-rest-framework-social-oauth2==1.1.0 From 27bef663982742703ce9ed6f64cb7ddcc06d49e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Tue, 12 Nov 2019 15:02:20 +0300 Subject: [PATCH 2/4] Add old_id for category --- apps/tag/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/tag/models.py b/apps/tag/models.py index 59e2339c..d68479e1 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -128,6 +128,7 @@ class TagCategory(TranslatedFieldsMixin, models.Model): value_type = models.CharField(_('value type'), max_length=255, choices=VALUE_TYPE_CHOICES, default=LIST, ) + old_id = models.IntegerField(blank=True, null=True) objects = TagCategoryQuerySet.as_manager() From 13db34237b01574c817e0a854176c5a4bac8283b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Tue, 12 Nov 2019 17:36:48 +0300 Subject: [PATCH 3/4] Add tag for product --- .../management/commands/add_product_tag.py | 98 +++++++++++++++---- .../tag/migrations/0011_auto_20191112_1203.py | 23 +++++ apps/tag/models.py | 1 + project/settings/local.py | 2 +- 4 files changed, 106 insertions(+), 18 deletions(-) create mode 100644 apps/tag/migrations/0011_auto_20191112_1203.py diff --git a/apps/product/management/commands/add_product_tag.py b/apps/product/management/commands/add_product_tag.py index 9e63d213..a0f56b71 100644 --- a/apps/product/management/commands/add_product_tag.py +++ b/apps/product/management/commands/add_product_tag.py @@ -1,10 +1,11 @@ from django.core.management.base import BaseCommand from django.db import connections -from social_django.models import UserSocialAuth from establishment.management.commands.add_position import namedtuplefetchall -from account.models import User from tag.models import Tag, TagCategory +from product.models import Product from tqdm import tqdm +from django.db.models.functions import Lower + class Command(BaseCommand): help = '''Add add product tags networks from old db to new db. @@ -13,26 +14,89 @@ class Command(BaseCommand): def category_sql(self): with connections['legacy'].cursor() as cursor: cursor.execute(''' - select DISTINCT - id, - key_name as category, - value_type as value_type - from product_key_value_metadata + select + DISTINCT + trim(CONVERT(v.key_name USING utf8)) as category, + trim(CONVERT(v.value_type USING utf8)) as value_type + FROM product_metadata m + JOIN product_key_value_metadata v on v.id = m.product_key_value_metadatum_id + ''') return namedtuplefetchall(cursor) def add_category_tag(self): objects = [] - for c in tqdm(self.category_sql()): - cat = TagCategory.objects.filter(index_name=c.category, - value_type=c.value_type, - label={'en-GB':c.category}) - if not cat.exists(): - objects.append(TagCategory(index_name=c.category, - value_type=c.value_type, - label={'en-GB': c.category})) + for c in tqdm(self.category_sql(), desc='Add category tags'): + categories = TagCategory.objects.filter(index_name=c.category, + value_type=c.value_type + ) + if not categories.exists(): + objects.append( + TagCategory(label={"en-GB": c.category}, + value_type=c.value_type, + index_name=c.category + ) + ) TagCategory.objects.bulk_create(objects) - self.stdout.write(self.style.WARNING(f'Created tag category.')) + self.stdout.write(self.style.WARNING(f'Add or get tag category objects.')) + + def tag_sql(self): + with connections['legacy'].cursor() as cursor: + cursor.execute(''' + select + DISTINCT + trim(CONVERT(m.value USING utf8)) as tag_value, + trim(CONVERT(v.key_name USING utf8)) as tag_category + FROM product_metadata m + join product_key_value_metadata v on v.id = m.product_key_value_metadatum_id + ''') + return namedtuplefetchall(cursor) + + def add_tag(self): + objects = [] + for t in tqdm(self.tag_sql(), desc='Add tags'): + category = TagCategory.objects.get(index_name=t.tag_category) + + tag = Tag.objects.filter(label={"en-GB": t.tag_value}, + category=category, + value=t.tag_value + ) + + if not tag.exists(): + objects.append(Tag(label={"en-GB": t.tag_value}, + category=category, + value=t.tag_value + )) + Tag.objects.bulk_create(objects) + self.stdout.write(self.style.WARNING(f'Add or get tag objects.')) + + def product_sql(self): + with connections['legacy'].cursor() as cursor: + cursor.execute(''' + select + DISTINCT + m.product_id, + lower(trim(CONVERT(m.value USING utf8))) as tag_value, + trim(CONVERT(v.key_name USING utf8)) as tag_category + FROM product_metadata m + JOIN product_key_value_metadata v on v.id = m.product_key_value_metadatum_id + ''') + return namedtuplefetchall(cursor) + + def add_product_tag(self): + objects = [] + for t in tqdm(self.product_sql(), desc='Add product tag'): + category = TagCategory.objects.get(index_name=t.tag_category) + tag = Tag.objects.annotate(lower_value=Lower('value')) + tag.filter(lower_value=t.tag_value, category=category) + products = Product.objects.filter(old_id=t.product_id) + if products.exists(): + products.tags.add(tag) + products.save() + + self.stdout.write(self.style.WARNING(f'Add or get tag objects.')) def handle(self, *args, **kwargs): - self.add_category_tag() \ No newline at end of file + self.add_category_tag() + self.add_tag() + self.add_product_tag() \ No newline at end of file diff --git a/apps/tag/migrations/0011_auto_20191112_1203.py b/apps/tag/migrations/0011_auto_20191112_1203.py new file mode 100644 index 00000000..498eccda --- /dev/null +++ b/apps/tag/migrations/0011_auto_20191112_1203.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.7 on 2019-11-12 12:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tag', '0010_auto_20191112_0104'), + ] + + operations = [ + migrations.AddField( + model_name='tagcategory', + name='old_id', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='tagcategory', + name='value_type', + field=models.CharField(choices=[('string', 'string'), ('list', 'list'), ('integer', 'integer'), ('float', 'float')], default='list', max_length=255, verbose_name='value type'), + ), + ] diff --git a/apps/tag/models.py b/apps/tag/models.py index d68479e1..4262b446 100644 --- a/apps/tag/models.py +++ b/apps/tag/models.py @@ -36,6 +36,7 @@ class Tag(TranslatedFieldsMixin, models.Model): verbose_name=_('Category')) chosen_tag_settings = models.ManyToManyField(Country, through='ChosenTagSettings') priority = models.PositiveIntegerField(null=True, default=0) + old_id = models.PositiveIntegerField(_('old id'), blank=True, null=True, default=None) objects = TagQuerySet.as_manager() diff --git a/project/settings/local.py b/project/settings/local.py index 62298faa..6745d272 100644 --- a/project/settings/local.py +++ b/project/settings/local.py @@ -36,7 +36,7 @@ THUMBNAIL_DEBUG = True # ADDED TRANSFER APP -INSTALLED_APPS.append('transfer.apps.TransferConfig') +# INSTALLED_APPS.append('transfer.apps.TransferConfig') # DATABASES From 89fa7c5d569cf4cfc92c86adbb00baca2e26ef68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B8=D0=BA=D1=82=D0=BE=D1=80=20=D0=93=D0=BB=D0=B0?= =?UTF-8?q?=D0=B4=D0=BA=D0=B8=D1=85?= Date: Tue, 12 Nov 2019 18:37:43 +0300 Subject: [PATCH 4/4] Fix for merge --- .../management/commands/update_employee.py | 10 +++------- .../management/commands/add_product_tag.py | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/apps/establishment/management/commands/update_employee.py b/apps/establishment/management/commands/update_employee.py index dfe8b5a0..41831f0d 100644 --- a/apps/establishment/management/commands/update_employee.py +++ b/apps/establishment/management/commands/update_employee.py @@ -2,8 +2,7 @@ from django.core.management.base import BaseCommand from django.db import connections from establishment.management.commands.add_position import namedtuplefetchall from establishment.models import Employee -from django.db.models import Q - +from tqdm import tqdm class Command(BaseCommand): help = 'Add employee from old db to new db.' @@ -28,10 +27,7 @@ class Command(BaseCommand): def handle(self, *args, **options): objects = [] - for e in self.employees_sql(): - empl = Employee.objects.filter(old_id=e.profile_id) - if empl.exists(): - empl.name = e.name - empl.save() + for e in tqdm(self.employees_sql()): + empl = Employee.objects.filter(old_id=e.profile_id).update(name=e.name) self.stdout.write(self.style.WARNING(f'Update employee name objects.')) diff --git a/apps/product/management/commands/add_product_tag.py b/apps/product/management/commands/add_product_tag.py index a0f56b71..5633230c 100644 --- a/apps/product/management/commands/add_product_tag.py +++ b/apps/product/management/commands/add_product_tag.py @@ -57,7 +57,7 @@ class Command(BaseCommand): for t in tqdm(self.tag_sql(), desc='Add tags'): category = TagCategory.objects.get(index_name=t.tag_category) - tag = Tag.objects.filter(label={"en-GB": t.tag_value}, + tag = Tag.objects.filter( category=category, value=t.tag_value ) @@ -96,7 +96,21 @@ class Command(BaseCommand): self.stdout.write(self.style.WARNING(f'Add or get tag objects.')) + def check_tag(self): + tags = Tag.objects.filter(value__contains='_').all() + for tag in tqdm(tags, desc='Check label for tag'): + new_label = {} + for k, v in tag.label.items(): + if isinstance(v, str) and '_' in v: + sp = v.split('_') + v = ' '.join([sp[0].capitalize()] + sp[1:]) + new_label[k] = v + tag.label = new_label + tag.save() + + def handle(self, *args, **kwargs): self.add_category_tag() self.add_tag() - self.add_product_tag() \ No newline at end of file + self.add_product_tag() + self.check_tag() \ No newline at end of file