added migrator, refactored migrators

apps/location/transfer_data.py

@@ -182,9 +182,9 @@ def update_flags():
 data_types = {
     "dictionaries": [
         transfer_countries,
-        # transfer_regions,
+        transfer_regions,
-        # transfer_cities,
+        transfer_cities,
-        # transfer_addresses,
+        transfer_addresses,
         transfer_wine_region,
         transfer_wine_sub_region,
         transfer_wine_village,

apps/product/models.py

@@ -112,30 +112,12 @@ class ProductQuerySet(models.QuerySet):
         return self.filter(subtypes__index_name=product_subtype)
 
 
-class ProductBrandQuerySet(models.QuerySet):
-    """QuerySet for model ProductBrand."""
-
-
-class ProductBrand(models.Model):
-
-    name = models.CharField(max_length=255, unique=True,
-                            verbose_name=_('product brand'))
-
-    objects = ProductBrandQuerySet.as_manager()
-
-    class Meta:
-        """Meta class."""
-        verbose_name = _('Brand')
-        verbose_name_plural = _('Brands')
-
-    def __str__(self):
-        """Overridden str dunder."""
-        return self.name
-
-
 class Product(TranslatedFieldsMixin, BaseAttributes):
     """Product models."""
 
+    EARLIEST_VINTAGE_YEAR = 1700
+    LATEST_VINTAGE_YEAR = 2100
+
     COMMON = 0
     ONLINE = 1
 
@@ -165,6 +147,7 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
                                      verbose_name=_('Country'))
     available = models.BooleanField(_('Available'), default=True)
     product_type = models.ForeignKey(ProductType, on_delete=models.PROTECT,
+                                     null=True,
                                      related_name='products', verbose_name=_('Type'))
     subtypes = models.ManyToManyField(ProductSubType, blank=True,
                                       related_name='products',
@@ -183,9 +166,13 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
                                         related_name='wines',
                                         blank=True, null=True, default=None,
                                         verbose_name=_('wine sub region'))
-    classifications = models.ManyToManyField('product.ProductClassification',
+    classifications = models.ManyToManyField('ProductClassification',
                                              blank=True,
                                              verbose_name=_('classifications'))
+    standards = models.ManyToManyField('ProductStandard',
+                                       blank=True,
+                                       verbose_name=_('standards'),
+                                       help_text=_('attribute from legacy db'))
     wine_village = models.ForeignKey('location.WineVillage', on_delete=models.PROTECT,
                                      blank=True, null=True,
                                      verbose_name=_('wine appellation'))
@@ -198,8 +185,6 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
                                         default=WAITING,
                                         verbose_name=_('state'),
                                         help_text=_('attribute from legacy db'))
-    brand = models.ForeignKey(ProductBrand, on_delete=models.PROTECT, null=True,
-                              verbose_name=_('brand'))
     tags = models.ManyToManyField('tag.Tag', related_name='products',
                                   verbose_name=_('Tag'))
     old_unique_key = models.CharField(max_length=255, unique=True,
@@ -207,8 +192,8 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
                                       help_text=_('attribute from legacy db'))
     vintage = models.IntegerField(verbose_name=_('vintage year'),
                                   null=True, blank=True, default=None,
-                                  validators=[MinValueValidator(1900),
+                                  validators=[MinValueValidator(EARLIEST_VINTAGE_YEAR),
-                                              MaxValueValidator(2100)])
+                                              MaxValueValidator(LATEST_VINTAGE_YEAR)])
 
     objects = ProductManager.from_queryset(ProductQuerySet)()
 

apps/product/transfer_data.py

@@ -139,27 +139,6 @@ def transfer_wine_classifications():
         pprint(f"transfer_wine_classifications errors: {serialized_data.errors}")
 
 
-def transfer_product_brand():
-    raw_queryset = transfer_models.Products.objects.raw(
-        """
-        SELECT
-           DISTINCT brand,
-           1 as id
-        FROM products
-        WHERE brand IS NOT NULL 
-        AND brand !='';
-        """
-    )
-    queryset = [vars(query) for query in raw_queryset]
-    serialized_data = product_serializers.ProductBrandSerializer(
-      data=queryset,
-      many=True)
-    if serialized_data.is_valid():
-        serialized_data.save()
-    else:
-        pprint(f"transfer_product_brand errors: {serialized_data.errors}")
-
-
 def transfer_product():
     queryset = transfer_models.Products.objects.all()
     serialized_data = product_serializers.ProductSerializer(
@@ -168,7 +147,22 @@ def transfer_product():
     if serialized_data.is_valid():
         serialized_data.save()
     else:
-        pprint(f"transfer_product errors: {serialized_data.errors}")
+        errors = []
+        for d in serialized_data.errors: errors.append(d) if d else None
+        pprint(f"transfer_product errors: {errors}")
+
+
+def transfer_plates():
+    queryset = transfer_models.Merchandise.objects.all()
+    serialized_data = product_serializers.PlateSerializer(
+      data=list(queryset.values()),
+      many=True)
+    if serialized_data.is_valid():
+        serialized_data.save()
+    else:
+        errors = []
+        for d in serialized_data.errors: errors.append(d) if d else None
+        pprint(f"transfer_plates errors: {errors}")
 
 
 data_types = {
@@ -186,7 +180,7 @@ data_types = {
         transfer_wine_classifications,
         ],
     "product": [
-        transfer_product_brand,
+        transfer_product,
-        transfer_product
+        transfer_plates,
     ],
 }

apps/transfer/management/commands/transfer.py

@@ -27,6 +27,7 @@ class Command(BaseCommand):
 
     LONG_DATA_TYPES = [
         'update_country_flag',
+        'product_type',
         'wine_characteristics',
         'product',
     ]

apps/transfer/mixins.py

@@ -3,7 +3,7 @@ from django.forms.models import model_to_dict
 from rest_framework import serializers
 from tag import models as tag_models
 from django.conf import settings
-from product.models import ProductType
+from product.models import ProductType, ProductSubType
 
 
 class SecondDbManager(models.Manager):
@@ -50,3 +50,44 @@ class TransferSerializerMixin(serializers.ModelSerializer):
                     'public': True
                 })
             return tag_category
+
+    def get_vintage_year(self, vintage):
+        earliest_year = self.Meta.model.EARLIEST_VINTAGE_YEAR
+        latest_year = self.Meta.model.LATEST_VINTAGE_YEAR
+        if vintage:
+            if vintage.isdigit():
+                if len(vintage) == 2:
+                    if vintage == '16':
+                        return 2016
+                elif len(vintage) == 4:
+                    if earliest_year < int(vintage) < latest_year:
+                        return int(vintage)
+                    elif vintage == '1584':
+                        return 1984
+                    elif vintage == '1017':
+                        return 2017
+                elif len(vintage) == 5:
+                    if vintage == '20115':
+                        return 2015
+                    elif vintage == '20174':
+                        return 2017
+            elif vintage.endswith('er'):
+                return self.get_vintage_year(vintage[:-2])
+
+    def get_product_type(self, product_type):
+        if isinstance(product_type, ProductType):
+            return product_type
+        if product_type:
+            qs = ProductType.objects.filter(
+                index_name__icontains=product_type)
+            if qs.exists():
+                return qs.first()
+
+    def get_product_sub_type(self, product_type, product_sub_type):
+        product_type = self.get_product_type(product_type)
+        if product_type and product_sub_type:
+            qs = ProductSubType.objects.filter(
+                product_type=product_type,
+                index_name__icontains=product_type)
+            if qs.exists():
+                return qs.first()

apps/transfer/models.py

@@ -1031,3 +1031,13 @@ class WineLocations(MigrateMixin):
     class Meta:
         managed = False
         db_table = 'wine_locations'
+
+
+class Merchandise(MigrateMixin):
+    using = 'legacy'
+
+    name = models.CharField(max_length=255)
+    vintage = models.CharField(max_length=255)
+    highlighted = models.CharField(max_length=255)
+    site = models.ForeignKey('Sites', models.DO_NOTHING)
+    attachment_suffix_url = models.CharField(max_length=255)

apps/transfer/serializers/product.py

@@ -65,7 +65,7 @@ class WineBottlesProducedSerializer(TransferSerializerMixin):
     CATEGORY_LABEL = 'Bottles produced'
     CATEGORY_INDEX_NAME = slugify(CATEGORY_LABEL)
 
-    bottles_produced = serializers.CharField()
+    bottles_produced = serializers.CharField(allow_null=True)
 
     class Meta:
         model = tag_models.Tag
@@ -87,7 +87,7 @@ class WineBottlesProducedSerializer(TransferSerializerMixin):
 
         parted = value.split(' ')
         if len(parted) > 1:
-            values = [int(i) if i.isdigit() else 0 for i in parted]
+            values = [int(i) for i in parted if i.isdigit()]
             return reduce(lambda a, b: a + b, values)
 
         lowered = value.lower()
@@ -225,21 +225,6 @@ class ProductClassificationSerializer(TransferSerializerMixin):
                 return qs.first()
 
 
-class ProductBrandSerializer(TransferSerializerMixin):
-
-    brand = serializers.CharField()
-
-    class Meta:
-        model = models.ProductBrand
-        fields = (
-            'brand',
-        )
-
-    def validate(self, attrs):
-        attrs['name'] = attrs.pop('brand')
-        return attrs
-
-
 class ProductSerializer(TransferSerializerMixin):
 
     establishment_id = serializers.PrimaryKeyRelatedField(
@@ -263,25 +248,29 @@ class ProductSerializer(TransferSerializerMixin):
     village_id = serializers.PrimaryKeyRelatedField(
         queryset=transfer_models.WineLocations.objects.all(),
         allow_null=True)
-    vineyard_id = serializers.PrimaryKeyRelatedField(
+    # same as wine_village
-        queryset=transfer_models.WineLocations.objects.all(),
+    # vineyard_id = serializers.PrimaryKeyRelatedField(
-        allow_null=True)
+    #     queryset=transfer_models.WineLocations.objects.all(),
+    #     allow_null=True)
     wine_quality_id = serializers.PrimaryKeyRelatedField(
         queryset=transfer_models.ProductClassification.objects.all(),
         allow_null=True)
 
-    brand = serializers.CharField(allow_null=True)
+    # same as establishment name
-    name = serializers.CharField(allow_null=True)
+    id = serializers.IntegerField()
+    brand = serializers.CharField(allow_null=True, allow_blank=True)
+    name = serializers.CharField(allow_null=True, allow_blank=True)
     vintage = serializers.CharField(allow_null=True)
     type = serializers.CharField(allow_null=True)
     wine_sub_region_id = serializers.CharField(allow_null=True)
     state = serializers.CharField()
-    bottles_produced = serializers.CharField(allow_null=True)
+    bottles_produced = serializers.CharField(allow_null=True, allow_blank=True)
     unique_key = serializers.CharField(allow_null=True)
 
     class Meta:
         model = models.Product
         fields = (
+            'id',
             'establishment_id',  # done
             'classification_id',  # done
             'wine_region_id',  # done
@@ -289,9 +278,9 @@ class ProductSerializer(TransferSerializerMixin):
             'wine_color_id',  # done
             'appellation_id',  # done
             'village_id',  # done
-            # 'vineyard_id',  duplicate wine_village
+            # 'vineyard_id',
             'wine_quality_id',  # done
-            'brand',  # done
+            'brand',
             'name',  # done
             'vintage',  # done
             'type',  # done
@@ -302,11 +291,11 @@ class ProductSerializer(TransferSerializerMixin):
         )
 
     def validate(self, attrs):
+        name = attrs.pop('name', None)
         establishment = attrs.pop('establishment_id', None)
-        state = attrs.pop('state', None)
         brand = attrs.pop('brand', None)
         classification = attrs.pop('classification_id', None)
-        vintage = attrs.pop('vintage_year', None)
+        vintage = attrs.pop('vintage', None)
         product_type = attrs.pop('type')
         wine_region = attrs.pop('wine_region_id', None)
         wine_sub_region_id = attrs.pop('wine_sub_region_id', None)
@@ -316,14 +305,19 @@ class ProductSerializer(TransferSerializerMixin):
         appellation = attrs.pop('appellation_id', None)
         village = attrs.pop('village_id', None)
         wine_quality = attrs.pop('wine_quality_id', None)
+        name = self.get_name(name, brand)
+        old_id = attrs.pop('id')
+        state = self.get_state(attrs.pop('state', None))
 
+        attrs['old_id'] = old_id
+        attrs['name'] = name
         attrs['old_unique_key'] = attrs.pop('unique_key')
         attrs['establishment'] = self.get_establishment(establishment)
-        attrs['state'] = self.get_state(state)
+        attrs['state'] = state
-        attrs['brand'] = self.get_brand(brand)
+        # attrs['brand'] = self.get_brand(brand)
         attrs['wine_classification'] = self.get_wine_classification(classification)
-        attrs['wine_quiality'] = self.get_wine_standard(classification,
+        attrs['wine_quality'] = self.get_wine_standard(wine_quality,
-                                                        models.ProductStandard.WINEQUALITY)
+                                                       models.ProductStandard.WINEQUALITY)
         attrs['wine_appellation'] = self.get_wine_standard(appellation,
                                                            models.ProductStandard.APPELLATION)
         attrs['product_type'] = self.get_product_type(product_type)
@@ -337,8 +331,41 @@ class ProductSerializer(TransferSerializerMixin):
         attrs['wine_color_tag'] = self.get_tag(wine_color,
                                                WineColorSerializer.CATEGORY_INDEX_NAME)
         attrs['wine_village'] = self.get_wine_village(village)
+        attrs['available'] = self.get_availability(state)
+        attrs['slug'] = self.get_slug(name, old_id)
         return attrs
 
+    def create(self, validated_data):
+        qs = self.Meta.model.objects.filter(old_id=validated_data.get('old_id'))
+        # classifications
+        classifications = [validated_data.pop('wine_classification', None)]
+        # standards
+        standards = [validated_data.pop('wine_quality', None),
+                     validated_data.pop('wine_appellation', None)]
+        # tags
+        tags = [validated_data.pop('bottles_produced_tag', None),
+                validated_data.pop('wine_type_tag', None),
+                validated_data.pop('wine_color_tag', None)]
+
+        if not qs.exists():
+            obj = super().create(validated_data)
+        else:
+            obj = qs.first()
+
+        # adding classification
+        obj.classifications.add(*[i for i in classifications if i])
+        # adding standard
+        obj.standards.add(*[i for i in standards if i])
+        # adding tags
+        obj.tags.add(*[i for i in tags if i])
+        return obj
+
+    def get_name(self, name, brand):
+        if name:
+            return name
+        if brand and not name:
+            return brand
+
     def get_establishment(self, establishment):
         if establishment:
             qs = establishment_models.Establishment.objects.filter(
@@ -353,12 +380,6 @@ class ProductSerializer(TransferSerializerMixin):
             return models.Product.OUT_OF_PRODUCTION
         return models.Product.WAITING
 
-    def get_brand(self, brand):
-        if brand:
-            qs = models.ProductBrand.objects.filter(brand__icontains=brand)
-            if qs.exists():
-                return qs.first()
-
     def get_wine_classification(self, classification):
         if classification:
             classification_qs = models.ProductClassification.objects.filter(
@@ -366,33 +387,6 @@ class ProductSerializer(TransferSerializerMixin):
             if classification_qs.exists():
                 return classification_qs.first()
 
-    def get_vintage_year(self, vintage):
-        if vintage.isdigit():
-            if len(vintage) == 2:
-                if vintage == '16':
-                    return 2016
-            elif len(vintage) == 4:
-                if 1900 < int(vintage) < 2100:
-                    return int(vintage)
-                elif vintage == '1584':
-                    return 1984
-                elif vintage == '1017':
-                    return 2017
-            elif len(vintage) == 5:
-                if vintage == '20115':
-                    return 2015
-                elif vintage == '20174':
-                    return 2017
-        elif vintage.endswith('er'):
-            return self.get_vintage_year(vintage[:-2])
-
-    def get_product_type_tag(self, product_type):
-        if product_type:
-            qs = models.ProductType.objects.filter(
-                index_name__icontains=product_type)
-            if qs.exists():
-                return qs.first()
-
     def get_wine_region(self, wine_region):
         if wine_region:
             wine_region_qs = location_models.WineRegion.objects.filter(
@@ -425,9 +419,59 @@ class ProductSerializer(TransferSerializerMixin):
 
     def get_wine_standard(self, standard, standard_type):
         if standard:
-            standard = standard.id if hasattr(standard, 'id') else standard
+            if isinstance(standard, transfer_models.ProductClassification):
+                standard = standard.id
             standard_qs = models.ProductStandard.objects.filter(
                 standard_type=standard_type,
-                old_id=standard.id)
+                old_id=standard)
             if standard_qs.exists():
-                return standard_qs.first()
+                return standard_qs.first()
+
+    def get_availability(self, state: int):
+        if state == models.Product.PUBLISHED:
+            return True
+        return False
+
+    def get_slug(self, name, old_id):
+        return slugify(f'{name}-{old_id}')
+
+
+class PlateSerializer(TransferSerializerMixin):
+
+    PRODUCT_TYPE_INDEX_NAME = 'souvenir'
+    PRODUCT_SUB_TYPE_INDEX_NAME = 'plate'
+
+    id = serializers.IntegerField()
+    name = serializers.CharField()
+    vintage = serializers.CharField()
+
+    class Meta(ProductSerializer.Meta):
+        fields = (
+            'id',
+            'name',
+            'vintage',
+        )
+
+    def validate(self, attrs):
+        product_type = self.get_product_type(self.PRODUCT_TYPE_INDEX_NAME)
+
+        attrs['old_id'] = attrs.pop('id')
+        attrs['vintage'] = self.get_vintage_year(attrs.pop('vintage'))
+        attrs['product_type'] = product_type
+        attrs['subtype'] = self.get_product_sub_type(product_type,
+                                                     self.PRODUCT_SUB_TYPE_INDEX_NAME)
+        return attrs
+
+    def create(self, validated_data):
+        qs = self.Meta.model.objects.filter(old_id=validated_data.get('old_id'))
+        # subtypes
+        subtypes = [validated_data.pop('subtype', None)]
+
+        if not qs.exists():
+            obj = super().create(validated_data)
+        else:
+            obj = qs.first()
+
+        # adding classification
+        obj.subtypes.add(*[i for i in subtypes if i])
+        return obj