phzhik/gault-millau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added migrator, refactored migrators

Browse Source
This commit is contained in:
Anatoly 2019-11-09 22:58:34 +03:00
parent 0f2d1ca071
commit efb468f00d
7 changed files with 196 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/location/transfer_data.py
View File

@ -182,9 +182,9 @@ def update_flags():
data_types = {
"dictionaries": [
transfer_countries,
# transfer_regions,
# transfer_cities,
# transfer_addresses,
transfer_regions,
transfer_cities,
transfer_addresses,
transfer_wine_region,
transfer_wine_sub_region,
transfer_wine_village,

37
apps/product/models.py
View File

@ -112,30 +112,12 @@ class ProductQuerySet(models.QuerySet):
return self.filter(subtypes__index_name=product_subtype)
class ProductBrandQuerySet(models.QuerySet):
"""QuerySet for model ProductBrand."""
class ProductBrand(models.Model):
name = models.CharField(max_length=255, unique=True,
verbose_name=_('product brand'))
objects = ProductBrandQuerySet.as_manager()
class Meta:
"""Meta class."""
verbose_name = _('Brand')
verbose_name_plural = _('Brands')
def __str__(self):
"""Overridden str dunder."""
return self.name
class Product(TranslatedFieldsMixin, BaseAttributes):
"""Product models."""
EARLIEST_VINTAGE_YEAR = 1700
LATEST_VINTAGE_YEAR = 2100
COMMON = 0
ONLINE = 1
@ -165,6 +147,7 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
verbose_name=_('Country'))
available = models.BooleanField(_('Available'), default=True)
product_type = models.ForeignKey(ProductType, on_delete=models.PROTECT,
null=True,
related_name='products', verbose_name=_('Type'))
subtypes = models.ManyToManyField(ProductSubType, blank=True,
related_name='products',
@ -183,9 +166,13 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
related_name='wines',
blank=True, null=True, default=None,
verbose_name=_('wine sub region'))
classifications = models.ManyToManyField('product.ProductClassification',
classifications = models.ManyToManyField('ProductClassification',
blank=True,
verbose_name=_('classifications'))
standards = models.ManyToManyField('ProductStandard',
blank=True,
verbose_name=_('standards'),
help_text=_('attribute from legacy db'))
wine_village = models.ForeignKey('location.WineVillage', on_delete=models.PROTECT,
blank=True, null=True,
verbose_name=_('wine appellation'))
@ -198,8 +185,6 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
default=WAITING,
verbose_name=_('state'),
help_text=_('attribute from legacy db'))
brand = models.ForeignKey(ProductBrand, on_delete=models.PROTECT, null=True,
verbose_name=_('brand'))
tags = models.ManyToManyField('tag.Tag', related_name='products',
verbose_name=_('Tag'))
old_unique_key = models.CharField(max_length=255, unique=True,
@ -207,8 +192,8 @@ class Product(TranslatedFieldsMixin, BaseAttributes):
help_text=_('attribute from legacy db'))
vintage = models.IntegerField(verbose_name=_('vintage year'),
null=True, blank=True, default=None,
validators=[MinValueValidator(1900),
MaxValueValidator(2100)])
validators=[MinValueValidator(EARLIEST_VINTAGE_YEAR),
MaxValueValidator(LATEST_VINTAGE_YEAR)])
objects = ProductManager.from_queryset(ProductQuerySet)()

42
apps/product/transfer_data.py
View File

@ -139,27 +139,6 @@ def transfer_wine_classifications():
pprint(f"transfer_wine_classifications errors: {serialized_data.errors}")
def transfer_product_brand():
raw_queryset = transfer_models.Products.objects.raw(
"""
SELECT
DISTINCT brand,
1 as id
FROM products
WHERE brand IS NOT NULL
AND brand !='';
"""
)
queryset = [vars(query) for query in raw_queryset]
serialized_data = product_serializers.ProductBrandSerializer(
data=queryset,
many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"transfer_product_brand errors: {serialized_data.errors}")
def transfer_product():
queryset = transfer_models.Products.objects.all()
serialized_data = product_serializers.ProductSerializer(
@ -168,7 +147,22 @@ def transfer_product():
if serialized_data.is_valid():
serialized_data.save()
else:
pprint(f"transfer_product errors: {serialized_data.errors}")
errors = []
for d in serialized_data.errors: errors.append(d) if d else None
pprint(f"transfer_product errors: {errors}")
def transfer_plates():
queryset = transfer_models.Merchandise.objects.all()
serialized_data = product_serializers.PlateSerializer(
data=list(queryset.values()),
many=True)
if serialized_data.is_valid():
serialized_data.save()
else:
errors = []
for d in serialized_data.errors: errors.append(d) if d else None
pprint(f"transfer_plates errors: {errors}")
data_types = {
@ -186,7 +180,7 @@ data_types = {
transfer_wine_classifications,
],
"product": [
transfer_product_brand,
transfer_product
transfer_product,
transfer_plates,
],
}

1
apps/transfer/management/commands/transfer.py
View File

@ -27,6 +27,7 @@ class Command(BaseCommand):
LONG_DATA_TYPES = [
'update_country_flag',
'product_type',
'wine_characteristics',
'product',
]

43
apps/transfer/mixins.py
View File

@ -3,7 +3,7 @@ from django.forms.models import model_to_dict
from rest_framework import serializers
from tag import models as tag_models
from django.conf import settings
from product.models import ProductType
from product.models import ProductType, ProductSubType
class SecondDbManager(models.Manager):
@ -50,3 +50,44 @@ class TransferSerializerMixin(serializers.ModelSerializer):
'public': True
})
return tag_category
def get_vintage_year(self, vintage):
earliest_year = self.Meta.model.EARLIEST_VINTAGE_YEAR
latest_year = self.Meta.model.LATEST_VINTAGE_YEAR
if vintage:
if vintage.isdigit():
if len(vintage) == 2:
if vintage == '16':
return 2016
elif len(vintage) == 4:
if earliest_year < int(vintage) < latest_year:
return int(vintage)
elif vintage == '1584':
return 1984
elif vintage == '1017':
return 2017
elif len(vintage) == 5:
if vintage == '20115':
return 2015
elif vintage == '20174':
return 2017
elif vintage.endswith('er'):
return self.get_vintage_year(vintage[:-2])
def get_product_type(self, product_type):
if isinstance(product_type, ProductType):
return product_type
if product_type:
qs = ProductType.objects.filter(
index_name__icontains=product_type)
if qs.exists():
return qs.first()
def get_product_sub_type(self, product_type, product_sub_type):
product_type = self.get_product_type(product_type)
if product_type and product_sub_type:
qs = ProductSubType.objects.filter(
product_type=product_type,
index_name__icontains=product_type)
if qs.exists():
return qs.first()

10
apps/transfer/models.py
View File

@ -1031,3 +1031,13 @@ class WineLocations(MigrateMixin):
class Meta:
managed = False
db_table = 'wine_locations'
class Merchandise(MigrateMixin):
using = 'legacy'
name = models.CharField(max_length=255)
vintage = models.CharField(max_length=255)
highlighted = models.CharField(max_length=255)
site = models.ForeignKey('Sites', models.DO_NOTHING)
attachment_suffix_url = models.CharField(max_length=255)

178
apps/transfer/serializers/product.py
View File

@ -65,7 +65,7 @@ class WineBottlesProducedSerializer(TransferSerializerMixin):
CATEGORY_LABEL = 'Bottles produced'
CATEGORY_INDEX_NAME = slugify(CATEGORY_LABEL)
bottles_produced = serializers.CharField()
bottles_produced = serializers.CharField(allow_null=True)
class Meta:
model = tag_models.Tag
@ -87,7 +87,7 @@ class WineBottlesProducedSerializer(TransferSerializerMixin):
parted = value.split(' ')
if len(parted) > 1:
values = [int(i) if i.isdigit() else 0 for i in parted]
values = [int(i) for i in parted if i.isdigit()]
return reduce(lambda a, b: a + b, values)
lowered = value.lower()
@ -225,21 +225,6 @@ class ProductClassificationSerializer(TransferSerializerMixin):
return qs.first()
class ProductBrandSerializer(TransferSerializerMixin):
brand = serializers.CharField()
class Meta:
model = models.ProductBrand
fields = (
'brand',
)
def validate(self, attrs):
attrs['name'] = attrs.pop('brand')
return attrs
class ProductSerializer(TransferSerializerMixin):
establishment_id = serializers.PrimaryKeyRelatedField(
@ -263,25 +248,29 @@ class ProductSerializer(TransferSerializerMixin):
village_id = serializers.PrimaryKeyRelatedField(
queryset=transfer_models.WineLocations.objects.all(),
allow_null=True)
vineyard_id = serializers.PrimaryKeyRelatedField(
queryset=transfer_models.WineLocations.objects.all(),
allow_null=True)
# same as wine_village
# vineyard_id = serializers.PrimaryKeyRelatedField(
# queryset=transfer_models.WineLocations.objects.all(),
# allow_null=True)
wine_quality_id = serializers.PrimaryKeyRelatedField(
queryset=transfer_models.ProductClassification.objects.all(),
allow_null=True)
brand = serializers.CharField(allow_null=True)
name = serializers.CharField(allow_null=True)
# same as establishment name
id = serializers.IntegerField()
brand = serializers.CharField(allow_null=True, allow_blank=True)
name = serializers.CharField(allow_null=True, allow_blank=True)
vintage = serializers.CharField(allow_null=True)
type = serializers.CharField(allow_null=True)
wine_sub_region_id = serializers.CharField(allow_null=True)
state = serializers.CharField()
bottles_produced = serializers.CharField(allow_null=True)
bottles_produced = serializers.CharField(allow_null=True, allow_blank=True)
unique_key = serializers.CharField(allow_null=True)
class Meta:
model = models.Product
fields = (
'id',
'establishment_id', # done
'classification_id', # done
'wine_region_id', # done
@ -289,9 +278,9 @@ class ProductSerializer(TransferSerializerMixin):
'wine_color_id', # done
'appellation_id', # done
'village_id', # done
# 'vineyard_id', duplicate wine_village
# 'vineyard_id',
'wine_quality_id', # done
'brand', # done
'brand',
'name', # done
'vintage', # done
'type', # done
@ -302,11 +291,11 @@ class ProductSerializer(TransferSerializerMixin):
)
def validate(self, attrs):
name = attrs.pop('name', None)
establishment = attrs.pop('establishment_id', None)
state = attrs.pop('state', None)
brand = attrs.pop('brand', None)
classification = attrs.pop('classification_id', None)
vintage = attrs.pop('vintage_year', None)
vintage = attrs.pop('vintage', None)
product_type = attrs.pop('type')
wine_region = attrs.pop('wine_region_id', None)
wine_sub_region_id = attrs.pop('wine_sub_region_id', None)
@ -316,14 +305,19 @@ class ProductSerializer(TransferSerializerMixin):
appellation = attrs.pop('appellation_id', None)
village = attrs.pop('village_id', None)
wine_quality = attrs.pop('wine_quality_id', None)
name = self.get_name(name, brand)
old_id = attrs.pop('id')
state = self.get_state(attrs.pop('state', None))
attrs['old_id'] = old_id
attrs['name'] = name
attrs['old_unique_key'] = attrs.pop('unique_key')
attrs['establishment'] = self.get_establishment(establishment)
attrs['state'] = self.get_state(state)
attrs['brand'] = self.get_brand(brand)
attrs['state'] = state
# attrs['brand'] = self.get_brand(brand)
attrs['wine_classification'] = self.get_wine_classification(classification)
attrs['wine_quiality'] = self.get_wine_standard(classification,
models.ProductStandard.WINEQUALITY)
attrs['wine_quality'] = self.get_wine_standard(wine_quality,
models.ProductStandard.WINEQUALITY)
attrs['wine_appellation'] = self.get_wine_standard(appellation,
models.ProductStandard.APPELLATION)
attrs['product_type'] = self.get_product_type(product_type)
@ -337,8 +331,41 @@ class ProductSerializer(TransferSerializerMixin):
attrs['wine_color_tag'] = self.get_tag(wine_color,
WineColorSerializer.CATEGORY_INDEX_NAME)
attrs['wine_village'] = self.get_wine_village(village)
attrs['available'] = self.get_availability(state)
attrs['slug'] = self.get_slug(name, old_id)
return attrs
def create(self, validated_data):
qs = self.Meta.model.objects.filter(old_id=validated_data.get('old_id'))
# classifications
classifications = [validated_data.pop('wine_classification', None)]
# standards
standards = [validated_data.pop('wine_quality', None),
validated_data.pop('wine_appellation', None)]
# tags
tags = [validated_data.pop('bottles_produced_tag', None),
validated_data.pop('wine_type_tag', None),
validated_data.pop('wine_color_tag', None)]
if not qs.exists():
obj = super().create(validated_data)
else:
obj = qs.first()
# adding classification
obj.classifications.add(*[i for i in classifications if i])
# adding standard
obj.standards.add(*[i for i in standards if i])
# adding tags
obj.tags.add(*[i for i in tags if i])
return obj
def get_name(self, name, brand):
if name:
return name
if brand and not name:
return brand
def get_establishment(self, establishment):
if establishment:
qs = establishment_models.Establishment.objects.filter(
@ -353,12 +380,6 @@ class ProductSerializer(TransferSerializerMixin):
return models.Product.OUT_OF_PRODUCTION
return models.Product.WAITING
def get_brand(self, brand):
if brand:
qs = models.ProductBrand.objects.filter(brand__icontains=brand)
if qs.exists():
return qs.first()
def get_wine_classification(self, classification):
if classification:
classification_qs = models.ProductClassification.objects.filter(
@ -366,33 +387,6 @@ class ProductSerializer(TransferSerializerMixin):
if classification_qs.exists():
return classification_qs.first()
def get_vintage_year(self, vintage):
if vintage.isdigit():
if len(vintage) == 2:
if vintage == '16':
return 2016
elif len(vintage) == 4:
if 1900 < int(vintage) < 2100:
return int(vintage)
elif vintage == '1584':
return 1984
elif vintage == '1017':
return 2017
elif len(vintage) == 5:
if vintage == '20115':
return 2015
elif vintage == '20174':
return 2017
elif vintage.endswith('er'):
return self.get_vintage_year(vintage[:-2])
def get_product_type_tag(self, product_type):
if product_type:
qs = models.ProductType.objects.filter(
index_name__icontains=product_type)
if qs.exists():
return qs.first()
def get_wine_region(self, wine_region):
if wine_region:
wine_region_qs = location_models.WineRegion.objects.filter(
@ -425,9 +419,59 @@ class ProductSerializer(TransferSerializerMixin):
def get_wine_standard(self, standard, standard_type):
if standard:
standard = standard.id if hasattr(standard, 'id') else standard
if isinstance(standard, transfer_models.ProductClassification):
standard = standard.id
standard_qs = models.ProductStandard.objects.filter(
standard_type=standard_type,
old_id=standard.id)
old_id=standard)
if standard_qs.exists():
return standard_qs.first()
return standard_qs.first()
def get_availability(self, state: int):
if state == models.Product.PUBLISHED:
return True
return False
def get_slug(self, name, old_id):
return slugify(f'{name}-{old_id}')
class PlateSerializer(TransferSerializerMixin):
PRODUCT_TYPE_INDEX_NAME = 'souvenir'
PRODUCT_SUB_TYPE_INDEX_NAME = 'plate'
id = serializers.IntegerField()
name = serializers.CharField()
vintage = serializers.CharField()
class Meta(ProductSerializer.Meta):
fields = (
'id',
'name',
'vintage',
)
def validate(self, attrs):
product_type = self.get_product_type(self.PRODUCT_TYPE_INDEX_NAME)
attrs['old_id'] = attrs.pop('id')
attrs['vintage'] = self.get_vintage_year(attrs.pop('vintage'))
attrs['product_type'] = product_type
attrs['subtype'] = self.get_product_sub_type(product_type,
self.PRODUCT_SUB_TYPE_INDEX_NAME)
return attrs
def create(self, validated_data):
qs = self.Meta.model.objects.filter(old_id=validated_data.get('old_id'))
# subtypes
subtypes = [validated_data.pop('subtype', None)]
if not qs.exists():
obj = super().create(validated_data)
else:
obj = qs.first()
# adding classification
obj.subtypes.add(*[i for i in subtypes if i])
return obj