<commit_before># coding: utf-8
import datetime
import logging
from werkzeug.utils import import_string
from flask import session, current_app
from flask.ext.babel import lazy_gettext as _l
from quokka.utils import get_current_user, lazy_str_setting
from quokka.core.templates import render_template
from quokka.core.db import db
from quokka.core.models import Publishable, Ordered, Dated, Content
from quokka.modules.media.models import Image

logger = logging.getLogger()

class BaseProductReference(object):
    def get_title(self):
        return getattr(self, 'title', None)

    def get_description(self):
        return getattr(self, 'description', None)

    def get_unity_value(self):
        return getattr(self, 'unity_value', None)

    def get_weight(self):
        return getattr(self, 'weight', None)

    def get_dimensions(self):
        return getattr(self, 'dimensions', None)

    def get_summary(self):
        summary = getattr(self, 'summary', None)
        if not summary:
            try:
                return self.get_description()[:255]
            except:
                pass
        return summary

    def get_extra_value(self):
        return getattr(self, 'extra_value', None)

    def get_uid(self):
        return str(self.id)

    def set_status(self, *args, **kwargs):
        pass

    def remove_item(self, *args, **kwargs):
        pass

class BaseProduct(BaseProductReference, Content):
    description = db.StringField(required=True)
    unity_value = db.FloatField()
    weight = db.FloatField()
    dimensions = db.StringField()
    extra_value = db.FloatField()

    meta = {
        'allow_inheritance': True
    }

class Item(Ordered, Dated, db.EmbeddedDocument):
    product = db.ReferenceField(Content)
    reference = db.GenericReferenceField()  # customized product
    """
    Must implement all the BaseProduct methods/ its optional
    if None, "product" will be considered
    """
    uid = db.StringField()
    title = db.StringField(required=True, max_length=255)
    description = db.StringField(required=True)
    link = db.StringField()
    quantity = db.FloatField(default=1)
    unity_value = db.FloatField(required=True)
    total_value = db.FloatField()
    weight = db.FloatField()
    dimensions = db.StringField()
    extra_value = db.FloatField()
    allowed_to_set = db.ListField(db.StringField(), default=['quantity'])
    pipeline = db.ListField(db.StringField(), default=[])

    def set_status(self, status):
        if self.reference and hasattr(self.reference, 'set_status'):
            self.reference.set_status(status)
        if self.product and hasattr(self.product, 'set_status'):
            self.product.set_status(status)

    def get_main_image_url(self, thumb=False, default=None):
        try:
            return self.product.get_main_image_url(thumb, default)
        except:
            return None

    @classmethod
    def normalize(cls, kwargs):
        new = {}
        for k, v in kwargs.items():
            field = cls._fields.get(k)
            if not field:
                continue
            new[k] = field.to_python(v)
        return new

    def __unicode__(self):
        return u"{i.uid} {i.title}".format(i=self)

    def get_uid(self):
        try:
            return self.product.get_uid()
        except:
            return self.uid

    @property
    def unity_plus_extra(self):
        return float(self.unity_value or 0) + float(self.extra_value or 0)

    @property
    def total(self):
        self.clean()
        self.total_value = self.unity_plus_extra * float(self.quantity or 1)
        return self.total_value

    def clean(self):
        mapping = [
            ('title', 'get_title'),
            ('description', 'get_description'),
            ('link', 'get_absolute_url'),
            ('unity_value', 'get_unity_value'),
            ('weight', 'get_weight'),
            ('dimensions', 'get_dimensions'),
            ('extra_value', 'get_extra_value'),
            ('uid', 'get_uid'),
        ]

        references = [self.reference, self.product]

        for ref in references:
            if not ref:
                continue
            for attr, method in mapping:
                current = getattr(self, attr, None)
                if current is not None:
                    continue
                setattr(self, attr, getattr(ref, method, lambda: None)())

class Payment(db.EmbeddedDocument):
    uid = db.StringField()
    payment_system = db.StringField()
    method = db.StringField()
    value = db.FloatField()
    extra_value = db.FloatField()
    date = db.DateTimeField()
    confirmed_at = db.DateTimeField()
    status = db.StringField()

class Processor(Publishable, db.DynamicDocument):
    identifier = db.StringField(max_length=100, unique=True)
    module = db.StringField(max_length=255)
    requires = db.ListField(db.StringField(max_length=255))
    description = db.StringField()
    title = db.StringField()
    image = db.ReferenceField(Image, reverse_delete_rule=db.NULLIFY)
    link = db.StringField(max_length=255)
    config = db.DictField(default=lambda: {})
    pipeline = db.ListField(db.StringField(max_length=255), default=[])

    def import_processor(self):
        return import_string(self.module)

    def get_instance(self, *args, **kwargs):
        if not 'config' in kwargs:
            kwargs['config'] = self.config
        kwargs['_record'] = self
        return self.import_processor()(*args, **kwargs)

    def clean(self, *args, **kwargs):
        for item in (self.requires or []):
            import_string(item)
        super(Processor, self).clean(*args, **kwargs)

    def __unicode__(self):
        return self.identifier

    @classmethod
    def get_instance_by_identifier(cls, identifier, cart=None):
        processor = cls.objects.get(identifier=identifier)
        return processor.get_instance(cart=cart)

    @classmethod
    def get_default_processor(cls):
        default = lazy_str_setting(
            'CART_DEFAULT_PROCESSOR',
            default={
                'module': 'quokka.modules.cart.processors.Dummy',
                'identifier': 'dummy',
                'published': True,
                'title': "Test"
            }
        )

        try:
            return cls.objects.get(identifier=default['identifier'])
        except:
            return cls.objects.create(**default)

    def save(self, *args, **kwargs):
        self.import_processor()
        super(Processor, self).save(*args, **kwargs)

class Cart(Publishable, db.DynamicDocument):
    STATUS = (
        ("pending", _l("Pending")),  # not checked out
        ("checked_out", _l("Checked out")),  # not confirmed (payment)
        ("analysing", _l("Analysing")),  # Analysing payment
        ("confirmed", _l("Confirmed")),  # Payment confirmed
        ("completed", _l("Completed")),  # Payment completed (money released)
        ("refunding", _l("Refunding")),  # Buyer asks refund
        ("refunded", _l("Refunded")),  # Money refunded to buyer
        ("cancelled", _l("Cancelled")),  # Cancelled without processing
        ("abandoned", _l("Abandoned")),  # Long time no update
    )
    reference = db.GenericReferenceField()
    """reference must implement set_status(**kwargs) method
    arguments: status(str), value(float), date, uid(str), msg(str)
    and extra(dict).
    Also reference must implement get_uid() which will return
    the unique identifier for this transaction"""

    belongs_to = db.ReferenceField('User',
                                   # default=get_current_user,
                                   reverse_delete_rule=db.NULLIFY)
    items = db.ListField(db.EmbeddedDocumentField(Item))
    payment = db.ListField(db.EmbeddedDocumentField(Payment))
    status = db.StringField(choices=STATUS, default='pending')
    total = db.FloatField(default=0)
    extra_costs = db.DictField(default=lambda: {})
    sender_data = db.DictField(default=lambda: {})
    shipping_data = db.DictField(default=lambda: {})
    shipping_cost = db.FloatField(default=0)
    tax = db.FloatField(default=0)
    processor = db.ReferenceField(Processor,
                                  default=Processor.get_default_processor,
                                  reverse_delete_rule=db.NULLIFY)
    reference_code = db.StringField()  # Reference code for filtering
    checkout_code = db.StringField()  # The UID for transaction checkout
    transaction_code = db.StringField()  # The UID for transaction
    requires_login = db.BooleanField(default=True)
    continue_shopping_url = db.StringField(
        default=lambda: current_app.config.get(
            'CART_CONTINUE_SHOPPING_URL', '/'
        )
    )
    pipeline = db.ListField(db.StringField(), default=[])
    log = db.ListField(db.StringField(), default=[])
    config = db.DictField(default=lambda: {})

    def send_response(self, response, identifier):
        if self.reference and hasattr(self.reference, 'get_response'):
            self.reference.get_response(response, identifier)

        for item in self.items:
            if hasattr(item, 'get_response'):
                item.get_response(response, identifier)

    def set_tax(self, tax, save=False):
        """
        set tax and send to references
        """
        try:
            tax = float(tax)
            self.tax = tax
            self.set_reference_tax(tax)
        except Exception as e:
            self.addlog("impossible to set tax: %s" % str(e))

    def set_status(self, status, save=False):
        """
        THis method will be called by the processor
        which will pass a valid status as in STATUS
        so, this method will dispatch the STATUS to
        all the items and also the 'reference' if set
        """
        if self.status != status:
            self.status = status

        self.set_reference_statuses(status)

        if save:
            self.save()

    def set_reference_statuses(self, status):
        if self.reference and hasattr(self.reference, 'set_status'):
            self.reference.set_status(status)

        for item in self.items:
            item.set_status(status)

    def set_reference_tax(self, tax):
        if self.reference and hasattr(self.reference, 'set_tax'):
            self.reference.set_tax(tax)

        for item in self.items:
            if hasattr(item, 'set_tax'):
                item.set_tax(tax)

    def addlog(self, msg, save=True):
        self.log.append(u"{0},{1}".format(datetime.datetime.now(), msg))
        logger.debug(msg)
        save and self.save()

    @property
    def uid(self):
        return self.get_uid()

    def get_uid(self):
        try:
            return self.reference.get_uid() or str(self.id)
        except Exception as e:
            self.addlog("Using self.id as reference", save=False)
            return str(self.id)

    def __unicode__(self):
        return u"{o.uid} - {o.processor.identifier}".format(o=self)

    def get_extra_costs(self):
        if self.extra_costs:
            return sum(self.extra_costs.values())

    @classmethod
    def get_cart(cls):
        """
        get or create a new cart related to the session
        if there is a current logged in user it will be set
        else it will be set during the checkout.
        """
        session.permanent = current_app.config.get(
            "CART_PERMANENT_SESSION", True)
        try:
            cart = cls.objects.get(
                id=session.get('cart_id'),
                status='pending'
            )
            cart.save()
        except (cls.DoesNotExist, db.ValidationError):
            cart = cls(status="pending")
            cart.save()
            session['cart_id'] = str(cart.id)
            try:
                session.pop('cart_pipeline_index')
                session.pop('cart_pipeline_args')
            except KeyError:
                pass

        return cart

    def assign(self):
        self.belongs_to = self.belongs_to or get_current_user()

    def save(self, *args, **kwargs):
        self.total = sum([item.total for item in self.items])
        self.assign()
        self.reference_code = self.get_uid()
        super(Cart, self).save(*args, **kwargs)
        self.set_reference_statuses(self.status)

    def get_item(self, uid):
        # MongoEngine/mongoengine#503
        return self.items.get(uid=uid)

    def set_item(self, **kwargs):
        if 'product' in kwargs:
            if not isinstance(kwargs['product'], Content):
                try:
                    kwargs['product'] = Content.objects.get(
                        id=kwargs['product'])
                except Content.DoesNotExist:
                    kwargs['product'] = None

        uid = kwargs.get(
            'uid',
            kwargs['product'].get_uid() if kwargs.get('product') else None
        )

        if not uid:
            self.addlog("Cannot add item without an uid %s" % kwargs)
            return

        item = self.get_item(uid)

        kwargs = Item.normalize(kwargs)

        if not item:
            # items should only be added if there is a product (for safety)
            if not kwargs.get('product'):
                self.addlog("there is no product to add item")
                return
            allowed = ['product', 'quantity']
            item = self.items.create(
                **{k: v for k, v in kwargs.items() if k in allowed}
            )
            self.addlog("New item created %s" % item, save=False)
        else:
            # update only allowed attributes
            item = self.items.update(
                {k: v for k, v in kwargs.items() if k in item.allowed_to_set},
                uid=item.uid
            )
            self.addlog("Item updated %s" % item, save=False)

            if int(kwargs.get('quantity', "1")) == 0:
                self.addlog("quantity is 0, removed %s" % kwargs, save=False)
                self.remove_item(**kwargs)

        self.save()
        self.reload()
        return item

    def remove_item(self, **kwargs):
        deleted = self.items.delete(**kwargs)
        print self.reference
        if self.reference and hasattr(self.reference, 'remove_item'):
            self.reference.remove_item(**kwargs)
            print self.reference
        return deleted

    def checkout(self, processor=None, *args, **kwargs):
        self.set_processor(processor)
        processor_instance = self.processor.get_instance(self, *args, **kwargs)
        if processor_instance.validate():
            response = processor_instance.process()
            self.status = 'checked_out'
            self.save()
            return response
        else:
            self.addlog("Cart did not validate")
            raise Exception("Cart did not validate")  # todo: specialize this

    def get_items_pipeline(self):
        if not self.items:
            return []

        return reduce(
            lambda x, y: x + y, [item.pipeline for item in self.items]
        )

    def build_pipeline(self):
        items = ['quokka.modules.cart.pipelines:StartPipeline']
        items.extend(current_app.config.get('CART_PIPELINE', []))
        items.extend(self.get_items_pipeline())
        items.extend(self.pipeline)
        items.extend(self.processor and self.processor.pipeline or [])
        return items

    def process_pipeline(self):
        if not self.items:
            return render_template('cart/empty_cart.html',
                                   url=self.continue_shopping_url)

        pipelines = self.build_pipeline()
        index = session.get('cart_pipeline_index', 0)
        Pipeline = import_string(pipelines[index])
        return Pipeline(self, pipelines, index)._preprocess()

    def set_processor(self, processor=None):
        if not self.processor:
            self.processor = Processor.get_default_processor()
            self.save()

        if not processor:
            return

        if isinstance(processor, Processor):
            self.processor = processor
            self.save()
            return

        try:
            self.processor = Processor.objects.get(id=processor)
        except:
            self.processor = Processor.objects.get(identifier=processor)

        self.save()

    def get_available_processors(self):
        return Processor.objects(published=True)
<commit_msg>Trying to understand the UNicodeError<commit_after># coding: utf-8
import datetime
import logging
from werkzeug.utils import import_string
from flask import session, current_app
from flask.ext.babel import lazy_gettext as _l
from quokka.utils import get_current_user, lazy_str_setting
from quokka.core.templates import render_template
from quokka.core.db import db
from quokka.core.models import Publishable, Ordered, Dated, Content
from quokka.modules.media.models import Image

logger = logging.getLogger()

class BaseProductReference(object):
    def get_title(self):
        return getattr(self, 'title', None)

    def get_description(self):
        return getattr(self, 'description', None)

    def get_unity_value(self):
        return getattr(self, 'unity_value', None)

    def get_weight(self):
        return getattr(self, 'weight', None)

    def get_dimensions(self):
        return getattr(self, 'dimensions', None)

    def get_summary(self):
        summary = getattr(self, 'summary', None)
        if not summary:
            try:
                return self.get_description()[:255]
            except:
                pass
        return summary

    def get_extra_value(self):
        return getattr(self, 'extra_value', None)

    def get_uid(self):
        return str(self.id)

    def set_status(self, *args, **kwargs):
        pass

    def remove_item(self, *args, **kwargs):
        pass

class BaseProduct(BaseProductReference, Content):
    description = db.StringField(required=True)
    unity_value = db.FloatField()
    weight = db.FloatField()
    dimensions = db.StringField()
    extra_value = db.FloatField()

    meta = {
        'allow_inheritance': True
    }

class Item(Ordered, Dated, db.EmbeddedDocument):
    product = db.ReferenceField(Content)
    reference = db.GenericReferenceField()  # customized product
    """
    Must implement all the BaseProduct methods/ its optional
    if None, "product" will be considered
    """
    uid = db.StringField()
    title = db.StringField(required=True, max_length=255)
    description = db.StringField(required=True)
    link = db.StringField()
    quantity = db.FloatField(default=1)
    unity_value = db.FloatField(required=True)
    total_value = db.FloatField()
    weight = db.FloatField()
    dimensions = db.StringField()
    extra_value = db.FloatField()
    allowed_to_set = db.ListField(db.StringField(), default=['quantity'])
    pipeline = db.ListField(db.StringField(), default=[])

    def set_status(self, status):
        if self.reference and hasattr(self.reference, 'set_status'):
            self.reference.set_status(status)
        if self.product and hasattr(self.product, 'set_status'):
            self.product.set_status(status)

    def get_main_image_url(self, thumb=False, default=None):
        try:
            return self.product.get_main_image_url(thumb, default)
        except:
            return None

    @classmethod
    def normalize(cls, kwargs):
        new = {}
        for k, v in kwargs.items():
            field = cls._fields.get(k)
            if not field:
                continue
            new[k] = field.to_python(v)
        return new

    def __unicode__(self):
        return u"{i.uid} {i.title}".format(i=self)

    def get_uid(self):
        try:
            return self.product.get_uid()
        except:
            return self.uid

    @property
    def unity_plus_extra(self):
        return float(self.unity_value or 0) + float(self.extra_value or 0)

    @property
    def total(self):
        self.clean()
        self.total_value = self.unity_plus_extra * float(self.quantity or 1)
        return self.total_value

    def clean(self):
        mapping = [
            ('title', 'get_title'),
            ('description', 'get_description'),
            ('link', 'get_absolute_url'),
            ('unity_value', 'get_unity_value'),
            ('weight', 'get_weight'),
            ('dimensions', 'get_dimensions'),
            ('extra_value', 'get_extra_value'),
            ('uid', 'get_uid'),
        ]

        references = [self.reference, self.product]

        for ref in references:
            if not ref:
                continue
            for attr, method in mapping:
                current = getattr(self, attr, None)
                if current is not None:
                    continue
                setattr(self, attr, getattr(ref, method, lambda: None)())

class Payment(db.EmbeddedDocument):
    uid = db.StringField()
    payment_system = db.StringField()
    method = db.StringField()
    value = db.FloatField()
    extra_value = db.FloatField()
    date = db.DateTimeField()
    confirmed_at = db.DateTimeField()
    status = db.StringField()

class Processor(Publishable, db.DynamicDocument):
    identifier = db.StringField(max_length=100, unique=True)
    module = db.StringField(max_length=255)
    requires = db.ListField(db.StringField(max_length=255))
    description = db.StringField()
    title = db.StringField()
    image = db.ReferenceField(Image, reverse_delete_rule=db.NULLIFY)
    link = db.StringField(max_length=255)
    config = db.DictField(default=lambda: {})
    pipeline = db.ListField(db.StringField(max_length=255), default=[])

    def import_processor(self):
        return import_string(self.module)

    def get_instance(self, *args, **kwargs):
        if not 'config' in kwargs:
            kwargs['config'] = self.config
        kwargs['_record'] = self
        return self.import_processor()(*args, **kwargs)

    def clean(self, *args, **kwargs):
        for item in (self.requires or []):
            import_string(item)
        super(Processor, self).clean(*args, **kwargs)

    def __unicode__(self):
        return self.identifier

    @classmethod
    def get_instance_by_identifier(cls, identifier, cart=None):
        processor = cls.objects.get(identifier=identifier)
        return processor.get_instance(cart=cart)

    @classmethod
    def get_default_processor(cls):
        default = lazy_str_setting(
            'CART_DEFAULT_PROCESSOR',
            default={
                'module': 'quokka.modules.cart.processors.Dummy',
                'identifier': 'dummy',
                'published': True,
                'title': "Test"
            }
        )

        try:
            return cls.objects.get(identifier=default['identifier'])
        except:
            return cls.objects.create(**default)

    def save(self, *args, **kwargs):
        self.import_processor()
        super(Processor, self).save(*args, **kwargs)

class Cart(Publishable, db.DynamicDocument):
    STATUS = (
        ("pending", _l("Pending")),  # not checked out
        ("checked_out", _l("Checked out")),  # not confirmed (payment)
        ("analysing", _l("Analysing")),  # Analysing payment
        ("confirmed", _l("Confirmed")),  # Payment confirmed
        ("completed", _l("Completed")),  # Payment completed (money released)
        ("refunding", _l("Refunding")),  # Buyer asks refund
        ("refunded", _l("Refunded")),  # Money refunded to buyer
        ("cancelled", _l("Cancelled")),  # Cancelled without processing
        ("abandoned", _l("Abandoned")),  # Long time no update
    )
    reference = db.GenericReferenceField()
    """reference must implement set_status(**kwargs) method
    arguments: status(str), value(float), date, uid(str), msg(str)
    and extra(dict).
    Also reference must implement get_uid() which will return
    the unique identifier for this transaction"""

    belongs_to = db.ReferenceField('User',
                                   # default=get_current_user,
                                   reverse_delete_rule=db.NULLIFY)
    items = db.ListField(db.EmbeddedDocumentField(Item))
    payment = db.ListField(db.EmbeddedDocumentField(Payment))
    status = db.StringField(choices=STATUS, default='pending')
    total = db.FloatField(default=0)
    extra_costs = db.DictField(default=lambda: {})
    sender_data = db.DictField(default=lambda: {})
    shipping_data = db.DictField(default=lambda: {})
    shipping_cost = db.FloatField(default=0)
    tax = db.FloatField(default=0)
    processor = db.ReferenceField(Processor,
                                  default=Processor.get_default_processor,
                                  reverse_delete_rule=db.NULLIFY)
    reference_code = db.StringField()  # Reference code for filtering
    checkout_code = db.StringField()  # The UID for transaction checkout
    transaction_code = db.StringField()  # The UID for transaction
    requires_login = db.BooleanField(default=True)
    continue_shopping_url = db.StringField(
        default=lambda: current_app.config.get(
            'CART_CONTINUE_SHOPPING_URL', '/'
        )
    )
    pipeline = db.ListField(db.StringField(), default=[])
    log = db.ListField(db.StringField(), default=[])
    config = db.DictField(default=lambda: {})

    def send_response(self, response, identifier):
        if self.reference and hasattr(self.reference, 'get_response'):
            self.reference.get_response(response, identifier)

        for item in self.items:
            if hasattr(item, 'get_response'):
                item.get_response(response, identifier)

    def set_tax(self, tax, save=False):
        """
        set tax and send to references
        """
        try:
            tax = float(tax)
            self.tax = tax
            self.set_reference_tax(tax)
        except Exception as e:
            self.addlog("impossible to set tax: %s" % str(e))

    def set_status(self, status, save=False):
        """
        THis method will be called by the processor
        which will pass a valid status as in STATUS
        so, this method will dispatch the STATUS to
        all the items and also the 'reference' if set
        """
        if self.status != status:
            self.status = status

        self.set_reference_statuses(status)

        if save:
            self.save()

    def set_reference_statuses(self, status):
        if self.reference and hasattr(self.reference, 'set_status'):
            self.reference.set_status(status)

        for item in self.items:
            item.set_status(status)

    def set_reference_tax(self, tax):
        if self.reference and hasattr(self.reference, 'set_tax'):
            self.reference.set_tax(tax)

        for item in self.items:
            if hasattr(item, 'set_tax'):
                item.set_tax(tax)

    def addlog(self, msg, save=True):
        try:
            self.log.append(u"{0},{1}".format(datetime.datetime.now(), msg))
            logger.debug(msg)
            save and self.save()
        except UnicodeDecodeError as e:
            logger.info(msg)
            logger.error(str(e))

    @property
    def uid(self):
        return self.get_uid()

    def get_uid(self):
        try:
            return self.reference.get_uid() or str(self.id)
        except Exception as e:
            self.addlog("Using self.id as reference", save=False)
            return str(self.id)

    def __unicode__(self):
        return u"{o.uid} - {o.processor.identifier}".format(o=self)

    def get_extra_costs(self):
        if self.extra_costs:
            return sum(self.extra_costs.values())

    @classmethod
    def get_cart(cls):
        """
        get or create a new cart related to the session
        if there is a current logged in user it will be set
        else it will be set during the checkout.
        """
        session.permanent = current_app.config.get(
            "CART_PERMANENT_SESSION", True)
        try:
            cart = cls.objects.get(
                id=session.get('cart_id'),
                status='pending'
            )
            cart.save()
        except (cls.DoesNotExist, db.ValidationError):
            cart = cls(status="pending")
            cart.save()
            session['cart_id'] = str(cart.id)
            try:
                session.pop('cart_pipeline_index')
                session.pop('cart_pipeline_args')
            except KeyError:
                pass

        return cart

    def assign(self):
        self.belongs_to = self.belongs_to or get_current_user()

    def save(self, *args, **kwargs):
        self.total = sum([item.total for item in self.items])
        self.assign()
        self.reference_code = self.get_uid()
        super(Cart, self).save(*args, **kwargs)
        self.set_reference_statuses(self.status)

    def get_item(self, uid):
        # MongoEngine/mongoengine#503
        return self.items.get(uid=uid)

    def set_item(self, **kwargs):
        if 'product' in kwargs:
            if not isinstance(kwargs['product'], Content):
                try:
                    kwargs['product'] = Content.objects.get(
                        id=kwargs['product'])
                except Content.DoesNotExist:
                    kwargs['product'] = None

        uid = kwargs.get(
            'uid',
            kwargs['product'].get_uid() if kwargs.get('product') else None
        )

        if not uid:
            self.addlog("Cannot add item without an uid %s" % kwargs)
            return

        item = self.get_item(uid)

        kwargs = Item.normalize(kwargs)

        if not item:
            # items should only be added if there is a product (for safety)
            if not kwargs.get('product'):
                self.addlog("there is no product to add item")
                return
            allowed = ['product', 'quantity']
            item = self.items.create(
                **{k: v for k, v in kwargs.items() if k in allowed}
            )
            self.addlog("New item created %s" % item, save=False)
        else:
            # update only allowed attributes
            item = self.items.update(
                {k: v for k, v in kwargs.items() if k in item.allowed_to_set},
                uid=item.uid
            )
            self.addlog("Item updated %s" % item, save=False)

            if int(kwargs.get('quantity', "1")) == 0:
                self.addlog("quantity is 0, removed %s" % kwargs, save=False)
                self.remove_item(**kwargs)

        self.save()
        self.reload()
        return item

    def remove_item(self, **kwargs):
        deleted = self.items.delete(**kwargs)
        print self.reference
        if self.reference and hasattr(self.reference, 'remove_item'):
            self.reference.remove_item(**kwargs)
            print self.reference
        return deleted

    def checkout(self, processor=None, *args, **kwargs):
        self.set_processor(processor)
        processor_instance = self.processor.get_instance(self, *args, **kwargs)
        if processor_instance.validate():
            response = processor_instance.process()
            self.status = 'checked_out'
            self.save()
            return response
        else:
            self.addlog("Cart did not validate")
            raise Exception("Cart did not validate")  # todo: specialize this

    def get_items_pipeline(self):
        if not self.items:
            return []

        return reduce(
            lambda x, y: x + y, [item.pipeline for item in self.items]
        )

    def build_pipeline(self):
        items = ['quokka.modules.cart.pipelines:StartPipeline']
        items.extend(current_app.config.get('CART_PIPELINE', []))
        items.extend(self.get_items_pipeline())
        items.extend(self.pipeline)
        items.extend(self.processor and self.processor.pipeline or [])
        return items

    def process_pipeline(self):
        if not self.items:
            return render_template('cart/empty_cart.html',
                                   url=self.continue_shopping_url)

        pipelines = self.build_pipeline()
        index = session.get('cart_pipeline_index', 0)
        Pipeline = import_string(pipelines[index])
        return Pipeline(self, pipelines, index)._preprocess()

    def set_processor(self, processor=None):
        if not self.processor:
            self.processor = Processor.get_default_processor()
            self.save()

        if not processor:
            return

        if isinstance(processor, Processor):
            self.processor = processor
            self.save()
            return

        try:
            self.processor = Processor.objects.get(id=processor)
        except:
            self.processor = Processor.objects.get(identifier=processor)

        self.save()

    def get_available_processors(self):
        return Processor.objects(published=True)
<|endoftext|>


<commit_before>import csv

class OutputText(object):
    """OutputText the clustering result to various types of file such as txt, csv, and html.
    """
    @staticmethod
    def txt_perline(perline_file, perline_analysis, max_cluster_id, cluster_labels, original_logs):
        """Write clustering result per line to txt file. This clustering result is for all members in a node.

        Parameters
        ----------
        perline_file            : str
            Filename of output file per line.
        perline_analysis        : dict
            Dictionary of cluster label per log line.
        cluster_labels          : iterable
            List of cluster labels in auth.log.
        max_cluster_id          : int
            Maximum value of cluster identifier.
        original_logs           : iterable
            List of original event logs.
        """
        fopen = open(perline_file, 'w')
        for rowid, cluster_id in perline_analysis.iteritems():
            cluster_label = 'undefined' if cluster_id > max_cluster_id else cluster_labels[cluster_id]
            fopen.write(str(cluster_id) + '; ' + cluster_label + '; ' + original_logs[rowid])
        fopen.close()

    @staticmethod
    def txt_percluster(percluster_file, clusters, mode, graph, original_logs):
        """Write clustering result to txt file.

        Parameters
        ----------
        percluster_file         : str
            Filename of output file per cluster.
        clusters                : dict
            Dictionary of a list containing node identifier per cluster.
        mode                    : str
            Mode of clustering method, i.e., graph or text.
        graph                   : graph
            The graph which its clustering result to be written to a file.
        original_logs           : iterable
            List of original event logs.
        """
        f = open(percluster_file, 'w')
        for cluster_id, nodes in clusters.iteritems():
            f.write('Cluster #' + str(cluster_id) + '\n')
            for node in nodes:
                if mode == 'graph':
                    members = graph.node[node]['member']
                    for member in members:
                        f.write(original_logs[member])
                elif mode == 'text':
                    f.write(original_logs[node] + '\n')
            f.write('\n')

        f.close()

    @staticmethod
    def csv_cluster_property(report_file, cluster_property, cluster_abstraction, anomaly_score, quadratic_score,
                             normalized_score, sentiment_score, anomaly_decision, evaluation_metrics):
        """Write cluster property to a csv file.

        Parameters
        ----------
        report_file             : str
            Filename and full path for anomaly score and cluster property.
        cluster_property        : dict
            Dictionary of cluster property.
        cluster_abstraction     : dict
            Dictionary of cluster abstraction.
        anomaly_score           : dict
            Dictionary of anomaly score per cluster.
        quadratic_score         : dict
            Dictionary of quadratic transformation of anomaly score.
        normalized_score        : dict
            Dictionary of normalized anomaly score.
        sentiment_score         : dict
            Dictionary of sentiment score per cluster.
        anomaly_decision        : dict
            Dictionary of anomaly decision per cluster.
        evaluation_metrics      : dict
            Dictionary of all evaluation metrics, both internal and external evaluation.
        """
        # write to csv
        f = open(report_file, 'wt')
        writer = csv.writer(f)

        # set header
        header = ('cluster_id', 'cluster_abstraction') + tuple(cluster_property[0].keys()) + \
                 ('anomaly_score', 'quadratic_score', 'normalized_score', 'sentiment_score', 'final_score', 'decision')
        writer.writerow(header)

        # write data
        for cluster_id, abstract in cluster_abstraction.iteritems():
            row = (cluster_id, abstract) + \
                  tuple(cluster_property[cluster_id].values()) + \
                  (anomaly_score[cluster_id], quadratic_score[cluster_id], normalized_score[cluster_id],
                   sentiment_score[cluster_id]) + anomaly_decision[cluster_id]
            writer.writerow(row)

        # write evaluation metrics
        writer.writerow(('',))
        writer.writerow(('Evaluation metrics of clustering:',))
        for metrics, value in evaluation_metrics.iteritems():
            writer.writerow((metrics, value))

        f.close()

    @staticmethod
    def txt_anomaly_perline(anomaly_decision, clusters, graph, anomaly_perline_file, original_logs):
        """Write anomaly detection result per log line.

        Parameters
        ----------
        anomaly_decision        : dict
            Dictionary of anomaly decision per cluster.
        clusters                : dict
            Dictionary of a list containing node identifier per cluster.
        graph                   : graph
            The graph which its clustering result to be written to a file.
        anomaly_perline_file    : str
            Filename for result of anomaly label per line.
        original_logs           : iterable
            List of original event logs.
        """
        decision_perlog = {}
        for cluster_id, decision in anomaly_decision.iteritems():
            for node in clusters[cluster_id]:
                members = graph.node[node]['member']
                for member in members:
                    decision_perlog[member] = decision

        # write to file
        f = open(anomaly_perline_file, 'w')
        for rowid, decision in decision_perlog.iteritems():
            f.write(decision[1] + '; ' + original_logs[rowid])
        f.close()
<commit_msg>Add mode text-h5<commit_after>import csv

class OutputText(object):
    """OutputText the clustering result to various types of file such as txt, csv, and html.
    """
    @staticmethod
    def txt_perline(perline_file, perline_analysis, max_cluster_id, cluster_labels, original_logs):
        """Write clustering result per line to txt file. This clustering result is for all members in a node.

        Parameters
        ----------
        perline_file            : str
            Filename of output file per line.
        perline_analysis        : dict
            Dictionary of cluster label per log line.
        cluster_labels          : iterable
            List of cluster labels in auth.log.
        max_cluster_id          : int
            Maximum value of cluster identifier.
        original_logs           : iterable
            List of original event logs.
        """
        fopen = open(perline_file, 'w')
        for rowid, cluster_id in perline_analysis.iteritems():
            cluster_label = 'undefined' if cluster_id > max_cluster_id else cluster_labels[cluster_id]
            fopen.write(str(cluster_id) + '; ' + cluster_label + '; ' + original_logs[rowid])
        fopen.close()

    @staticmethod
    def txt_percluster(percluster_file, clusters, mode, graph, original_logs):
        """Write clustering result to txt file.

        Parameters
        ----------
        percluster_file         : str
            Filename of output file per cluster.
        clusters                : dict
            Dictionary of a list containing node identifier per cluster.
        mode                    : str
            Mode of clustering method, i.e., graph or text.
        graph                   : graph
            The graph which its clustering result to be written to a file.
        original_logs           : iterable
            List of original event logs.
        """
        f = open(percluster_file, 'w')
        for cluster_id, nodes in clusters.iteritems():
            f.write('Cluster #' + str(cluster_id) + '\n')
            for node in nodes:
                if mode == 'graph':
                    members = graph.node[node]['member']
                    for member in members:
                        f.write(original_logs[member])
                elif mode == 'text' or mode == 'text-h5':
                    f.write(original_logs[node] + '\n')
            f.write('\n')

        f.close()

    @staticmethod
    def csv_cluster_property(report_file, cluster_property, cluster_abstraction, anomaly_score, quadratic_score,
                             normalized_score, sentiment_score, anomaly_decision, evaluation_metrics):
        """Write cluster property to a csv file.

        Parameters
        ----------
        report_file             : str
            Filename and full path for anomaly score and cluster property.
        cluster_property        : dict
            Dictionary of cluster property.
        cluster_abstraction     : dict
            Dictionary of cluster abstraction.
        anomaly_score           : dict
            Dictionary of anomaly score per cluster.
        quadratic_score         : dict
            Dictionary of quadratic transformation of anomaly score.
        normalized_score        : dict
            Dictionary of normalized anomaly score.
        sentiment_score         : dict
            Dictionary of sentiment score per cluster.
        anomaly_decision        : dict
            Dictionary of anomaly decision per cluster.
        evaluation_metrics      : dict
            Dictionary of all evaluation metrics, both internal and external evaluation.
        """
        # write to csv
        f = open(report_file, 'wt')
        writer = csv.writer(f)

        # set header
        header = ('cluster_id', 'cluster_abstraction') + tuple(cluster_property[0].keys()) + \
                 ('anomaly_score', 'quadratic_score', 'normalized_score', 'sentiment_score', 'final_score', 'decision')
        writer.writerow(header)

        # write data
        for cluster_id, abstract in cluster_abstraction.iteritems():
            row = (cluster_id, abstract) + \
                  tuple(cluster_property[cluster_id].values()) + \
                  (anomaly_score[cluster_id], quadratic_score[cluster_id], normalized_score[cluster_id],
                   sentiment_score[cluster_id]) + anomaly_decision[cluster_id]
            writer.writerow(row)

        # write evaluation metrics
        writer.writerow(('',))
        writer.writerow(('Evaluation metrics of clustering:',))
        for metrics, value in evaluation_metrics.iteritems():
            writer.writerow((metrics, value))

        f.close()

    @staticmethod
    def txt_anomaly_perline(anomaly_decision, clusters, graph, anomaly_perline_file, original_logs):
        """Write anomaly detection result per log line.

        Parameters
        ----------
        anomaly_decision        : dict
            Dictionary of anomaly decision per cluster.
        clusters                : dict
            Dictionary of a list containing node identifier per cluster.
        graph                   : graph
            The graph which its clustering result to be written to a file.
        anomaly_perline_file    : str
            Filename for result of anomaly label per line.
        original_logs           : iterable
            List of original event logs.
        """
        decision_perlog = {}
        for cluster_id, decision in anomaly_decision.iteritems():
            for node in clusters[cluster_id]:
                members = graph.node[node]['member']
                for member in members:
                    decision_perlog[member] = decision

        # write to file
        f = open(anomaly_perline_file, 'w')
        for rowid, decision in decision_perlog.iteritems():
            f.write(decision[1] + '; ' + original_logs[rowid])
        f.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of the pyMor project (http://www.pymor.org).
# Copyright Holders: Felix Albrecht, Rene Milk, Stephan Rave
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

from __future__ import absolute_import, division, print_function
import abc
import types
import itertools
import contracts
import copy
import inspect
import functools
from types import NoneType

import numpy as np

from pymor.core import decorators, backports, logger
from pymor.core.exceptions import ConstError

class UIDProvider(object):
    def __init__(self):
        self.counter = 0
        import uuid
        self.prefix = '{}_'.format(uuid.uuid4())

    def __call__(self):
        uid = self.prefix + str(self.counter)
        self.counter += 1
        return uid

uid_provider = UIDProvider()

class UberMeta(abc.ABCMeta):

    def __init__(cls, name, bases, namespace):
        '''I copy my class docstring if deriving class has none. I tell base classes when I derive
        a new class from them. I publish a new contract type for each new class I create.
        '''
        doc = namespace.get("__doc__", None)
        if not doc:
            for base in cls.__mro__[1:]:
                if base.__doc__:
                    doc = base.__doc__
                    break
        cls.__doc__ = doc

        # monkey a new contract into the decorator module so checking for that type at runtime can work
        dname = (cls.__module__ + '.' + name).replace('__main__.', 'main.').replace('.', '_')
        if not dname in decorators.__dict__:
            decorators.__dict__[dname] = contracts.new_contract(dname, lambda x: isinstance(x, cls))

        # all bases except object get the derived class' name appended
        for base in [b for b in bases if b != object]:
            derived = cls
            # mangle the name to the base scope
            attribute = '_%s__implementors' % base.__name__
            if hasattr(base, attribute):
                getattr(base, attribute).append(derived)
            else:
                setattr(base, attribute, [derived])
        cls.logger = logger.getLogger('{}.{}'.format(cls.__module__.replace('__main__', 'pymor'), name))
        abc.ABCMeta.__init__(cls, name, bases, namespace)

    def __new__(cls, classname, bases, classdict):
        '''I copy contract decorations and docstrings from base class methods to deriving classes.
        I also forward "abstract{class|static}method" decorations in the base class to "{class|static}method"
        decorations in the new subclass.
        '''
        if 'init_arguments' in classdict:
            raise ValueError('init_arguments is a reserved class attribute for subclasses of BasicInterface')

        for attr, item in classdict.items():
            if isinstance(item, types.FunctionType):
                # first copy/fixup docs
                item.__doc__ = decorators.fixup_docstring(item.__doc__)
                base_doc = None
                contract_kwargs = dict()
                for base in bases:
                    has_contract = False
                    base_func = getattr(base, item.__name__, None)
                    if base_func:
                        base_doc = getattr(base_func, '__doc__', None)
                        has_contract = getattr(base_func, 'decorated', None) == 'contract'
                        contract_kwargs = getattr(base_func, 'contract_kwargs', contract_kwargs)
                    if base_doc:
                        doc = decorators.fixup_docstring(getattr(item, '__doc__', ''))
                        has_base_contract_docs = decorators.contains_contract(base_doc)
                        has_contract_docs = decorators.contains_contract(doc)
                        if has_base_contract_docs and not has_contract_docs:
                            base_doc += doc
                        elif not has_base_contract_docs and doc is not None:
                            base_doc = doc
                        item.__doc__ = base_doc
                    if has_contract:
                        # TODO why is the rebind necessary?
                        classdict['_H_%s' % attr] = item
                        contract_kwargs = contract_kwargs or dict()
                        p = decorators.contracts_decorate(item, modify_docstring=True, **contract_kwargs)
                        classdict[attr] = p
                    if (hasattr(base_func, "__isabstractstaticmethod__") and
                            getattr(base_func, "__isabstractstaticmethod__")):
                        classdict[attr] = staticmethod(classdict[attr])
                    if (hasattr(base_func, "__isabstractclassmethod__") and
                            getattr(base_func, "__isabstractclassmethod__")):
                        classdict[attr] = classmethod(classdict[attr])

        c = abc.ABCMeta.__new__(cls, classname, bases, classdict)

        # Beware! The following will probably break in python 3 if there are
        # keyword-only arguemnts
        args, varargs, keywords, defaults = inspect.getargspec(c.__init__)
        if varargs:
            raise NotImplementedError
        assert args[0] == 'self'
        c.init_arguments = tuple(args[1:])
        return c

class BasicInterface(object):
    ''' All other interface classes should be a subclass of mine.
    '''

    __metaclass__ = UberMeta
    _locked = False
    _lock_whitelist = set()

    def __init__(self):
        pass

    def __setattr__(self, key, value):
        '''depending on _locked state I delegate the setattr call to object or
        raise an Exception
        '''
        if not self._locked:
            return object.__setattr__(self, key, value)
        elif key.startswith('_') or key in self._lock_whitelist:
            return object.__setattr__(self, key, value)
        else:
            raise ConstError('Changing "%s" is not allowed in locked "%s"' % (key, self.__class__))

    @property
    def locked(self):
        return self._locked

    def lock(self, doit=True, whitelist=None):
        '''Calling me results in subsequent changes to members throwing errors'''
        object.__setattr__(self, '_locked', doit)
        if whitelist is not None:
            object.__setattr__(self, '_lock_whitelist', whitelist)

    def unlock(self):
        object.__setattr__(self, '_locked', False)

    @property
    def with_arguments(self):
        argnames = set(inspect.getargspec(self.__init__)[0])
        argnames.remove('self')
        for arg in argnames:
            if not hasattr(self, arg):
                return set()
        return argnames

    def with_(self, **kwargs):
        '''Returns a copy with changed attributes.

        Parameters
        ----------
        **kwargs
            Names of attributes to change with their new values. Each attribute name
            has to be contained in `with_arguments`.

        Returns
        -------
        Copy of `self` with changed attributes.
        '''
        if not set(kwargs.keys()) <= self.with_arguments:
            raise ConstError('Changing "{}" using with() is not allowed in {} (only "{}")'.format(
                kwargs.keys(), self.__class__, self.with_arguments))
        return self._with_via_init(kwargs)

    def _with_via_init(self, kwargs, new_class=None):
        '''Default implementation for with_ by calling __init__.

        Parameters which are missing in `kwargs` and for which there is no attribute in `self` are set
        to None.
        '''
        my_type = type(self) if new_class is None else new_class
        argnames = inspect.getargspec(my_type.__init__)[0]
        init_args = kwargs
        for arg in argnames:
            if arg == 'self':
                continue
            if arg not in init_args:
                init_args[arg] = getattr(self, arg, None)
        c = my_type(**init_args)
        if self.logging_disabled:
            c.disable_logging()
        return c

    _added_attributes = None
    def add_attributes(self, **kwargs):
        assert not any(hasattr(self, k) for k in kwargs)
        self.__dict__.update(kwargs)
        if self._added_attributes is None:
            self._added_attributes = kwargs.keys()
        else:
            self._added_attributes.extend(kwargs.keys())

    logging_disabled = False
    def disable_logging(self, doit=True):
        locked = self._locked
        self.unlock()
        if doit:
            self.logger = logger.dummy_logger
            self.logging_disabled = True
        else:
            self.logger = type(self).logger
            self.logging_disabled = False
        self.lock(locked)

    def enable_logging(self, doit=True):
        self.disable_logging(not doit)

    @classmethod
    def implementors(cls, descend=False):
        '''I return a, potentially empty, list of my subclass-objects.
        If descend is True I traverse my entire subclass hierarchy and return a flattened list.
        '''
        if not hasattr(cls, '_%s__implementors' % cls.__name__):
            return []
        level = getattr(cls, '_%s__implementors' % cls.__name__)
        if not descend:
            return level
        subtrees = itertools.chain.from_iterable([sub.implementors() for sub in level if sub.implementors() != []])
        level.extend(subtrees)
        return level

    @classmethod
    def implementor_names(cls, descend=False):
        '''For convenience I return a list of my implementor names instead of class objects'''
        return [c.__name__ for c in cls.implementors(descend)]

    @classmethod
    def has_interface_name(cls):
        name = cls.__name__
        return name.endswith('Interface')

    _uid = None
    @property
    def uid(self):
        if self._uid is None:
            self._uid = uid_provider()
        return self._uid

contract = decorators.contract
abstractmethod = abc.abstractmethod
abstractproperty = abc.abstractproperty

import sys
if sys.version_info >= (3, 1, 0):
    abstractclassmethod_base = abc.abstractclassmethod
    abstractstaticmethod_base = abc.abstractstaticmethod
else:
    # backport path for issue5867
    abstractclassmethod_base = backports.abstractclassmethod
    abstractstaticmethod_base = backports.abstractstaticmethod

class abstractclassmethod(abstractclassmethod_base):
    '''I mark my wrapped function with an additional __isabstractclassmethod__ member,
    where my abstractclassmethod_base sets __isabstractmethod__ = True.
    '''

    def __init__(self, callable_method):
        callable_method.__isabstractclassmethod__ = True
        super(abstractclassmethod, self).__init__(callable_method)

class abstractstaticmethod(abstractstaticmethod_base):
    '''I mark my wrapped function with an additional __isabstractstaticmethod__ member,
    where my abstractclassmethod_base sets __isabstractmethod__ = True.
    '''

    def __init__(self, callable_method):
        callable_method.__isabstractstaticmethod__ = True
        super(abstractstaticmethod, self).__init__(callable_method)

def _calculate_sid(obj, name):
    if hasattr(obj, 'sid'):
        return obj.sid
    else:
        t_obj = type(obj)
        if t_obj in (tuple, list):
            return tuple(_calculate_sid(o, '{}[{}]'.format(name, i)) for i, o in enumerate(obj))
        elif t_obj is dict:
            return tuple((k, _calculate_sid(v, '{}[{}]'.format(name, k))) for k, v in sorted(obj.iteritems()))
        elif t_obj in (NoneType, str, int, float, bool):
            return obj
        elif t_obj is np.ndarray:
            if obj.size < 64:
                return ('array', obj.shape,obj.dtype.str, obj.tostring())
            else:
                raise ValueError('sid calculation faild at large numpy array {}'.format(name))
        else:
            raise ValueError('sid calculation failed at {}={}'.format(name,type(obj)))

def inject_sid(obj, context, *args):
    try:
        sid = tuple((context, tuple(_calculate_sid(o, i) for i, o in enumerate(args))))
        obj.sid = sid
        ImmutableMeta.sids_created += 1
    except ValueError as e:
        obj.sid_failure = str(e)

    if isinstance(obj, BasicInterface):
        obj.lock()
    elif isinstance(obj, np.ndarray):
        obj.flags.writable = False

class generate_sid(object):

    def __init__(self, func=None, ignore=None):
        if func is not None and not hasattr(func, '__call__'):
            assert ignore is None
            ignore = func
            func = None
        if isinstance(ignore, str):
            ignore = (ignore,)
        self.ignore = ignore if ignore is not None else tuple()
        self.set_func(func)

    def set_func(self, func):
        self.func = func
        if func is not None:
            # Beware! The following will probably break in python 3 if there are
            # keyword-only arguemnts
            args, varargs, keywords, defaults  = inspect.getargspec(func)
            if varargs:
                raise NotImplementedError
            assert args[0] == 'self'
            self.func_arguments = tuple(args[1:])
            functools.update_wrapper(self, func)

    def __call__(self, *args, **kwargs):
        if self.func is None:
            assert len(kwargs) == 0 and len(args) == 1
            self.set_func(args[0])
            return self
        else:
            r = self.func(*args, **kwargs)
            assert isinstance(r, BasicInterface)

            if not hasattr(r, 'sid'):
                r.unlock()
                try:
                    kwargs.update((k, o) for k, o in itertools.izip(self.func_arguments, args[1:]))
                    kwarg_sids = tuple((k, _calculate_sid(o, k))
                                       for k, o in sorted(kwargs.iteritems())
                                       if k not in self.ignore)
                    r.sid = (type(r), args[0].sid, self.__name__,  kwarg_sids)
                    ImmutableMeta.sids_created += 1
                except (ValueError, AttributeError) as e:
                    instance.sid_failure = str(e)
                r.lock()

            return r

    def __get__(self, obj, obj_type):
        return functools.partial(self.__call__, obj)

def disable_sid_generation():
    if hasattr(ImmutableMeta, '__call__'):
        del ImmutableMeta.__call__

def enable_sid_generation():
    ImmutableMeta.__call__ = ImmutableMeta._call

class ImmutableMeta(UberMeta):

    sids_created = 0
    init_arguments_never_warn = ('name', 'caching')

    def __new__(cls, classname, bases, classdict):
        c = UberMeta.__new__(cls, classname, bases, classdict)
        init_arguments = c.init_arguments
        try:
            for a in c.sid_ignore:
                if a not in init_arguments and a not in ImmutableMeta.init_arguments_never_warn:
                    raise ValueError(a)
        except ValueError as e:
            c.logger.warn('sid_ignore contains "{}" which is not an __init__ argument!'.format(e))
        return c

    def _call(self, *args, **kwargs):
        instance = super(ImmutableMeta, self).__call__(*args, **kwargs)
        if instance.calculate_sid:
            try:
                kwargs.update((k, o) for k, o in itertools.izip(instance.init_arguments, args))
                kwarg_sids = tuple((k, _calculate_sid(o, k))
                                   for k, o in sorted(kwargs.iteritems())
                                   if k not in instance.sid_ignore)
                instance.sid = (type(instance), kwarg_sids)
                ImmutableMeta.sids_created += 1
            except ValueError as e:
                instance.sid_failure = str(e)
        else:
            instance.sid_failure = 'disabled'

        instance.lock()
        return instance

    __call__ = _call

class ImmutableInterface(BasicInterface):
    __metaclass__ = ImmutableMeta
    calculate_sid = True
    sid_ignore = ('name', 'caching')

    # Unlocking an immutable object will result in the deletion of its sid.
    # However, this will not delete the sids of objects referencing it.
    # You really should not unlock an object unless you really know what
    # you are doing. (One exception might be the modification of a newly
    # created copy of an immutable object.)
    def lock(self, doit=True, whitelist=None):
        super(ImmutableInterface, self).lock(doit, whitelist)
        if not self.locked and hasattr(self, 'sid'):
            del self.sid
            self.sid_failure = 'unlocked'

    def unlock(self):
        super(ImmutableInterface, self).unlock()
        if hasattr(self, 'sid'):
            del self.sid
            self.sid_failure = 'unlocked'
<commit_msg>[core.interfaces] add _with_arguments_error<commit_after># -*- coding: utf-8 -*-
# This file is part of the pyMor project (http://www.pymor.org).
# Copyright Holders: Felix Albrecht, Rene Milk, Stephan Rave
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

from __future__ import absolute_import, division, print_function
import abc
import types
import itertools
import contracts
import copy
import inspect
import functools
from types import NoneType

import numpy as np

from pymor.core import decorators, backports, logger
from pymor.core.exceptions import ConstError

class UIDProvider(object):
    def __init__(self):
        self.counter = 0
        import uuid
        self.prefix = '{}_'.format(uuid.uuid4())

    def __call__(self):
        uid = self.prefix + str(self.counter)
        self.counter += 1
        return uid

uid_provider = UIDProvider()

class UberMeta(abc.ABCMeta):

    def __init__(cls, name, bases, namespace):
        '''I copy my class docstring if deriving class has none. I tell base classes when I derive
        a new class from them. I publish a new contract type for each new class I create.
        '''
        doc = namespace.get("__doc__", None)
        if not doc:
            for base in cls.__mro__[1:]:
                if base.__doc__:
                    doc = base.__doc__
                    break
        cls.__doc__ = doc

        # monkey a new contract into the decorator module so checking for that type at runtime can work
        dname = (cls.__module__ + '.' + name).replace('__main__.', 'main.').replace('.', '_')
        if not dname in decorators.__dict__:
            decorators.__dict__[dname] = contracts.new_contract(dname, lambda x: isinstance(x, cls))

        # all bases except object get the derived class' name appended
        for base in [b for b in bases if b != object]:
            derived = cls
            # mangle the name to the base scope
            attribute = '_%s__implementors' % base.__name__
            if hasattr(base, attribute):
                getattr(base, attribute).append(derived)
            else:
                setattr(base, attribute, [derived])
        cls.logger = logger.getLogger('{}.{}'.format(cls.__module__.replace('__main__', 'pymor'), name))
        abc.ABCMeta.__init__(cls, name, bases, namespace)

    def __new__(cls, classname, bases, classdict):
        '''I copy contract decorations and docstrings from base class methods to deriving classes.
        I also forward "abstract{class|static}method" decorations in the base class to "{class|static}method"
        decorations in the new subclass.
        '''
        if 'init_arguments' in classdict:
            raise ValueError('init_arguments is a reserved class attribute for subclasses of BasicInterface')

        for attr, item in classdict.items():
            if isinstance(item, types.FunctionType):
                # first copy/fixup docs
                item.__doc__ = decorators.fixup_docstring(item.__doc__)
                base_doc = None
                contract_kwargs = dict()
                for base in bases:
                    has_contract = False
                    base_func = getattr(base, item.__name__, None)
                    if base_func:
                        base_doc = getattr(base_func, '__doc__', None)
                        has_contract = getattr(base_func, 'decorated', None) == 'contract'
                        contract_kwargs = getattr(base_func, 'contract_kwargs', contract_kwargs)
                    if base_doc:
                        doc = decorators.fixup_docstring(getattr(item, '__doc__', ''))
                        has_base_contract_docs = decorators.contains_contract(base_doc)
                        has_contract_docs = decorators.contains_contract(doc)
                        if has_base_contract_docs and not has_contract_docs:
                            base_doc += doc
                        elif not has_base_contract_docs and doc is not None:
                            base_doc = doc
                        item.__doc__ = base_doc
                    if has_contract:
                        # TODO why is the rebind necessary?
                        classdict['_H_%s' % attr] = item
                        contract_kwargs = contract_kwargs or dict()
                        p = decorators.contracts_decorate(item, modify_docstring=True, **contract_kwargs)
                        classdict[attr] = p
                    if (hasattr(base_func, "__isabstractstaticmethod__") and
                            getattr(base_func, "__isabstractstaticmethod__")):
                        classdict[attr] = staticmethod(classdict[attr])
                    if (hasattr(base_func, "__isabstractclassmethod__") and
                            getattr(base_func, "__isabstractclassmethod__")):
                        classdict[attr] = classmethod(classdict[attr])

        c = abc.ABCMeta.__new__(cls, classname, bases, classdict)

        # Beware! The following will probably break in python 3 if there are
        # keyword-only arguemnts
        args, varargs, keywords, defaults = inspect.getargspec(c.__init__)
        if varargs:
            raise NotImplementedError
        assert args[0] == 'self'
        c.init_arguments = tuple(args[1:])
        return c

class BasicInterface(object):
    ''' All other interface classes should be a subclass of mine.
    '''

    __metaclass__ = UberMeta
    _locked = False
    _lock_whitelist = set()

    def __init__(self):
        pass

    def __setattr__(self, key, value):
        '''depending on _locked state I delegate the setattr call to object or
        raise an Exception
        '''
        if not self._locked:
            return object.__setattr__(self, key, value)
        elif key.startswith('_') or key in self._lock_whitelist:
            return object.__setattr__(self, key, value)
        else:
            raise ConstError('Changing "%s" is not allowed in locked "%s"' % (key, self.__class__))

    @property
    def locked(self):
        return self._locked

    def lock(self, doit=True, whitelist=None):
        '''Calling me results in subsequent changes to members throwing errors'''
        object.__setattr__(self, '_locked', doit)
        if whitelist is not None:
            object.__setattr__(self, '_lock_whitelist', whitelist)

    def unlock(self):
        object.__setattr__(self, '_locked', False)

    @property
    def with_arguments(self):
        argnames = set(inspect.getargspec(self.__init__)[0])
        argnames.remove('self')
        for arg in argnames:
            if not hasattr(self, arg):
                self._with_arguments_error = "Instance does not have attribute for __init__ argument '{}'".format(arg)
                return set()
        return argnames

    def with_(self, **kwargs):
        '''Returns a copy with changed attributes.

        Parameters
        ----------
        **kwargs
            Names of attributes to change with their new values. Each attribute name
            has to be contained in `with_arguments`.

        Returns
        -------
        Copy of `self` with changed attributes.
        '''
        with_arguments = self.with_arguments      # ensure that property is called first
        if hasattr(self, '_with_arguments_error'):
            raise ConstError('Using with_ is not possible because of the following Error: '
                             + self._with_arguments_error)
        if not set(kwargs.keys()) <= with_arguments:
            raise ConstError('Changing "{}" using with() is not allowed in {} (only "{}")'.format(
                kwargs.keys(), self.__class__, self.with_arguments))
        return self._with_via_init(kwargs)

    def _with_via_init(self, kwargs, new_class=None):
        '''Default implementation for with_ by calling __init__.

        Parameters which are missing in `kwargs` and for which there is no attribute in `self` are set
        to None.
        '''
        my_type = type(self) if new_class is None else new_class
        argnames = inspect.getargspec(my_type.__init__)[0]
        init_args = kwargs
        for arg in argnames:
            if arg == 'self':
                continue
            if arg not in init_args:
                init_args[arg] = getattr(self, arg, None)
        c = my_type(**init_args)
        if self.logging_disabled:
            c.disable_logging()
        return c

    _added_attributes = None
    def add_attributes(self, **kwargs):
        assert not any(hasattr(self, k) for k in kwargs)
        self.__dict__.update(kwargs)
        if self._added_attributes is None:
            self._added_attributes = kwargs.keys()
        else:
            self._added_attributes.extend(kwargs.keys())

    logging_disabled = False
    def disable_logging(self, doit=True):
        locked = self._locked
        self.unlock()
        if doit:
            self.logger = logger.dummy_logger
            self.logging_disabled = True
        else:
            self.logger = type(self).logger
            self.logging_disabled = False
        self.lock(locked)

    def enable_logging(self, doit=True):
        self.disable_logging(not doit)

    @classmethod
    def implementors(cls, descend=False):
        '''I return a, potentially empty, list of my subclass-objects.
        If descend is True I traverse my entire subclass hierarchy and return a flattened list.
        '''
        if not hasattr(cls, '_%s__implementors' % cls.__name__):
            return []
        level = getattr(cls, '_%s__implementors' % cls.__name__)
        if not descend:
            return level
        subtrees = itertools.chain.from_iterable([sub.implementors() for sub in level if sub.implementors() != []])
        level.extend(subtrees)
        return level

    @classmethod
    def implementor_names(cls, descend=False):
        '''For convenience I return a list of my implementor names instead of class objects'''
        return [c.__name__ for c in cls.implementors(descend)]

    @classmethod
    def has_interface_name(cls):
        name = cls.__name__
        return name.endswith('Interface')

    _uid = None
    @property
    def uid(self):
        if self._uid is None:
            self._uid = uid_provider()
        return self._uid

contract = decorators.contract
abstractmethod = abc.abstractmethod
abstractproperty = abc.abstractproperty

import sys
if sys.version_info >= (3, 1, 0):
    abstractclassmethod_base = abc.abstractclassmethod
    abstractstaticmethod_base = abc.abstractstaticmethod
else:
    # backport path for issue5867
    abstractclassmethod_base = backports.abstractclassmethod
    abstractstaticmethod_base = backports.abstractstaticmethod

class abstractclassmethod(abstractclassmethod_base):
    '''I mark my wrapped function with an additional __isabstractclassmethod__ member,
    where my abstractclassmethod_base sets __isabstractmethod__ = True.
    '''

    def __init__(self, callable_method):
        callable_method.__isabstractclassmethod__ = True
        super(abstractclassmethod, self).__init__(callable_method)

class abstractstaticmethod(abstractstaticmethod_base):
    '''I mark my wrapped function with an additional __isabstractstaticmethod__ member,
    where my abstractclassmethod_base sets __isabstractmethod__ = True.
    '''

    def __init__(self, callable_method):
        callable_method.__isabstractstaticmethod__ = True
        super(abstractstaticmethod, self).__init__(callable_method)

def _calculate_sid(obj, name):
    if hasattr(obj, 'sid'):
        return obj.sid
    else:
        t_obj = type(obj)
        if t_obj in (tuple, list):
            return tuple(_calculate_sid(o, '{}[{}]'.format(name, i)) for i, o in enumerate(obj))
        elif t_obj is dict:
            return tuple((k, _calculate_sid(v, '{}[{}]'.format(name, k))) for k, v in sorted(obj.iteritems()))
        elif t_obj in (NoneType, str, int, float, bool):
            return obj
        elif t_obj is np.ndarray:
            if obj.size < 64:
                return ('array', obj.shape,obj.dtype.str, obj.tostring())
            else:
                raise ValueError('sid calculation faild at large numpy array {}'.format(name))
        else:
            raise ValueError('sid calculation failed at {}={}'.format(name,type(obj)))

def inject_sid(obj, context, *args):
    try:
        sid = tuple((context, tuple(_calculate_sid(o, i) for i, o in enumerate(args))))
        obj.sid = sid
        ImmutableMeta.sids_created += 1
    except ValueError as e:
        obj.sid_failure = str(e)

    if isinstance(obj, BasicInterface):
        obj.lock()
    elif isinstance(obj, np.ndarray):
        obj.flags.writable = False

class generate_sid(object):

    def __init__(self, func=None, ignore=None):
        if func is not None and not hasattr(func, '__call__'):
            assert ignore is None
            ignore = func
            func = None
        if isinstance(ignore, str):
            ignore = (ignore,)
        self.ignore = ignore if ignore is not None else tuple()
        self.set_func(func)

    def set_func(self, func):
        self.func = func
        if func is not None:
            # Beware! The following will probably break in python 3 if there are
            # keyword-only arguemnts
            args, varargs, keywords, defaults  = inspect.getargspec(func)
            if varargs:
                raise NotImplementedError
            assert args[0] == 'self'
            self.func_arguments = tuple(args[1:])
            functools.update_wrapper(self, func)

    def __call__(self, *args, **kwargs):
        if self.func is None:
            assert len(kwargs) == 0 and len(args) == 1
            self.set_func(args[0])
            return self
        else:
            r = self.func(*args, **kwargs)
            assert isinstance(r, BasicInterface)

            if not hasattr(r, 'sid'):
                r.unlock()
                try:
                    kwargs.update((k, o) for k, o in itertools.izip(self.func_arguments, args[1:]))
                    kwarg_sids = tuple((k, _calculate_sid(o, k))
                                       for k, o in sorted(kwargs.iteritems())
                                       if k not in self.ignore)
                    r.sid = (type(r), args[0].sid, self.__name__,  kwarg_sids)
                    ImmutableMeta.sids_created += 1
                except (ValueError, AttributeError) as e:
                    instance.sid_failure = str(e)
                r.lock()

            return r

    def __get__(self, obj, obj_type):
        return functools.partial(self.__call__, obj)

def disable_sid_generation():
    if hasattr(ImmutableMeta, '__call__'):
        del ImmutableMeta.__call__

def enable_sid_generation():
    ImmutableMeta.__call__ = ImmutableMeta._call

class ImmutableMeta(UberMeta):

    sids_created = 0
    init_arguments_never_warn = ('name', 'caching')

    def __new__(cls, classname, bases, classdict):
        c = UberMeta.__new__(cls, classname, bases, classdict)
        init_arguments = c.init_arguments
        try:
            for a in c.sid_ignore:
                if a not in init_arguments and a not in ImmutableMeta.init_arguments_never_warn:
                    raise ValueError(a)
        except ValueError as e:
            c.logger.warn('sid_ignore contains "{}" which is not an __init__ argument!'.format(e))
        return c

    def _call(self, *args, **kwargs):
        instance = super(ImmutableMeta, self).__call__(*args, **kwargs)
        if instance.calculate_sid:
            try:
                kwargs.update((k, o) for k, o in itertools.izip(instance.init_arguments, args))
                kwarg_sids = tuple((k, _calculate_sid(o, k))
                                   for k, o in sorted(kwargs.iteritems())
                                   if k not in instance.sid_ignore)
                instance.sid = (type(instance), kwarg_sids)
                ImmutableMeta.sids_created += 1
            except ValueError as e:
                instance.sid_failure = str(e)
        else:
            instance.sid_failure = 'disabled'

        instance.lock()
        return instance

    __call__ = _call

class ImmutableInterface(BasicInterface):
    __metaclass__ = ImmutableMeta
    calculate_sid = True
    sid_ignore = ('name', 'caching')

    # Unlocking an immutable object will result in the deletion of its sid.
    # However, this will not delete the sids of objects referencing it.
    # You really should not unlock an object unless you really know what
    # you are doing. (One exception might be the modification of a newly
    # created copy of an immutable object.)
    def lock(self, doit=True, whitelist=None):
        super(ImmutableInterface, self).lock(doit, whitelist)
        if not self.locked and hasattr(self, 'sid'):
            del self.sid
            self.sid_failure = 'unlocked'

    def unlock(self):
        super(ImmutableInterface, self).unlock()
        if hasattr(self, 'sid'):
            del self.sid
            self.sid_failure = 'unlocked'
<|endoftext|>


<commit_before>#! /usr/bin/env python3.6
# coding: utf-8
from kamera.logger import log

from pathlib import Path
import datetime as dt
from functools import partial

import pytz
from timezonefinderL import TimezoneFinder
import dropbox
import requests
import redis

from kamera import config
from kamera import image_processing
from kamera.mediatypes import KameraEntry

from typing import Callable, Optional, Dict

class Cloud:
    dbx_cache: Dict[str, dropbox.Dropbox] = {}
    settings_cache: Dict[str, Dict] = {}

    redis_client: redis.Redis = None

    def __init__(self, account_id):
        if self.redis_client is None:
            self.redis_client = redis.from_url(config.redis_url)

        self.account_id = account_id
        try:
            self.dbx = self.dbx_cache[account_id]
            self.settings = self.settings_cache[account_id]
        except KeyError:
            self.dbx = dropbox.Dropbox(config.get_dbx_token(self.redis_client, account_id))
            self.dbx_cache[account_id] = self.dbx
            self.settings = config.Settings(self.dbx)
            self.settings_cache[account_id] = self.settings

    def _execute_transfer(self, transfer_func: Callable, destination_folder: Path):
        try:
            transfer_func()
        except requests.exceptions.SSLError:
            log.info("Encountered SSL error during transfer. Trying again")
            transfer_func()
        except dropbox.exceptions.BadInputError:
            log.info(f"Making folder: {destination_folder}")
            self.dbx.files_create_folder(destination_folder.as_posix())
            transfer_func()

    def move_entry(
            self,
            from_path: Path,
            out_dir: Path,
            date: Optional[dt.datetime] = None
    ):
        if date is not None:
            destination = (
                out_dir /
                str(date.year) /
                self.settings.folder_names[date.month] /
                from_path.name
            )
        else:
            destination = out_dir / from_path.name

        transfer_func = partial(
            self.dbx.files_move,
            from_path=from_path.as_posix(),
            to_path=destination.as_posix(),
            autorename=True
        )

        log.info(f"{from_path.stem}: Moving to dest: {destination}")
        self._execute_transfer(transfer_func, destination.parent)

    def copy_entry(
            self,
            from_path: Path,
            out_dir: Path,
            date: dt.datetime
    ):
        destination = (
            out_dir /
            str(date.year) /
            self.settings.folder_names[date.month] /
            from_path.name
        )

        transfer_func = partial(
            self.dbx.files_copy,
            from_path=from_path.as_posix(),
            to_path=destination.as_posix(),
            autorename=True
        )

        log.info(f"{from_path.stem}: Copying to dest: {destination}")
        self._execute_transfer(transfer_func, destination.parent)

    def upload_entry(
            self,
            from_path: Path,
            new_data: bytes,
            out_dir: Path,
            date: dt.datetime
    ):
        new_name = from_path.with_suffix(".jpg").name
        destination = (
            out_dir /
            str(date.year) /
            self.settings.folder_names[date.month] /
            new_name
        )

        transfer_func = partial(
            self.dbx.files_upload,
            f=new_data,
            path=destination.as_posix(),
            autorename=True
        )

        log.info(f"{destination.stem}: Uploading to dest: {destination}")
        self._execute_transfer(transfer_func, destination.parent)

    def download_entry(self, path_str: str):
        try:
            return self.dbx.files_download(path_str)
        except requests.exceptions.SSLError:
            log.info("Encountered SSL error during transfer. Trying again")
            return self.dbx.files_download(path_str)

def parse_date(entry: KameraEntry, settings: config.Settings) -> dt.datetime:
    naive_date = entry.time_taken if entry.time_taken is not None else entry.client_modified
    utc_date = naive_date.replace(tzinfo=dt.timezone.utc)
    if entry.coordinates is not None:
        img_tz = TimezoneFinder().timezone_at(
            lat=entry.coordinates.latitude,
            lng=entry.coordinates.longitude
        )
        if img_tz:
            local_date = utc_date.astimezone(tz=pytz.timezone(img_tz))
            return local_date
    local_date = utc_date.astimezone(tz=pytz.timezone(settings.default_tz))
    return local_date

def process_entry(
        entry: KameraEntry,
        out_dir: Path,
        backup_dir: Path,
        error_dir: Path,
):
    log.info(f"{entry}: Processing")
    start_time = dt.datetime.now()
    cloud = Cloud(entry.account_id)
    try:
        date = parse_date(entry, cloud.settings)
        if entry.path.suffix.lower() in config.video_extensions:
            cloud.copy_entry(entry.path, out_dir, date)
        elif entry.path.suffix.lower() in config.image_extensions:
            _, response = cloud.download_entry(entry.path.as_posix())
            new_data = image_processing.main(
                data=response.raw.data,
                filepath=entry.path,
                date=date,
                settings=cloud.settings,
                coordinates=entry.coordinates,
                dimensions=entry.dimensions,
            )

            if new_data is None:
                cloud.copy_entry(entry.path, out_dir, date)
            else:
                cloud.upload_entry(entry.path, new_data, out_dir, date)
        else:
            return
        cloud.move_entry(entry.path, out_dir=backup_dir, date=date)
    except Exception:
        log.exception(f"Exception occured, moving to Error subfolder: {entry}")
        cloud.move_entry(entry.path, out_dir=error_dir)
    finally:
        end_time = dt.datetime.now()
        duration = end_time - start_time
        log.info(f"{entry}, duration: {duration}")
        log.info("\n")

def run_once(
        in_dir: Path,
        out_dir: Path,
        backup_dir: Path,
        error_dir: Path,
) -> None:
    entries = Cloud.list_entries(in_dir)
    for entry in entries:
        process_entry(
            entry=entry,
            out_dir=out_dir,
            backup_dir=backup_dir,
            error_dir=error_dir,
        )
<commit_msg>remove run_once function<commit_after>#! /usr/bin/env python3.6
# coding: utf-8
from kamera.logger import log

from pathlib import Path
import datetime as dt
from functools import partial

import pytz
from timezonefinderL import TimezoneFinder
import dropbox
import requests
import redis

from kamera import config
from kamera import image_processing
from kamera.mediatypes import KameraEntry

from typing import Callable, Optional, Dict

class Cloud:
    dbx_cache: Dict[str, dropbox.Dropbox] = {}
    settings_cache: Dict[str, Dict] = {}

    redis_client: redis.Redis = None

    def __init__(self, account_id):
        if self.redis_client is None:
            self.redis_client = redis.from_url(config.redis_url)

        self.account_id = account_id
        try:
            self.dbx = self.dbx_cache[account_id]
            self.settings = self.settings_cache[account_id]
        except KeyError:
            self.dbx = dropbox.Dropbox(config.get_dbx_token(self.redis_client, account_id))
            self.dbx_cache[account_id] = self.dbx
            self.settings = config.Settings(self.dbx)
            self.settings_cache[account_id] = self.settings

    def _execute_transfer(self, transfer_func: Callable, destination_folder: Path):
        try:
            transfer_func()
        except requests.exceptions.SSLError:
            log.info("Encountered SSL error during transfer. Trying again")
            transfer_func()
        except dropbox.exceptions.BadInputError:
            log.info(f"Making folder: {destination_folder}")
            self.dbx.files_create_folder(destination_folder.as_posix())
            transfer_func()

    def move_entry(
            self,
            from_path: Path,
            out_dir: Path,
            date: Optional[dt.datetime] = None
    ):
        if date is not None:
            destination = (
                out_dir /
                str(date.year) /
                self.settings.folder_names[date.month] /
                from_path.name
            )
        else:
            destination = out_dir / from_path.name

        transfer_func = partial(
            self.dbx.files_move,
            from_path=from_path.as_posix(),
            to_path=destination.as_posix(),
            autorename=True
        )

        log.info(f"{from_path.stem}: Moving to dest: {destination}")
        self._execute_transfer(transfer_func, destination.parent)

    def copy_entry(
            self,
            from_path: Path,
            out_dir: Path,
            date: dt.datetime
    ):
        destination = (
            out_dir /
            str(date.year) /
            self.settings.folder_names[date.month] /
            from_path.name
        )

        transfer_func = partial(
            self.dbx.files_copy,
            from_path=from_path.as_posix(),
            to_path=destination.as_posix(),
            autorename=True
        )

        log.info(f"{from_path.stem}: Copying to dest: {destination}")
        self._execute_transfer(transfer_func, destination.parent)

    def upload_entry(
            self,
            from_path: Path,
            new_data: bytes,
            out_dir: Path,
            date: dt.datetime
    ):
        new_name = from_path.with_suffix(".jpg").name
        destination = (
            out_dir /
            str(date.year) /
            self.settings.folder_names[date.month] /
            new_name
        )

        transfer_func = partial(
            self.dbx.files_upload,
            f=new_data,
            path=destination.as_posix(),
            autorename=True
        )

        log.info(f"{destination.stem}: Uploading to dest: {destination}")
        self._execute_transfer(transfer_func, destination.parent)

    def download_entry(self, path_str: str):
        try:
            return self.dbx.files_download(path_str)
        except requests.exceptions.SSLError:
            log.info("Encountered SSL error during transfer. Trying again")
            return self.dbx.files_download(path_str)

def parse_date(entry: KameraEntry, settings: config.Settings) -> dt.datetime:
    naive_date = entry.time_taken if entry.time_taken is not None else entry.client_modified
    utc_date = naive_date.replace(tzinfo=dt.timezone.utc)
    if entry.coordinates is not None:
        img_tz = TimezoneFinder().timezone_at(
            lat=entry.coordinates.latitude,
            lng=entry.coordinates.longitude
        )
        if img_tz:
            local_date = utc_date.astimezone(tz=pytz.timezone(img_tz))
            return local_date
    local_date = utc_date.astimezone(tz=pytz.timezone(settings.default_tz))
    return local_date

def process_entry(
        entry: KameraEntry,
        out_dir: Path,
        backup_dir: Path,
        error_dir: Path,
):
    log.info(f"{entry}: Processing")
    start_time = dt.datetime.now()
    cloud = Cloud(entry.account_id)
    try:
        date = parse_date(entry, cloud.settings)
        if entry.path.suffix.lower() in config.video_extensions:
            cloud.copy_entry(entry.path, out_dir, date)
        elif entry.path.suffix.lower() in config.image_extensions:
            _, response = cloud.download_entry(entry.path.as_posix())
            new_data = image_processing.main(
                data=response.raw.data,
                filepath=entry.path,
                date=date,
                settings=cloud.settings,
                coordinates=entry.coordinates,
                dimensions=entry.dimensions,
            )

            if new_data is None:
                cloud.copy_entry(entry.path, out_dir, date)
            else:
                cloud.upload_entry(entry.path, new_data, out_dir, date)
        else:
            return
        cloud.move_entry(entry.path, out_dir=backup_dir, date=date)
    except Exception:
        log.exception(f"Exception occured, moving to Error subfolder: {entry}")
        cloud.move_entry(entry.path, out_dir=error_dir)
    finally:
        end_time = dt.datetime.now()
        duration = end_time - start_time
        log.info(f"{entry}, duration: {duration}")
        log.info("\n")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import zipfile

os.chdir(os.path.dirname(__file__))

me = os.path.basename(__file__)
output = "%s.xpi" % os.path.basename(os.path.abspath("."))

files = []

def recurse(path):
	for filename in os.listdir(path):
		if filename.startswith(".") \
		or filename.endswith("~") \
		or filename == output \
		or filename == me:
			continue
		relpath = os.path.join(path, filename)

		if os.path.isdir(relpath):
			recurse(relpath)
		else:
			files.append(relpath)

recurse(".")

fh = zipfile.ZipFile(output, "w")
for filename in sorted(files):
	print(filename)
	fh.write(filename)

<commit_msg>remove build script (use web-ext build instead)<commit_after><|endoftext|>