<commit_before>from datetime import datetime

from django.db import models
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.contrib.sites.models import Site
from django.template.defaultfilters import slugify
from django.core.urlresolvers import reverse
from django.contrib.redirects.models import Redirect

from ella.ellaadmin.utils import admin_url
from ella.core.managers import ListingManager, HitCountManager, PlacementManager, RelatedManager
from ella.core.cache import get_cached_object, get_cached_list, CachedGenericForeignKey
from ella.core.models.main import Category, Author, Source
from ella.photos.models import Photo
from ella.core.box import Box

PUBLISH_FROM_WHEN_EMPTY = datetime(3000, 1, 1)

def PublishableBox(publishable, box_type, nodelist, model=None):
    "add some content type info of self.target"
    if not model:
        model = publishable.content_type.model_class()
    box_class = model.box_class

    if box_class == PublishableBox:
        box_class = Box
    return box_class(publishable, box_type, nodelist, model=model)

class Publishable(models.Model):
    """
    Base class for all object that can be published in ella
    """
    box_class = staticmethod(PublishableBox)

    content_type = models.ForeignKey(ContentType)

    category = models.ForeignKey(Category, verbose_name=_('Category'))

    # Titles
    title = models.CharField(_('Title'), max_length=255)
    slug = models.SlugField(_('Slug'), max_length=255)

    # Authors and Sources
    authors = models.ManyToManyField(Author, verbose_name=_('Authors'))
    source = models.ForeignKey(Source, blank=True, null=True, verbose_name=_('Source'))

    # Main Photo
    photo = models.ForeignKey(Photo, blank=True, null=True, verbose_name=_('Photo'))

    # Description
    description = models.TextField(_('Description'), blank=True)

    # denormalized fields
    # the placement's publish_from
    publish_from = models.DateTimeField(_('Publish from'), editable=False, default=PUBLISH_FROM_WHEN_EMPTY)

    class Meta:
        app_label = 'core'
        verbose_name = _('Publishable object')
        verbose_name_plural = _('Publishable objects')

    @property
    def target(self):
        if not hasattr(self, '_target'):
            self._target = self.content_type.get_object_for_this_type(pk=self.pk)
        return self._target

    if 'ella.comments' in settings.INSTALLED_APPS:
        from ella.comments.models import Comment
        comments = generic.GenericRelation(Comment, object_id_field='target_id', content_type_field='target_ct')

    @property
    def main_placement(self):
        " Return object's main placement, that is the object's placement in its primary category "
        if hasattr(self, '_main_placement'):
            return self._main_placement

        current_site = Site.objects.get_current()

        # TODO: what if have got multiple listings on one site?
        placements = get_cached_list(
                Placement,
                publishable=self.pk,
                category__site=current_site,
            )
        if placements:
            if len(placements) == 1:
                return placements[0]
            else:
                # with multiple listings, one with first publish_from
                # primary
                first_published = None
                for placement in placements:
                    if first_published is None or placement.publish_from < first_published.publish_from:
                        first_published = placement

                assert first_published is not None
                return first_published

        try:
            # TODO - check and if we don't have category, take the only placement that exists in current site
            self._main_placement = get_cached_object(
                    Placement,
                    publishable=self.pk,
                    category=self.category_id
                )
        except Placement.DoesNotExist:
            self._main_placement = None

        return self._main_placement

    def get_absolute_url(self, domain=False):
        " Get object's URL. "
        placement = self.main_placement
        if placement:
            return placement.get_absolute_url(domain=domain)

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_admin_url(self):
        return admin_url(self)

    def save(self, **kwargs):
        self.content_type = ContentType.objects.get_for_model(self)
        if self.pk and hasattr(self, 'slug'): # only run on update
            # get old self
            old_slug = self.__class__._default_manager.get(pk=self.pk).slug
            # the slug has changed
            if old_slug != self.slug:
                for plc in Placement.objects.filter(publishable=self):
                    if plc.slug == old_slug:
                        plc.slug = self.slug
                        plc.save(force_update=True)
        return super(Publishable, self).save(**kwargs)

    def delete(self):
        url = self.get_absolute_url()
        Redirect.objects.filter(new_path=url).delete()
        return super(Publishable, self).delete()

    ##
    # various metadata
    ##
    def get_category(self):
        " Get object's primary Category."
        return get_cached_object(Category, pk=self.category_id)

    def get_photo(self):
        " Get object's Photo. "
        if not hasattr(self, '_photo'):
            try:
                self._photo = get_cached_object(Photo, pk=self.photo_id)
            except Photo.DoesNotExist:
                self._photo = None
        return self._photo

    def get_description(self):
        return self.description

    def get_title(self):
        myTitle=self.title
        if myTitle:
            return '%s' % (self.title,)
        else:
            return '%s' % (self.draw_title(),)

    def get_text(self):
        return self.text

    def __unicode__(self):
        return self.get_title()

# FIXME find another way to register!
if 'tagging' in settings.INSTALLED_APPS:
    import tagging
    tagging.register(Publishable)

class Placement(models.Model):
    # listing's target - a Publishable object
    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable object'))
    category = models.ForeignKey(Category, verbose_name=_('Category'), db_index=True)
    publish_from = models.DateTimeField(_("Start of visibility")) #, default=datetime.now)
    publish_to = models.DateTimeField(_("End of visibility"), null=True, blank=True)
    slug = models.SlugField(_('Slug'), max_length=255, blank=True)

    static = models.BooleanField(_('static'), default=False)

    objects = PlacementManager()

    class Meta:
        app_label = 'core'
        verbose_name = _('Placement')
        verbose_name_plural = _('Placements')

    def __unicode__(self):
        try:
            return u'%s placed in %s' % (self.publishable, self.category)
        except:
            return 'Broken placement'

    def is_active(self):
        "Return True if the Placement is currently active."
        now = datetime.now()
        return now > self.publish_from and (self.publish_to is None or now < self.publish_to)

    def delete(self):
        super(Placement, self).delete()
        try:
            publish_from = Placement.objects.filter(publishable=self.publishable).order_by('publish_from')[0].publish_from
        except IndexError, e:
            publish_from = PUBLISH_FROM_WHEN_EMPTY

        self.publishable.publish_from = publish_from
        Publishable.objects.filter(pk=self.publishable_id).update(publish_from=publish_from)

    def save(self, **kwargs):
        " If Listing is created, we create HitCount object "

        if not self.slug:
            self.slug = self.publishable.slug

        if self.pk:
            old_self = Placement.objects.get(pk=self.pk)

            old_path = old_self.get_absolute_url()
            new_path = self.get_absolute_url()

            if old_path != new_path and new_path:
                redirect, created = Redirect.objects.get_or_create(old_path=old_path, site=self.category.site)
                redirect.new_path=new_path
                redirect.save(force_update=True)
                Redirect.objects.filter(new_path=old_path).exclude(pk=redirect.pk).update(new_path=new_path)

        # First, save Placement
        super(Placement, self).save(**kwargs)
        # Then, save HitCount (needs placement_id)
        hc, created = HitCount.objects.get_or_create(placement=self, defaults={'hits': 0})

        # store the publish_from on the publishable for performance in the admin
        if self.publishable.publish_from > self.publish_from:
            self.publishable.publish_from = self.publish_from
            Publishable.objects.filter(pk=self.publishable_id).update(publish_from=self.publish_from)

    def get_absolute_url(self, domain=False):
        obj = self.publishable
        category = get_cached_object(Category, pk=self.category_id)

        kwargs = {
            'content_type' : slugify(obj.content_type.model_class()._meta.verbose_name_plural),
            'slug' : self.slug,
        }

        if self.static:
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('static_detail', kwargs=kwargs)
            else:
                url = reverse('home_static_detail', kwargs=kwargs)
        else:
            kwargs.update({
                    'year' : self.publish_from.year,
                    'month' : self.publish_from.month,
                    'day' : self.publish_from.day,
                })
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('object_detail', kwargs=kwargs)
            else:
                url = reverse('home_object_detail', kwargs=kwargs)

        if category.site_id != settings.SITE_ID or domain:
            site = get_cached_object(Site, pk=category.site_id)
            return 'http://' + site.domain + url
        return url

def ListingBox(listing, *args, **kwargs):
    " Delegate the boxing to the target's Box class. "
    obj = listing.placement.publishable
    return obj.box_class(obj, *args, **kwargs)

class Listing(models.Model):
    """
    Listing of an object in a category. Each and every object that have it's own detail page must have a Listing object
    that is valid (not expired) and places him in the object's main category. Any object can be listed in any number of
    categories (but only once per category). Even if the object is listed in other categories besides its main category,
    its detail page's url still belongs to the main one.

    see doc/listing.txt for more details on Listings
    """
    box_class = staticmethod(ListingBox)

    placement = models.ForeignKey(Placement, verbose_name=_('Placement'))
    category = models.ForeignKey(Category, verbose_name=_('Category'), db_index=True)

    publish_from = models.DateTimeField(_("Start of listing"))
    publish_to = models.DateTimeField(_("End of listing"), null=True, blank=True)

    priority_from = models.DateTimeField(_("Start of prioritized listing"), null=True, blank=True)
    priority_to = models.DateTimeField(_("End of prioritized listing"), null=True, blank=True)
    priority_value = models.IntegerField(_("Priority"), blank=True, null=True)

    commercial = models.BooleanField(_("Commercial"), default=False, help_text=_("Check this if the listing is of a commercial content."))

    objects = ListingManager()

    @property
    def target(self):
        return self.placement.publishable

    def get_absolute_url(self, domain=False):
        return self.placement.get_absolute_url(domain)

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_publish_from(self):
        return self.publish_from

    def __unicode__(self):
        try:
            return u'%s listed in %s' % (self.placement.publishable, self.category)
        except:
            return 'Broken listing'

    class Meta:
        app_label = 'core'
        verbose_name = _('Listing')
        verbose_name_plural = _('Listings')
        ordering = ('-publish_from',)

class HitCount(models.Model):
    """
    Count hits for individual objects.
    """
    placement = models.ForeignKey(Placement, primary_key=True)

    last_seen = models.DateTimeField(_('Last seen'), editable=False)
    hits = models.PositiveIntegerField(_('Hits'), default=1)

    objects = HitCountManager()

    def save(self, **kwargs):
        "update last seen automaticaly"
        self.last_seen = datetime.now()
        super(HitCount, self).save(**kwargs)

    def target(self):
        return self.placement.publishable

    class Meta:
        app_label = 'core'
        verbose_name = _('Hit Count')
        verbose_name_plural = _('Hit Counts')

class Related(models.Model):
    """
    Related objects - model for recording related items. For example related articles.
    """
    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable'))

    related_ct = models.ForeignKey(ContentType, verbose_name=_('Content type'))
    related_id = models.IntegerField(_('Object ID'))
    related = CachedGenericForeignKey('related_ct', 'related_id')

    objects = RelatedManager()

    def __unicode__(self):
        return u'%s relates to %s' % (self.publishable, self.related)

    class Meta:
        app_label = 'core'
        verbose_name = _('Related')
        verbose_name_plural = _('Related')

<commit_msg>preventing circular imports<commit_after>from datetime import datetime

from django.db import models
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.contrib.sites.models import Site
from django.template.defaultfilters import slugify
from django.core.urlresolvers import reverse
from django.contrib.redirects.models import Redirect

from ella.core.managers import ListingManager, HitCountManager, PlacementManager, RelatedManager
from ella.core.cache import get_cached_object, get_cached_list, CachedGenericForeignKey
from ella.core.models.main import Category, Author, Source
from ella.photos.models import Photo
from ella.core.box import Box

PUBLISH_FROM_WHEN_EMPTY = datetime(3000, 1, 1)

def PublishableBox(publishable, box_type, nodelist, model=None):
    "add some content type info of self.target"
    if not model:
        model = publishable.content_type.model_class()
    box_class = model.box_class

    if box_class == PublishableBox:
        box_class = Box
    return box_class(publishable, box_type, nodelist, model=model)

class Publishable(models.Model):
    """
    Base class for all object that can be published in ella
    """
    box_class = staticmethod(PublishableBox)

    content_type = models.ForeignKey(ContentType)

    category = models.ForeignKey(Category, verbose_name=_('Category'))

    # Titles
    title = models.CharField(_('Title'), max_length=255)
    slug = models.SlugField(_('Slug'), max_length=255)

    # Authors and Sources
    authors = models.ManyToManyField(Author, verbose_name=_('Authors'))
    source = models.ForeignKey(Source, blank=True, null=True, verbose_name=_('Source'))

    # Main Photo
    photo = models.ForeignKey(Photo, blank=True, null=True, verbose_name=_('Photo'))

    # Description
    description = models.TextField(_('Description'), blank=True)

    # denormalized fields
    # the placement's publish_from
    publish_from = models.DateTimeField(_('Publish from'), editable=False, default=PUBLISH_FROM_WHEN_EMPTY)

    class Meta:
        app_label = 'core'
        verbose_name = _('Publishable object')
        verbose_name_plural = _('Publishable objects')

    @property
    def target(self):
        if not hasattr(self, '_target'):
            self._target = self.content_type.get_object_for_this_type(pk=self.pk)
        return self._target

    if 'ella.comments' in settings.INSTALLED_APPS:
        from ella.comments.models import Comment
        comments = generic.GenericRelation(Comment, object_id_field='target_id', content_type_field='target_ct')

    @property
    def main_placement(self):
        " Return object's main placement, that is the object's placement in its primary category "
        if hasattr(self, '_main_placement'):
            return self._main_placement

        current_site = Site.objects.get_current()

        # TODO: what if have got multiple listings on one site?
        placements = get_cached_list(
                Placement,
                publishable=self.pk,
                category__site=current_site,
            )
        if placements:
            if len(placements) == 1:
                return placements[0]
            else:
                # with multiple listings, one with first publish_from
                # primary
                first_published = None
                for placement in placements:
                    if first_published is None or placement.publish_from < first_published.publish_from:
                        first_published = placement

                assert first_published is not None
                return first_published

        try:
            # TODO - check and if we don't have category, take the only placement that exists in current site
            self._main_placement = get_cached_object(
                    Placement,
                    publishable=self.pk,
                    category=self.category_id
                )
        except Placement.DoesNotExist:
            self._main_placement = None

        return self._main_placement

    def get_absolute_url(self, domain=False):
        " Get object's URL. "
        placement = self.main_placement
        if placement:
            return placement.get_absolute_url(domain=domain)

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_admin_url(self):
        from ella.ellaadmin.utils import admin_url
        return admin_url(self)

    def save(self, **kwargs):
        self.content_type = ContentType.objects.get_for_model(self)
        if self.pk and hasattr(self, 'slug'): # only run on update
            # get old self
            old_slug = self.__class__._default_manager.get(pk=self.pk).slug
            # the slug has changed
            if old_slug != self.slug:
                for plc in Placement.objects.filter(publishable=self):
                    if plc.slug == old_slug:
                        plc.slug = self.slug
                        plc.save(force_update=True)
        return super(Publishable, self).save(**kwargs)

    def delete(self):
        url = self.get_absolute_url()
        Redirect.objects.filter(new_path=url).delete()
        return super(Publishable, self).delete()

    ##
    # various metadata
    ##
    def get_category(self):
        " Get object's primary Category."
        return get_cached_object(Category, pk=self.category_id)

    def get_photo(self):
        " Get object's Photo. "
        if not hasattr(self, '_photo'):
            try:
                self._photo = get_cached_object(Photo, pk=self.photo_id)
            except Photo.DoesNotExist:
                self._photo = None
        return self._photo

    def get_description(self):
        return self.description

    def get_title(self):
        myTitle=self.title
        if myTitle:
            return '%s' % (self.title,)
        else:
            return '%s' % (self.draw_title(),)

    def get_text(self):
        return self.text

    def __unicode__(self):
        return self.get_title()

# FIXME find another way to register!
if 'tagging' in settings.INSTALLED_APPS:
    import tagging
    tagging.register(Publishable)

class Placement(models.Model):
    # listing's target - a Publishable object
    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable object'))
    category = models.ForeignKey(Category, verbose_name=_('Category'), db_index=True)
    publish_from = models.DateTimeField(_("Start of visibility")) #, default=datetime.now)
    publish_to = models.DateTimeField(_("End of visibility"), null=True, blank=True)
    slug = models.SlugField(_('Slug'), max_length=255, blank=True)

    static = models.BooleanField(_('static'), default=False)

    objects = PlacementManager()

    class Meta:
        app_label = 'core'
        verbose_name = _('Placement')
        verbose_name_plural = _('Placements')

    def __unicode__(self):
        try:
            return u'%s placed in %s' % (self.publishable, self.category)
        except:
            return 'Broken placement'

    def is_active(self):
        "Return True if the Placement is currently active."
        now = datetime.now()
        return now > self.publish_from and (self.publish_to is None or now < self.publish_to)

    def delete(self):
        super(Placement, self).delete()
        try:
            publish_from = Placement.objects.filter(publishable=self.publishable).order_by('publish_from')[0].publish_from
        except IndexError, e:
            publish_from = PUBLISH_FROM_WHEN_EMPTY

        self.publishable.publish_from = publish_from
        Publishable.objects.filter(pk=self.publishable_id).update(publish_from=publish_from)

    def save(self, **kwargs):
        " If Listing is created, we create HitCount object "

        if not self.slug:
            self.slug = self.publishable.slug

        if self.pk:
            old_self = Placement.objects.get(pk=self.pk)

            old_path = old_self.get_absolute_url()
            new_path = self.get_absolute_url()

            if old_path != new_path and new_path:
                redirect, created = Redirect.objects.get_or_create(old_path=old_path, site=self.category.site)
                redirect.new_path=new_path
                redirect.save(force_update=True)
                Redirect.objects.filter(new_path=old_path).exclude(pk=redirect.pk).update(new_path=new_path)

        # First, save Placement
        super(Placement, self).save(**kwargs)
        # Then, save HitCount (needs placement_id)
        hc, created = HitCount.objects.get_or_create(placement=self, defaults={'hits': 0})

        # store the publish_from on the publishable for performance in the admin
        if self.publishable.publish_from > self.publish_from:
            self.publishable.publish_from = self.publish_from
            Publishable.objects.filter(pk=self.publishable_id).update(publish_from=self.publish_from)

    def get_absolute_url(self, domain=False):
        obj = self.publishable
        category = get_cached_object(Category, pk=self.category_id)

        kwargs = {
            'content_type' : slugify(obj.content_type.model_class()._meta.verbose_name_plural),
            'slug' : self.slug,
        }

        if self.static:
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('static_detail', kwargs=kwargs)
            else:
                url = reverse('home_static_detail', kwargs=kwargs)
        else:
            kwargs.update({
                    'year' : self.publish_from.year,
                    'month' : self.publish_from.month,
                    'day' : self.publish_from.day,
                })
            if category.tree_parent_id:
                kwargs['category'] = category.tree_path
                url = reverse('object_detail', kwargs=kwargs)
            else:
                url = reverse('home_object_detail', kwargs=kwargs)

        if category.site_id != settings.SITE_ID or domain:
            site = get_cached_object(Site, pk=category.site_id)
            return 'http://' + site.domain + url
        return url

def ListingBox(listing, *args, **kwargs):
    " Delegate the boxing to the target's Box class. "
    obj = listing.placement.publishable
    return obj.box_class(obj, *args, **kwargs)

class Listing(models.Model):
    """
    Listing of an object in a category. Each and every object that have it's own detail page must have a Listing object
    that is valid (not expired) and places him in the object's main category. Any object can be listed in any number of
    categories (but only once per category). Even if the object is listed in other categories besides its main category,
    its detail page's url still belongs to the main one.

    see doc/listing.txt for more details on Listings
    """
    box_class = staticmethod(ListingBox)

    placement = models.ForeignKey(Placement, verbose_name=_('Placement'))
    category = models.ForeignKey(Category, verbose_name=_('Category'), db_index=True)

    publish_from = models.DateTimeField(_("Start of listing"))
    publish_to = models.DateTimeField(_("End of listing"), null=True, blank=True)

    priority_from = models.DateTimeField(_("Start of prioritized listing"), null=True, blank=True)
    priority_to = models.DateTimeField(_("End of prioritized listing"), null=True, blank=True)
    priority_value = models.IntegerField(_("Priority"), blank=True, null=True)

    commercial = models.BooleanField(_("Commercial"), default=False, help_text=_("Check this if the listing is of a commercial content."))

    objects = ListingManager()

    @property
    def target(self):
        return self.placement.publishable

    def get_absolute_url(self, domain=False):
        return self.placement.get_absolute_url(domain)

    def get_domain_url(self):
        return self.get_absolute_url(domain=True)

    def get_publish_from(self):
        return self.publish_from

    def __unicode__(self):
        try:
            return u'%s listed in %s' % (self.placement.publishable, self.category)
        except:
            return 'Broken listing'

    class Meta:
        app_label = 'core'
        verbose_name = _('Listing')
        verbose_name_plural = _('Listings')
        ordering = ('-publish_from',)

class HitCount(models.Model):
    """
    Count hits for individual objects.
    """
    placement = models.ForeignKey(Placement, primary_key=True)

    last_seen = models.DateTimeField(_('Last seen'), editable=False)
    hits = models.PositiveIntegerField(_('Hits'), default=1)

    objects = HitCountManager()

    def save(self, **kwargs):
        "update last seen automaticaly"
        self.last_seen = datetime.now()
        super(HitCount, self).save(**kwargs)

    def target(self):
        return self.placement.publishable

    class Meta:
        app_label = 'core'
        verbose_name = _('Hit Count')
        verbose_name_plural = _('Hit Counts')

class Related(models.Model):
    """
    Related objects - model for recording related items. For example related articles.
    """
    publishable = models.ForeignKey(Publishable, verbose_name=_('Publishable'))

    related_ct = models.ForeignKey(ContentType, verbose_name=_('Content type'))
    related_id = models.IntegerField(_('Object ID'))
    related = CachedGenericForeignKey('related_ct', 'related_id')

    objects = RelatedManager()

    def __unicode__(self):
        return u'%s relates to %s' % (self.publishable, self.related)

    class Meta:
        app_label = 'core'
        verbose_name = _('Related')
        verbose_name_plural = _('Related')

<|endoftext|>


<commit_before>#! /usr/local/bin/python
"""
See LICENSE file for copyright and license details.
"""

from generic.modules.calculator_finance import *
from modules.constant import *
from decimal import Decimal

class Calculator():
    """
        Calculator class
    """

    def __init__(self, pool, amount, tax, commission, shares, price, buy, market, commodity, account):
        """
            Initialize
        """
        self.pool = pool
        self.amount = amount
        self.tax = tax
        self.commission = commission
        self.shares = shares
        self.price = price
        self.buy = buy
        self.market = market
        self.commodity = commodity
        self.account = account
        # result
        self.result_general = {}
        self.result_buy = {}
        self.result_sell = {}

    def calculate(self):
        """
            Calculate all possible unknown values.
        """
        print('Test: ', self.account)
        try:
            #Note: the order is important...
            # Input values
            if self.shares == Decimal(-1.0):
                self.shares = calculate_shares_recommended()
                print('test: shares =', self.shares)
            if self.price == Decimal(-1.0):
                print('test: ', self.amount)
                if self.buy:
                    self.price = calculate_price(
                        Transaction.BUY
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
                else:
                    self.price = calculate_price(
                        Transaction.SELL
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
                print('price =', self.price)
            if self.commission == Decimal(-1.0):
                self.commission = calculate_commission(
                    self.account
                    , self.market
                    , self.commodity
                    , self.price
                    , self.shares)
            if self.amount == Decimal(-1.0):
                if self.buy:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.BUY
                        , self.tax
                        , self.commission)
                else:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.SELL
                        , self.tax
                        , self.commission)

            # Extra calculatable fields
            # GENERAL
            self.result_general["amount"] = self.amount
            self.result_general["tax"] = self.tax
            self.result_general["commission"] = self.commission
            self.result_general["amount_simple"] = calculate_amount_simple(self.shares, self.price)
            # BUY
            self.result_buy["cost_tax"] = cost_tax(Transaction.BUY, self.amount, self.commission, self.shares, self.price)
            self.result_buy["amount_with_tax"] = calculate_amount_with_tax(self.tax, self.shares, self.price)
            # SELL
            self.result_sell["cost_tax"] = cost_tax(Transaction.SELL, self.amount, self.commission, self.shares, self.price)
            self.result_sell["amount_with_tax"] = calculate_amount_with_tax(self.tax, self.shares, self.price)
        except Exception as ex:
            print('Error in calculate:', ex)

    def print_pretty(self):
        """
            Print the results with headers etc.
        """
        try:
            headers_general = ["amount", "tax", "commission", "amount_simple"]
            headers_buy_sell = ["cost_tax", "amount_with_tax"]

            print('GENERAL')
            print('-------')
            print(''.join(column.rjust(10) for header in headers_general))
            print(''.join(column.rjust(10) for value in result_general.values()))
            if self.buy:
                print('BUY')
                print('-------')
                print(''.join(column.rjust(10) for header in headers_buy_sell))
                print(''.join(column.rjust(10) for value in result_buy.values()))
            else:
                print('SELL')
                print('-------')
                print(''.join(column.rjust(10) for header in headers_buy_sell))
                print(''.join(column.rjust(10) for value in result_sell.values()))
        except Exception as ex:
            print('Error in print_pretty():', ex)

    def print_gnucash(self):
        """
            Print statements to enter in gnucash.
        """
        try:
            headers = ['account', 'shares', 'price', 'debit', 'credit']
            if self.buy:
                print('BUY')
                print('-------')
                print(''.join(column.rjust(10) for header in headers))
                print(
                    'assets:stock:<market>.<commodity>,' +
                    str(self.shares) +
                    ',' +
                    str(self.price) +
                    ',' +
                    str(self.result_general["amount_simple"]) +
                    ','
                )
                print(
                    'expenses:commission:stock:<market>.<commodity>,,' +
                    str(self.commission) +
                    ','
                )
                print(
                    'expenses:tax:stock:<market>.<commodity>,,' +
                    str(self.result_buy["cost_tax"])
                )
                print(
                    'assets:current_assets:stock:<bank account>,,,' +
                    str(self.amount)
                )
            else:
                print('SELL')
                print('-------')
                print(''.join(column.rjust(10) for header in headers))
                print(
                    'assets:stock:<market>.<commodity>,' +
                    str(self.shares) +
                    ',' +
                    str(self.price) +
                    ',' +
                    str(self.result_general["amount_simple"]) +
                    ','
                )
                print(
                    'expenses:commission:stock:<market>.<commodity>,,' +
                    str(self.commission) +
                    ','
                )
                print(
                    'expenses:tax:stock:<market>.<commodity>,,' +
                    str(self.result_sell["cost_tax"])
                )
                print(
                    'assets:current_assets:stock:<bank account>,,,' +
                    str(self.amount)
                )
        except Exception as ex:
            print('Error in print_gnucash():', ex)
<commit_msg>Preparing for extra output.<commit_after>#! /usr/local/bin/python
"""
See LICENSE file for copyright and license details.
"""

from generic.modules.calculator_finance import *
from modules.constant import *
from decimal import Decimal

class Calculator():
    """
        Calculator class
    """

    def __init__(self, pool, amount, tax, commission, shares, price, buy, market, commodity, account):
        """
            Initialize
        """
        self.pool = pool
        self.amount = amount
        self.tax = tax
        self.commission = commission
        self.shares = shares
        self.price = price
        self.buy = buy
        self.market = market
        self.commodity = commodity
        self.account = account
        # result
        self.result_general = {}
        self.result_buy = {}
        self.result_sell = {}

    def calculate(self):
        """
            Calculate all possible unknown values.
        """
        print('Test: ', self.account)
        try:
            #Note: the order is important...
            # Input values
            if self.shares == Decimal(-1.0):
                self.shares = calculate_shares_recommended()
                print('test: shares =', self.shares)
            if self.price == Decimal(-1.0):
                print('test: ', self.amount)
                if self.buy:
                    self.price = calculate_price(
                        Transaction.BUY
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
                else:
                    self.price = calculate_price(
                        Transaction.SELL
                        , self.amount
                        , self.shares
                        , self.tax
                        , self.commission)
                print('price =', self.price)
            if self.commission == Decimal(-1.0):
                self.commission = calculate_commission(
                    self.account
                    , self.market
                    , self.commodity
                    , self.price
                    , self.shares)
            if self.amount == Decimal(-1.0):
                if self.buy:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.BUY
                        , self.tax
                        , self.commission)
                else:
                    self.amount = calculate_amount(
                        self.price
                        , self.shares
                        , Transaction.SELL
                        , self.tax
                        , self.commission)

            # Extra calculatable fields
            # GENERAL - need to know
            # TODO: first fix calculate_shares_recommended in modules
            self.result_general["shares"] = calculate_shares_recommended(...)
            # GENERAL - input
            self.result_general["amount"] = self.amount
            self.result_general["tax"] = self.tax
            self.result_general["commission"] = self.commission
            self.result_general["amount_simple"] = calculate_amount_simple(self.shares, self.price)
            # BUY
            self.result_buy["cost_tax"] = cost_tax(Transaction.BUY, self.amount, self.commission, self.shares, self.price)
            self.result_buy["amount_with_tax"] = calculate_amount_with_tax(self.tax, self.shares, self.price)
            # SELL
            self.result_sell["cost_tax"] = cost_tax(Transaction.SELL, self.amount, self.commission, self.shares, self.price)
            self.result_sell["amount_with_tax"] = calculate_amount_with_tax(self.tax, self.shares, self.price)
            # GENERAL - extra
            self.result_extra["r_multiple"] = calculate_r_multiple(...)
        except Exception as ex:
            print('Error in calculate:', ex)

    def print_pretty(self):
        """
            Print the results with headers etc.
        """
        try:
            headers_general = ["amount", "tax", "commission", "amount_simple"]
            headers_buy_sell = ["cost_tax", "amount_with_tax"]

            print('GENERAL')
            print('-------')
            print(''.join(column.rjust(10) for header in headers_general))
            print(''.join(column.rjust(10) for value in result_general.values()))
            if self.buy:
                print('BUY')
                print('-------')
                print(''.join(column.rjust(10) for header in headers_buy_sell))
                print(''.join(column.rjust(10) for value in result_buy.values()))
            else:
                print('SELL')
                print('-------')
                print(''.join(column.rjust(10) for header in headers_buy_sell))
                print(''.join(column.rjust(10) for value in result_sell.values()))
        except Exception as ex:
            print('Error in print_pretty():', ex)

    def print_gnucash(self):
        """
            Print statements to enter in gnucash.
        """
        try:
            headers = ['account', 'shares', 'price', 'debit', 'credit']
            if self.buy:
                print('BUY')
                print('-------')
                print(''.join(column.rjust(10) for header in headers))
                print(
                    'assets:stock:<market>.<commodity>,' +
                    str(self.shares) +
                    ',' +
                    str(self.price) +
                    ',' +
                    str(self.result_general["amount_simple"]) +
                    ','
                )
                print(
                    'expenses:commission:stock:<market>.<commodity>,,' +
                    str(self.commission) +
                    ','
                )
                print(
                    'expenses:tax:stock:<market>.<commodity>,,' +
                    str(self.result_buy["cost_tax"])
                )
                print(
                    'assets:current_assets:stock:<bank account>,,,' +
                    str(self.amount)
                )
            else:
                print('SELL')
                print('-------')
                print(''.join(column.rjust(10) for header in headers))
                print(
                    'assets:stock:<market>.<commodity>,' +
                    str(self.shares) +
                    ',' +
                    str(self.price) +
                    ',' +
                    str(self.result_general["amount_simple"]) +
                    ','
                )
                print(
                    'expenses:commission:stock:<market>.<commodity>,,' +
                    str(self.commission) +
                    ','
                )
                print(
                    'expenses:tax:stock:<market>.<commodity>,,' +
                    str(self.result_sell["cost_tax"])
                )
                print(
                    'assets:current_assets:stock:<bank account>,,,' +
                    str(self.amount)
                )
        except Exception as ex:
            print('Error in print_gnucash():', ex)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import re

class EntityException(Exception):
    """Wrap entity specific errors"""
    pass

class Entity(object):
	"""Base implementation for an Asana entity containing
	common funcitonality"""

	# Keys which are filtered as part of the HTTP request
	_filter_keys = []

	#fields this object has. This affects what is returned from the Asana API
	#as well as serving as a lookup for lazy-loading
	_fields = []

	#define regex to match field names that should be wrapped with an instance
	#of this object
	_matchons = []

	#items that are sub-items of the current one such that the API endpoint is
	#/api/parent/<id>/subitme
	_children = {}

	def __init__(self, data):
		self._childrenValues = {}
		self._init(data)
		self._ready = True

	def _init(self, data, merge=False):
		"""Initializes this entity, either with entirely new data or with an
		update to be merged with the current data

		:param data: the data to use for the entity
		:param merge: if true only set keys from data that aren't already set
			internally
		"""
		if merge:
			for key, value in data.items():
				if key not in self._data or key == 'id':
					self._data[key] = value
		else:
			self._data = data
			self._dirty = set()

		#todo it would probably be better to subclass
		# dict and implement this in there
		for key in self._data:
			if not self._data[key]:
				continue

			for regex, cls in self._matchons.items():
				if re.search(regex, key):
					if isinstance(self._data[key], list):
						for idx, val in enumerate(self._data[key]):
							if isinstance(val, dict):
								self._data[key][idx] = cls(val)
					else:
						if isinstance(self._data[key], dict):
							self._data[key] = cls(self._data[key])

					break

	@classmethod
	def set_api(cls, api):
		cls.api = api

	@classmethod
	def _get_api(cls):
		if not cls.api:
			raise EntityException('The api must be set using Entity.set_api()')

		return cls.api

	@classmethod
	def _get_api_endpoint(cls):
		"""By default use name of class for endpoint"""
		return cls.__name__.lower() + 's'

	def _get_item_url(self):
		if not self.id:
			raise EntityException('Cannot get item URL without id set')

		return '/'.join([self._get_api_endpoint(), str(self.id)])

	@classmethod
	def find(cls, query={}):
		"""Find objects of this type that fit query

		:param query: dict of key/value pairs to match against. keys that the
			API natively handles are sent as part of the request if they have
			scalar values, other keys are filtered from the response.
			filter values can be either absolute values or lambdas. for lambdas
			the value of its key will be passed as the only argument and it
			will be considered passing if the lambda returns true
		"""
		return cls._run_find(cls._get_api_endpoint(), query)

	@classmethod
	def _run_find(cls, target, query):
		params = cls._get_default_params() #params that are part of the request

		#todo handle lambdas that are passed in for filter keys
		if cls._filter_keys:
			for key in query.keys():
				if key in cls._filter_keys:
					params[key] = query[key]
					del query[key]

		data = cls._get_api().get(target, params=params)

		return cls._build_result(query, data)

	@classmethod
	def _get_default_params(cls):
		"""Hook to add params that will always be part of a find request
		Default behavior checks for the 'fields' property and, if present,
		joins it with commas and passes it as the opt_fields param
		"""
		if cls._fields:
			return {
				'opt_fields': ','.join(cls._fields)
			}

		return {}

	@classmethod
	def _build_result(cls, query, data):
		"""Filters the result set based on a query returning the resulting
		objects as instances of the current class"""

		return [cls(ent) for ent in data if cls._filter_result_item(ent, query)]

	@classmethod
	def _filter_result_item(cls, entity, query):
		"""Filters a single entity dict against a dict of allowed values
		returning true if it passes
		"""

		for key, value in query.items():
			if key not in entity:
				raise EntityException('The key {0} is not a valid query for {1}'.format(key, cls.__name__))

			if (
				(callable(value) and not value(entity[key])) or
				(isinstance(value, basestring) and value != entity[key])
			):
				return False
		return True

	def load(self):
		"""Loads all of this items data using its ID"""

		#TODO check if sending in empty opt_fields will make us lose all fields
		self._init(self._get_api().get(self._get_item_url()), merge=True)

		return self

	def get_subitem(self, subitem_class, query={}):
		target = '/'.join([self._get_item_url(), subitem_class._get_api_endpoint()])

		return subitem_class._run_find(target, query)

	def save(self):
		"""Handles both creating and updating content
		The assumption is if there is no ID set this is
		a creation request
		"""

		if self.id:
			return self._do_update()
		else:
			#performing create - post
			return self._do_create()

	def _do_update(self):
		data = {}

		for key in self._dirty:
			data[key] = self._data[key]

		if not data:
			return

		return self._get_api().put(self._get_item_url(), data=data)

	def _do_create(self):
		return self._init(self._get_api().post(self._get_api_endpoint(), data=self._data))

	def delete(self):
		"""Deletes the specified resource. The ID must be set"""

		self._get_api().delete(self._get_item_url())

	def __getattr__(self, attr):

		if attr in self.__dict__:
			return self.__dict__[attr]

		if attr in self.__dict__['_data']:
			return self.__dict__['_data'][attr]

		if attr in self._fields:
			self.load()
			return self.__dict__['_data'][attr]

		if attr in self._children.keys():

			if not attr in self._childrenValues.keys():
				self._childrenValues[attr] = self.get_subitem(self._children[attr])

			return self._childrenValues[attr]

		raise Exception("Could not locate key " + attr)

	def __setattr__(self, attr, value):
		if attr[0] == '_':
			self.__dict__[attr] = value
		elif self._ready:

			if attr in self._fields:
				self._data[attr] = value
				self._dirty.add(attr)
			else:
				raise Exception("Cannot set attribute {0} - unknown name".foramt(attr))

	def __str__(self):
		return vars(self).__repr__()

	def __repr__(self):
		return self.__str__()

	def __hash__(self):
		return hash(self.id if hasattr(self, 'id') else frozenset(self._data.items()))

	def __eq__(self, other):
		return self.id == other.id if hasattr(self, 'id') else cmp(self._data, other._data)<commit_msg>Fix getting ID<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
import re

class EntityException(Exception):
    """Wrap entity specific errors"""
    pass

class Entity(object):
	"""Base implementation for an Asana entity containing
	common funcitonality"""

	# Keys which are filtered as part of the HTTP request
	_filter_keys = []

	#fields this object has. This affects what is returned from the Asana API
	#as well as serving as a lookup for lazy-loading
	_fields = []

	#define regex to match field names that should be wrapped with an instance
	#of this object
	_matchons = []

	#items that are sub-items of the current one such that the API endpoint is
	#/api/parent/<id>/subitme
	_children = {}

	def __init__(self, data):
		self._childrenValues = {}
		self._init(data)
		self._ready = True

	def _init(self, data, merge=False):
		"""Initializes this entity, either with entirely new data or with an
		update to be merged with the current data

		:param data: the data to use for the entity
		:param merge: if true only set keys from data that aren't already set
			internally
		"""
		if merge:
			for key, value in data.items():
				if key not in self._data or key == 'id':
					self._data[key] = value
		else:
			self._data = data
			self._dirty = set()

		#todo it would probably be better to subclass
		# dict and implement this in there
		for key in self._data:
			if not self._data[key]:
				continue

			for regex, cls in self._matchons.items():
				if re.search(regex, key):
					if isinstance(self._data[key], list):
						for idx, val in enumerate(self._data[key]):
							if isinstance(val, dict):
								self._data[key][idx] = cls(val)
					else:
						if isinstance(self._data[key], dict):
							self._data[key] = cls(self._data[key])

					break

	@classmethod
	def set_api(cls, api):
		cls.api = api

	@classmethod
	def _get_api(cls):
		if not cls.api:
			raise EntityException('The api must be set using Entity.set_api()')

		return cls.api

	@classmethod
	def _get_api_endpoint(cls):
		"""By default use name of class for endpoint"""
		return cls.__name__.lower() + 's'

	def _get_item_url(self):
		if not self.id:
			raise EntityException('Cannot get item URL without id set')

		return '/'.join([self._get_api_endpoint(), str(self.id)])

	@classmethod
	def find(cls, query={}):
		"""Find objects of this type that fit query

		:param query: dict of key/value pairs to match against. keys that the
			API natively handles are sent as part of the request if they have
			scalar values, other keys are filtered from the response.
			filter values can be either absolute values or lambdas. for lambdas
			the value of its key will be passed as the only argument and it
			will be considered passing if the lambda returns true
		"""
		return cls._run_find(cls._get_api_endpoint(), query)

	@classmethod
	def _run_find(cls, target, query):
		params = cls._get_default_params() #params that are part of the request

		#todo handle lambdas that are passed in for filter keys
		if cls._filter_keys:
			for key in query.keys():
				if key in cls._filter_keys:
					params[key] = query[key]
					del query[key]

		data = cls._get_api().get(target, params=params)

		return cls._build_result(query, data)

	@classmethod
	def _get_default_params(cls):
		"""Hook to add params that will always be part of a find request
		Default behavior checks for the 'fields' property and, if present,
		joins it with commas and passes it as the opt_fields param
		"""
		if cls._fields:
			return {
				'opt_fields': ','.join(cls._fields)
			}

		return {}

	@classmethod
	def _build_result(cls, query, data):
		"""Filters the result set based on a query returning the resulting
		objects as instances of the current class"""

		return [cls(ent) for ent in data if cls._filter_result_item(ent, query)]

	@classmethod
	def _filter_result_item(cls, entity, query):
		"""Filters a single entity dict against a dict of allowed values
		returning true if it passes
		"""

		for key, value in query.items():
			if key not in entity:
				raise EntityException('The key {0} is not a valid query for {1}'.format(key, cls.__name__))

			if (
				(callable(value) and not value(entity[key])) or
				(isinstance(value, basestring) and value != entity[key])
			):
				return False
		return True

	def load(self):
		"""Loads all of this items data using its ID"""

		#TODO check if sending in empty opt_fields will make us lose all fields
		self._init(self._get_api().get(self._get_item_url()), merge=True)

		return self

	def get_subitem(self, subitem_class, query={}):
		target = '/'.join([self._get_item_url(), subitem_class._get_api_endpoint()])

		return subitem_class._run_find(target, query)

	def save(self):
		"""Handles both creating and updating content
		The assumption is if there is no ID set this is
		a creation request
		"""

		if self.id:
			return self._do_update()
		else:
			#performing create - post
			return self._do_create()

	def _do_update(self):
		data = {}

		for key in self._dirty:
			data[key] = self._data[key]

		if not data:
			return

		return self._get_api().put(self._get_item_url(), data=data)

	def _do_create(self):
		return self._init(self._get_api().post(self._get_api_endpoint(), data=self._data))

	def delete(self):
		"""Deletes the specified resource. The ID must be set"""

		self._get_api().delete(self._get_item_url())

	def __getattr__(self, attr):

		if attr in self.__dict__:
			return self.__dict__[attr]

		if attr in self.__dict__['_data']:
			return self.__dict__['_data'][attr]

		if attr in self._fields:
			self.load()
			return self.__dict__['_data'][attr]

		if attr in self._children.keys():

			if not attr in self._childrenValues.keys():
				self._childrenValues[attr] = self.get_subitem(self._children[attr])

			return self._childrenValues[attr]

		if attr != 'id':
			raise Exception("Could not locate key " + attr)

	def __setattr__(self, attr, value):
		if attr[0] == '_':
			self.__dict__[attr] = value
		elif self._ready:

			if attr in self._fields:
				self._data[attr] = value
				self._dirty.add(attr)
			else:
				raise Exception("Cannot set attribute {0} - unknown name".foramt(attr))

	def __str__(self):
		return vars(self).__repr__()

	def __repr__(self):
		return self.__str__()

	def __hash__(self):
		return hash(self.id if hasattr(self, 'id') else frozenset(self._data.items()))

	def __eq__(self, other):
		return self.id == other.id if hasattr(self, 'id') else cmp(self._data, other._data)<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
 Juju agent-metadata-url validation on passing as an argument during
 juju bootstrap

 Juju agent-metadata-url validation using cloud definition yaml file to
 verify that agent-metadata-url specified in the yaml file is applied
 while running juju boostrap command.

 Usage: python assess_agent_metadata.py --agent-file=/path/to/juju-*.tgz

 Example: python assess_agent_metadata.py
                --agent-file=/home/juju/juju-2.0.1-xenial-amd64.tgz
"""

from __future__ import print_function

from argparse import ArgumentParser
from contextlib import contextmanager
from hashlib import sha256

import logging
import os
import subprocess
import sys
import json

from deploy_stack import (
    BootstrapManager,
    )

from utility import (
    add_basic_testing_arguments,
    configure_logging,
    JujuAssertionError,
    temp_dir,
)

from jujupy import (
    temp_yaml_file,
)

JUJU_TOOL_PATH = "tools/released/"

log = logging.getLogger("assess_agent_metadata")

def get_sha256_sum(filename):
    """
    Get SHA256 sum of the given filename
    :param filename: The filename
    """
    with open(filename, 'rb') as infile:
        return sha256(infile.read()).hexdigest()

def assert_cloud_details_are_correct(client, cloud_name, example_cloud):
    """
    Check juju add-cloud is performed succesffuly and it is available
    in the juju client.
    :param client: The juju client
    :param cloud_name: The name of the cloud added
    :param example_cloud: The content of the cloud
    """
    clouds = client.env.read_clouds()
    try:
        if clouds['clouds'][cloud_name] != example_cloud:
            raise JujuAssertionError('Cloud mismatch')
    except KeyError:
        raise JujuAssertionError('Cloud missing {}'.format(cloud_name))

def get_local_url_and_sha256(agent_dir, controller_url):
    """
    Get the agent URL (local file location: file:///) and SHA256
    of the agent-file passed
    :param agent_dir: The top level directory location of agent file.
    :param controller_url: The controller used agent file url
    """
    local_url = os.path.join(agent_dir,
                             JUJU_TOOL_PATH, os.path.basename(controller_url))
    local_sha256 = get_sha256_sum(local_url)
    local_file_path = "file://" + local_url
    return [local_file_path, local_sha256]

def get_controller_url_and_sha256(client):
    """
    Get the agent url and sha256 of the launched client
    :param client: Juju client
    """
    controller_client = client.get_controller_client()
    output = controller_client.run(
        ['cat /var/lib/juju/tools/machine-0/downloaded-tools.txt'],
        machines=['0'])
    output_ = json.loads(output[0]['Stdout'])
    return [output_['url'], output_['sha256']]

def assert_metadata_are_correct(agent_metadata_url, client):
    """
    verify the client agent-metadata-url uses the specified value
    :param agent_metadata_url: The agent file path.
    :param client: Juju client
    """
    data = client.get_model_config()
    if agent_metadata_url != data['agent-metadata-url']['value']:
        raise JujuAssertionError('Error, mismatch agent-metadata-url')

    log.info('bootstrap successfully with agent-metdadata-url={}'
             .format(data['agent-metadata-url']['value']))

def verify_deployed_tool(agent_dir, client):
    """
    Verify the bootstraped controller make use of the the specified
    agent-metadata-url.
    :param agent_dir:  The top level directory location of agent file.
    :param client: Juju client
    """
    controller_url, controller_sha256 = get_controller_url_and_sha256(client)

    log.debug("controller_url: {} and controller_sha256: {}".format(
               controller_url, controller_sha256))

    local_url, local_sha256 = get_local_url_and_sha256(
        agent_dir, controller_url)

    log.debug("local_url: {} and local_sha256: {}".format(
        local_url, local_sha256))

    if local_url != controller_url:
        raise JujuAssertionError(
            "mismatch local URL {} and controller URL {}".format(
                local_url, controller_url))

    if local_sha256 != controller_sha256:
        raise JujuAssertionError(
            "mismatch local SHA256 {} and controller SHA256 {}".format(
                local_sha256, controller_sha256))

def assess_metadata(args, agent_dir):
    """
    Bootstrap juju controller with agent-metadata-url value
    and verify that bootstraped controller make use of specified
    agent-metadata-url value.
    :param args: Parsed command line arguments
    :param agent_dir: The top level directory location of agent file.
    """
    bs_manager = BootstrapManager.from_args(args)
    client = bs_manager.client
    agent_metadata_url = os.path.join(agent_dir, JUJU_TOOL_PATH.split('/')[0])
    client.env.update_config({'agent-metadata-url': agent_metadata_url})
    log.info('bootstrap to use --agent_metadata_url={}'.format(
        agent_metadata_url))
    client.generate_tool(agent_dir)

    with bs_manager.booted_context(args.upload_tools):
        log.info('Metadata bootstrap successful.')
        assert_metadata_are_correct(agent_metadata_url, client)
        verify_deployed_tool(agent_dir, client)
        log.info("Successfully deployed and verified agent-metadata-url")

def assess_add_cloud(args, agent_dir):
    """
    Perform juju add-cloud by creating a yaml file for cloud
    with agent-metadata-url value and bootstrap the juju environment.
    :param args: Parsed command line arguments
    :param agent_dir: The top level directory location of agent file.
    """
    cloud_name = "testCloud"
    agent_metadata_url = os.path.join(agent_dir, JUJU_TOOL_PATH.split('/')[0])

    test_cloud_data = {'clouds': {'{0}'.format(
        cloud_name): {'type': 'lxd',
                      'config': {'agent-metadata-url': 'file://{}'.format(
                          agent_metadata_url)}}}}

    bs_manager = BootstrapManager.from_args(args)
    client = bs_manager.client
    with temp_yaml_file(test_cloud_data) as add_cloud_file:
        client.add_cloud(cloud_name, add_cloud_file)
        clouds = test_cloud_data['clouds'][cloud_name]
        assert_cloud_details_are_correct(client, cloud_name, clouds)

    with bs_manager.booted_context(args.upload_tools):
        log.info('Metadata bootstrap successful.')
        verify_deployed_tool(agent_dir, client)
        log.info("Successfully deployed and verified add-cloud")

def change_tgz_sha256_sum(src, dst):
    """
    Append empty string to the tgz file so that the SHA256 sum of the file
    get modified. We use this to make sure that controller deployed on
    bootstrap used of the changed tgz file.
    :param src: The source tgz file
    :param dst: The destination tgz file
    """
    if not src.endswith(".tgz"):
        raise Exception("{} is not tgz file".format(src))
    try:
        import pdb
        pdb.set_trace()
        command = "cat {}  <(echo -n ''| gzip)> {}".format(src, dst)
        subprocess.Popen(command, shell=True, executable='/bin/bash')
    except subprocess.CalledProcessError as e:
        raise Exception("Failed to create a tool file {} - {}". format(src, e))

@contextmanager
def make_unique_tool(agent_file):
    """
    Create a tool directory for juju agent tools and stream and invoke
    append_empty_string function for the agent-file passed.
    :param agent_file:
    """
    with temp_dir() as agent_dir:
        juju_tool_src = os.path.join(agent_dir, JUJU_TOOL_PATH)
        os.makedirs(juju_tool_src)
        change_tgz_sha256_sum(agent_file,
                              os.path.join(juju_tool_src,
                                           os.path.basename(agent_file)))
        log.debug("Created agent directory to perform bootstrap".format(
            agent_dir))
        yield agent_dir

def parse_args(argv):
    """Parse all arguments."""
    parser = ArgumentParser(
        description="Test bootstrap for agent-metdadata-url")

    add_basic_testing_arguments(parser)

    parser.add_argument('--agent-file', required=True,
                        action='store', default=None,
                        help='agent file to be used during bootstrap.')

    return parser.parse_args(argv)

def main(argv=None):
    args = parse_args(argv)
    configure_logging(args.verbose)

    if not os.path.isfile(args.agent_file):
        raise Exception(
           "Unable to find juju agent file {}".format(args.agent_file))

    with make_unique_tool(args.agent_file) as agent_dir:
        assess_metadata(args, agent_dir)

    with make_unique_tool(args.agent_file) as agent_dir:
        assess_add_cloud(args, agent_dir)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Fixed the build issue<commit_after>#!/usr/bin/env python
"""
 Juju agent-metadata-url validation on passing as an argument during
 juju bootstrap

 Juju agent-metadata-url validation using cloud definition yaml file to
 verify that agent-metadata-url specified in the yaml file is applied
 while running juju boostrap command.

 Usage: python assess_agent_metadata.py --agent-file=/path/to/juju-*.tgz

 Example: python assess_agent_metadata.py
                --agent-file=/home/juju/juju-2.0.1-xenial-amd64.tgz
"""

from __future__ import print_function

from argparse import ArgumentParser
from contextlib import contextmanager
from hashlib import sha256

import logging
import os
import subprocess
import sys
import json

from deploy_stack import (
    BootstrapManager,
    )

from utility import (
    add_basic_testing_arguments,
    configure_logging,
    JujuAssertionError,
    temp_dir,
)

from jujupy import (
    temp_yaml_file,
)

JUJU_TOOL_PATH = "tools/released/"

log = logging.getLogger("assess_agent_metadata")

def get_sha256_sum(filename):
    """
    Get SHA256 sum of the given filename
    :param filename: The filename
    """
    with open(filename, 'rb') as infile:
        return sha256(infile.read()).hexdigest()

def assert_cloud_details_are_correct(client, cloud_name, example_cloud):
    """
    Check juju add-cloud is performed succesffuly and it is available
    in the juju client.
    :param client: The juju client
    :param cloud_name: The name of the cloud added
    :param example_cloud: The content of the cloud
    """
    clouds = client.env.read_clouds()
    try:
        if clouds['clouds'][cloud_name] != example_cloud:
            raise JujuAssertionError('Cloud mismatch')
    except KeyError:
        raise JujuAssertionError('Cloud missing {}'.format(cloud_name))

def get_local_url_and_sha256(agent_dir, controller_url):
    """
    Get the agent URL (local file location: file:///) and SHA256
    of the agent-file passed
    :param agent_dir: The top level directory location of agent file.
    :param controller_url: The controller used agent file url
    """
    local_url = os.path.join(agent_dir,
                             JUJU_TOOL_PATH, os.path.basename(controller_url))
    local_sha256 = get_sha256_sum(local_url)
    local_file_path = "file://" + local_url
    return [local_file_path, local_sha256]

def get_controller_url_and_sha256(client):
    """
    Get the agent url and sha256 of the launched client
    :param client: Juju client
    """
    controller_client = client.get_controller_client()
    output = controller_client.run(
        ['cat /var/lib/juju/tools/machine-0/downloaded-tools.txt'],
        machines=['0'])
    output_ = json.loads(output[0]['Stdout'])
    return [output_['url'], output_['sha256']]

def assert_metadata_are_correct(agent_metadata_url, client):
    """
    verify the client agent-metadata-url uses the specified value
    :param agent_metadata_url: The agent file path.
    :param client: Juju client
    """
    data = client.get_model_config()
    if agent_metadata_url != data['agent-metadata-url']['value']:
        raise JujuAssertionError('Error, mismatch agent-metadata-url')

    log.info('bootstrap successfully with agent-metdadata-url={}'
             .format(data['agent-metadata-url']['value']))

def verify_deployed_tool(agent_dir, client):
    """
    Verify the bootstraped controller make use of the the specified
    agent-metadata-url.
    :param agent_dir:  The top level directory location of agent file.
    :param client: Juju client
    """
    controller_url, controller_sha256 = get_controller_url_and_sha256(client)

    log.debug("controller_url: {} and controller_sha256: {}".format(
               controller_url, controller_sha256))

    local_url, local_sha256 = get_local_url_and_sha256(
        agent_dir, controller_url)

    log.debug("local_url: {} and local_sha256: {}".format(
        local_url, local_sha256))

    if local_url != controller_url:
        raise JujuAssertionError(
            "mismatch local URL {} and controller URL {}".format(
                local_url, controller_url))

    if local_sha256 != controller_sha256:
        raise JujuAssertionError(
            "mismatch local SHA256 {} and controller SHA256 {}".format(
                local_sha256, controller_sha256))

def assess_metadata(args, agent_dir):
    """
    Bootstrap juju controller with agent-metadata-url value
    and verify that bootstraped controller make use of specified
    agent-metadata-url value.
    :param args: Parsed command line arguments
    :param agent_dir: The top level directory location of agent file.
    """
    bs_manager = BootstrapManager.from_args(args)
    client = bs_manager.client
    agent_metadata_url = os.path.join(agent_dir, JUJU_TOOL_PATH.split('/')[0])
    client.env.update_config({'agent-metadata-url': agent_metadata_url})
    log.info('bootstrap to use --agent_metadata_url={}'.format(
        agent_metadata_url))
    client.generate_tool(agent_dir)

    with bs_manager.booted_context(args.upload_tools):
        log.info('Metadata bootstrap successful.')
        assert_metadata_are_correct(agent_metadata_url, client)
        verify_deployed_tool(agent_dir, client)
        log.info("Successfully deployed and verified agent-metadata-url")

def assess_add_cloud(args, agent_dir):
    """
    Perform juju add-cloud by creating a yaml file for cloud
    with agent-metadata-url value and bootstrap the juju environment.
    :param args: Parsed command line arguments
    :param agent_dir: The top level directory location of agent file.
    """
    cloud_name = "testCloud"
    agent_metadata_url = os.path.join(agent_dir, JUJU_TOOL_PATH.split('/')[0])

    test_cloud_data = {'clouds': {'{0}'.format(
        cloud_name): {'type': 'lxd',
                      'config': {'agent-metadata-url': 'file://{}'.format(
                          agent_metadata_url)}}}}

    bs_manager = BootstrapManager.from_args(args)
    client = bs_manager.client
    with temp_yaml_file(test_cloud_data) as add_cloud_file:
        client.add_cloud(cloud_name, add_cloud_file)
        clouds = test_cloud_data['clouds'][cloud_name]
        assert_cloud_details_are_correct(client, cloud_name, clouds)

    with bs_manager.booted_context(args.upload_tools):
        log.info('Metadata bootstrap successful.')
        verify_deployed_tool(agent_dir, client)
        log.info("Successfully deployed and verified add-cloud")

def change_tgz_sha256_sum(src, dst):
    """
    Append empty string to the tgz file so that the SHA256 sum of the file
    get modified. We use this to make sure that controller deployed on
    bootstrap used of the changed tgz file.
    :param src: The source tgz file
    :param dst: The destination tgz file
    """
    if not src.endswith(".tgz"):
        raise Exception("{} is not tgz file".format(src))
    try:
        command = "cat {}  <(echo -n ''| gzip)> {}".format(src, dst)
        subprocess.Popen(command, shell=True, executable='/bin/bash')
    except subprocess.CalledProcessError as e:
        raise Exception("Failed to create a tool file {} - {}". format(src, e))

@contextmanager
def make_unique_tool(agent_file):
    """
    Create a tool directory for juju agent tools and stream and invoke
    append_empty_string function for the agent-file passed.
    :param agent_file:
    """
    with temp_dir() as agent_dir:
        juju_tool_src = os.path.join(agent_dir, JUJU_TOOL_PATH)
        os.makedirs(juju_tool_src)
        change_tgz_sha256_sum(agent_file,
                              os.path.join(juju_tool_src,
                                           os.path.basename(agent_file)))
        log.debug("Created agent directory to perform bootstrap".format(
            agent_dir))
        yield agent_dir

def parse_args(argv):
    """Parse all arguments."""
    parser = ArgumentParser(
        description="Test bootstrap for agent-metdadata-url")

    add_basic_testing_arguments(parser)

    parser.add_argument('--agent-file', required=True,
                        action='store', default=None,
                        help='agent file to be used during bootstrap.')

    return parser.parse_args(argv)

def main(argv=None):
    args = parse_args(argv)
    configure_logging(args.verbose)

    if not os.path.isfile(args.agent_file):
        raise Exception(
           "Unable to find juju agent file {}".format(args.agent_file))

    with make_unique_tool(args.agent_file) as agent_dir:
        assess_metadata(args, agent_dir)

    with make_unique_tool(args.agent_file) as agent_dir:
        assess_add_cloud(args, agent_dir)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import frappe, re
from frappe import _
from frappe.utils import cstr
from erpnext.regional.india import states, state_numbers
from erpnext.controllers.taxes_and_totals import get_itemised_tax, get_itemised_taxable_amount

def validate_gstin_for_india(doc, method):
	if not hasattr(doc, 'gstin'):
		return

	if doc.gstin:
		doc.gstin = doc.gstin.upper()
		if doc.gstin not in ["NA", "na"]:
			p = re.compile("[0-9]{2}[A-Z]{4}[0-9A-Z]{1}[0-9]{4}[A-Z]{1}[1-9A-Z]{1}[0-9A-Z]{1}[0-9A-Z]{1}")
			if not p.match(doc.gstin):
				frappe.throw(_("Invalid GSTIN or Enter NA for Unregistered"))

	if not doc.gst_state:
		if doc.state in states:
			doc.gst_state = doc.state

	if doc.gst_state:
		doc.gst_state_number = state_numbers[doc.gst_state]
		if doc.gstin and doc.gstin != "NA" and doc.gst_state_number != doc.gstin[:2]:
			frappe.throw(_("First 2 digits of GSTIN should match with State number {0}")
				.format(doc.gst_state_number))

def get_itemised_tax_breakup_header(item_doctype, tax_accounts):
	if frappe.get_meta(item_doctype).has_field('gst_hsn_code'):
		return [_("HSN/SAC"), _("Taxable Amount")] + tax_accounts
	else:
		return [_("Item"), _("Taxable Amount")] + tax_accounts

def get_itemised_tax_breakup_data(doc):
	itemised_tax = get_itemised_tax(doc.taxes)

	itemised_taxable_amount = get_itemised_taxable_amount(doc.items)

	if not frappe.get_meta(doc.doctype + " Item").has_field('gst_hsn_code'):
		return itemised_tax, itemised_taxable_amount

	item_hsn_map = frappe._dict()
	for d in doc.items:
		item_hsn_map.setdefault(d.item_code or d.item_name, d.get("gst_hsn_code"))

	hsn_tax = {}
	for item, taxes in itemised_tax.items():
		hsn_code = item_hsn_map.get(item)
		hsn_tax.setdefault(hsn_code, frappe._dict())
		for tax_account, tax_detail in taxes.items():
			hsn_tax[hsn_code].setdefault(tax_account, {"tax_rate": 0, "tax_amount": 0})
			hsn_tax[hsn_code][tax_account]["tax_rate"] = tax_detail.get("tax_rate")
			hsn_tax[hsn_code][tax_account]["tax_amount"] += tax_detail.get("tax_amount")

	# set taxable amount
	hsn_taxable_amount = frappe._dict()
	for item, taxable_amount in itemised_taxable_amount.items():
		hsn_code = item_hsn_map.get(item)
		hsn_taxable_amount.setdefault(hsn_code, 0)
		hsn_taxable_amount[hsn_code] += itemised_taxable_amount.get(item)

	return hsn_tax, hsn_taxable_amount

def set_place_of_supply(doc, method):
	if not frappe.get_meta('Address').has_field('gst_state'): return

	if doc.doctype in ("Sales Invoice", "Delivery Note"):
		address_name = doc.shipping_address_name or doc.customer_address
	elif doc.doctype == "Purchase Invoice":
		address_name = doc.shipping_address or doc.supplier_address

	if address_name:
		address = frappe.db.get_value("Address", address_name, ["gst_state", "gst_state_number"], as_dict=1)
		doc.place_of_supply = cstr(address.gst_state_number) + "-" + cstr(address.gst_state)

# don't remove this function it is used in tests
def test_method():
	'''test function'''
	return 'overridden'
<commit_msg>fix: 14th digit may not be zero<commit_after>import frappe, re
from frappe import _
from frappe.utils import cstr
from erpnext.regional.india import states, state_numbers
from erpnext.controllers.taxes_and_totals import get_itemised_tax, get_itemised_taxable_amount

def validate_gstin_for_india(doc, method):
	if not hasattr(doc, 'gstin'):
		return

	if doc.gstin:
		doc.gstin = doc.gstin.upper()
		if doc.gstin not in ["NA", "na"]:
			p = re.compile("[0-9]{2}[A-Z]{4}[0-9A-Z]{1}[0-9]{4}[A-Z]{1}[1-9A-Z]{1}[1-9A-Z]{1}[0-9A-Z]{1}")
			if not p.match(doc.gstin):
				frappe.throw(_("Invalid GSTIN or Enter NA for Unregistered"))

	if not doc.gst_state:
		if doc.state in states:
			doc.gst_state = doc.state

	if doc.gst_state:
		doc.gst_state_number = state_numbers[doc.gst_state]
		if doc.gstin and doc.gstin != "NA" and doc.gst_state_number != doc.gstin[:2]:
			frappe.throw(_("First 2 digits of GSTIN should match with State number {0}")
				.format(doc.gst_state_number))

def get_itemised_tax_breakup_header(item_doctype, tax_accounts):
	if frappe.get_meta(item_doctype).has_field('gst_hsn_code'):
		return [_("HSN/SAC"), _("Taxable Amount")] + tax_accounts
	else:
		return [_("Item"), _("Taxable Amount")] + tax_accounts

def get_itemised_tax_breakup_data(doc):
	itemised_tax = get_itemised_tax(doc.taxes)

	itemised_taxable_amount = get_itemised_taxable_amount(doc.items)

	if not frappe.get_meta(doc.doctype + " Item").has_field('gst_hsn_code'):
		return itemised_tax, itemised_taxable_amount

	item_hsn_map = frappe._dict()
	for d in doc.items:
		item_hsn_map.setdefault(d.item_code or d.item_name, d.get("gst_hsn_code"))

	hsn_tax = {}
	for item, taxes in itemised_tax.items():
		hsn_code = item_hsn_map.get(item)
		hsn_tax.setdefault(hsn_code, frappe._dict())
		for tax_account, tax_detail in taxes.items():
			hsn_tax[hsn_code].setdefault(tax_account, {"tax_rate": 0, "tax_amount": 0})
			hsn_tax[hsn_code][tax_account]["tax_rate"] = tax_detail.get("tax_rate")
			hsn_tax[hsn_code][tax_account]["tax_amount"] += tax_detail.get("tax_amount")

	# set taxable amount
	hsn_taxable_amount = frappe._dict()
	for item, taxable_amount in itemised_taxable_amount.items():
		hsn_code = item_hsn_map.get(item)
		hsn_taxable_amount.setdefault(hsn_code, 0)
		hsn_taxable_amount[hsn_code] += itemised_taxable_amount.get(item)

	return hsn_tax, hsn_taxable_amount

def set_place_of_supply(doc, method):
	if not frappe.get_meta('Address').has_field('gst_state'): return

	if doc.doctype in ("Sales Invoice", "Delivery Note"):
		address_name = doc.shipping_address_name or doc.customer_address
	elif doc.doctype == "Purchase Invoice":
		address_name = doc.shipping_address or doc.supplier_address

	if address_name:
		address = frappe.db.get_value("Address", address_name, ["gst_state", "gst_state_number"], as_dict=1)
		doc.place_of_supply = cstr(address.gst_state_number) + "-" + cstr(address.gst_state)

# don't remove this function it is used in tests
def test_method():
	'''test function'''
	return 'overridden'
<|endoftext|>


<commit_before>import mxnet as mx
import numpy as np
import cv2
from tools.rand_sampler import RandSampler

class DetIter(mx.io.DataIter):
    """
    Detection Iterator, which will feed data and label to network
    Optional data augmentation is performed when providing batch

    Parameters:
    ----------
    imdb : Imdb
        image database
    batch_size : int
        batch size
    data_shape : int or (int, int)
        image shape to be resized
    mean_pixels : float or float list
        [R, G, B], mean pixel values
    rand_samplers : list
        random cropping sampler list, if not specified, will
        use original image only
    rand_mirror : bool
        whether to randomly mirror input images, default False
    shuffle : bool
        whether to shuffle initial image list, default False
    rand_seed : int or None
        whether to use fixed random seed, default None
    max_crop_trial : bool
        if random crop is enabled, defines the maximum trial time
        if trial exceed this number, will give up cropping
    is_train : bool
        whether in training phase, default True, if False, labels might
        be ignored
    """
    def __init__(self, imdb, batch_size, data_shape, \
                 mean_pixels=[128, 128, 128], rand_samplers=[], \
                 rand_mirror=False, shuffle=False, rand_seed=None, \
                 is_train=True, max_crop_trial=50):
        super(DetIter, self).__init__()

        self._imdb = imdb
        self.batch_size = batch_size
        if isinstance(data_shape, int):
            data_shape = (data_shape, data_shape)
        self._data_shape = data_shape
        self._mean_pixels = mx.nd.array(mean_pixels).reshape((3,1,1))
        if not rand_samplers:
            self._rand_samplers = []
        else:
            if not isinstance(rand_samplers, list):
                rand_samplers = [rand_samplers]
            assert isinstance(rand_samplers[0], RandSampler), "Invalid rand sampler"
            self._rand_samplers = rand_samplers
        self.is_train = is_train
        self._rand_mirror = rand_mirror
        self._shuffle = shuffle
        if rand_seed:
            np.random.seed(rand_seed) # fix random seed
        self._max_crop_trial = max_crop_trial

        self._current = 0
        self._size = imdb.num_images
        self._index = np.arange(self._size)

        self._data = None
        self._label = None
        self._get_batch()

    @property
    def provide_data(self):
        return [(k, v.shape) for k, v in self._data.items()]

    @property
    def provide_label(self):
        if self.is_train:
            return [(k, v.shape) for k, v in self._label.items()]
        else:
            return []

    def reset(self):
        self._current = 0
        if self._shuffle:
            np.random.shuffle(self._index)

    def iter_next(self):
        return self._current < self._size

    def next(self):
        if self.iter_next():
            self._get_batch()
            data_batch = mx.io.DataBatch(data=self._data.values(),
                                   label=self._label.values(),
                                   pad=self.getpad(), index=self.getindex())
            self._current += self.batch_size
            return data_batch
        else:
            raise StopIteration

    def getindex(self):
        return self._current // self.batch_size

    def getpad(self):
        pad = self._current + self.batch_size - self._size
        return 0 if pad < 0 else pad

    def _get_batch(self):
        """
        Load data/label from dataset
        """
        batch_data = mx.nd.zeros((self.batch_size, 3, self._data_shape[0], self._data_shape[1]))
        batch_label = []
        for i in range(self.batch_size):
            if (self._current + i) >= self._size:
                if not self.is_train:
                    continue
                # use padding from middle in each epoch
                idx = (self._current + i + self._size // 2) % self._size
                index = self._index[idx]
            else:
                index = self._index[self._current + i]
            # index = self.debug_index
            im_path = self._imdb.image_path_from_index(index)
            with open(im_path, 'rb') as fp:
                img_content = fp.read()
            img = mx.img.imdecode(img_content)
            gt = self._imdb.label_from_index(index).copy() if self.is_train else None
            data, label = self._data_augmentation(img, gt)
            batch_data[i] = data
            if self.is_train:
                batch_label.append(label)
        self._data = {'data': batch_data}
        if self.is_train:
            self._label = {'label': mx.nd.array(np.array(batch_label))}
        else:
            self._label = {'label': None}

    def _data_augmentation(self, data, label):
        """
        perform data augmentations: crop, mirror, resize, sub mean, swap channels...
        """
        if self.is_train and self._rand_samplers:
            rand_crops = []
            for rs in self._rand_samplers:
                rand_crops += rs.sample(label)
            num_rand_crops = len(rand_crops)
            # randomly pick up one as input data
            if num_rand_crops > 0:
                index = int(np.random.uniform(0, 1) * num_rand_crops)
                width = data.shape[1]
                height = data.shape[0]
                crop = rand_crops[index][0]
                xmin = int(crop[0] * width)
                ymin = int(crop[1] * height)
                xmax = int(crop[2] * width)
                ymax = int(crop[3] * height)
                if xmin >= 0 and ymin >= 0 and xmax <= width and ymax <= height:
                    data = mx.img.fixed_crop(data, xmin, ymin, xmax-xmin, ymax-ymin)
                else:
                    # padding mode
                    new_width = xmax - xmin
                    new_height = ymax - ymin
                    offset_x = 0 - xmin
                    offset_y = 0 - ymin
                    data_bak = data
                    data = mx.nd.full((new_height, new_width, 3), 128, dtype='uint8')
                    data[offset_y:offset_y+height, offset_x:offset_x + width, :] = data_bak
                label = rand_crops[index][1]
        if self.is_train:
            interp_methods = [cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, \
                              cv2.INTER_NEAREST, cv2.INTER_LANCZOS4]
        else:
            interp_methods = [cv2.INTER_LINEAR]
        interp_method = interp_methods[int(np.random.uniform(0, 1) * len(interp_methods))]
        data = mx.img.imresize(data, self._data_shape[0], self._data_shape[1], interp_method)
        if self.is_train and self._rand_mirror:
            if np.random.uniform(0, 1) > 0.5:
                data = mx.nd.flip(data, axis=1)
                valid_mask = np.where(label[:, 0] > -1)[0]
                tmp = 1.0 - label[valid_mask, 1]
                label[valid_mask, 1] = 1.0 - label[valid_mask, 3]
                label[valid_mask, 3] = tmp
        data = mx.nd.transpose(data, (2,0,1))
        data = data.astype('float32')
        data = data - self._mean_pixels
        return data, label
<commit_msg>fix mx.img.resize arguments order (#5072)<commit_after>import mxnet as mx
import numpy as np
import cv2
from tools.rand_sampler import RandSampler

class DetIter(mx.io.DataIter):
    """
    Detection Iterator, which will feed data and label to network
    Optional data augmentation is performed when providing batch

    Parameters:
    ----------
    imdb : Imdb
        image database
    batch_size : int
        batch size
    data_shape : int or (int, int)
        image shape to be resized
    mean_pixels : float or float list
        [R, G, B], mean pixel values
    rand_samplers : list
        random cropping sampler list, if not specified, will
        use original image only
    rand_mirror : bool
        whether to randomly mirror input images, default False
    shuffle : bool
        whether to shuffle initial image list, default False
    rand_seed : int or None
        whether to use fixed random seed, default None
    max_crop_trial : bool
        if random crop is enabled, defines the maximum trial time
        if trial exceed this number, will give up cropping
    is_train : bool
        whether in training phase, default True, if False, labels might
        be ignored
    """
    def __init__(self, imdb, batch_size, data_shape, \
                 mean_pixels=[128, 128, 128], rand_samplers=[], \
                 rand_mirror=False, shuffle=False, rand_seed=None, \
                 is_train=True, max_crop_trial=50):
        super(DetIter, self).__init__()

        self._imdb = imdb
        self.batch_size = batch_size
        if isinstance(data_shape, int):
            data_shape = (data_shape, data_shape)
        self._data_shape = data_shape
        self._mean_pixels = mx.nd.array(mean_pixels).reshape((3,1,1))
        if not rand_samplers:
            self._rand_samplers = []
        else:
            if not isinstance(rand_samplers, list):
                rand_samplers = [rand_samplers]
            assert isinstance(rand_samplers[0], RandSampler), "Invalid rand sampler"
            self._rand_samplers = rand_samplers
        self.is_train = is_train
        self._rand_mirror = rand_mirror
        self._shuffle = shuffle
        if rand_seed:
            np.random.seed(rand_seed) # fix random seed
        self._max_crop_trial = max_crop_trial

        self._current = 0
        self._size = imdb.num_images
        self._index = np.arange(self._size)

        self._data = None
        self._label = None
        self._get_batch()

    @property
    def provide_data(self):
        return [(k, v.shape) for k, v in self._data.items()]

    @property
    def provide_label(self):
        if self.is_train:
            return [(k, v.shape) for k, v in self._label.items()]
        else:
            return []

    def reset(self):
        self._current = 0
        if self._shuffle:
            np.random.shuffle(self._index)

    def iter_next(self):
        return self._current < self._size

    def next(self):
        if self.iter_next():
            self._get_batch()
            data_batch = mx.io.DataBatch(data=self._data.values(),
                                   label=self._label.values(),
                                   pad=self.getpad(), index=self.getindex())
            self._current += self.batch_size
            return data_batch
        else:
            raise StopIteration

    def getindex(self):
        return self._current // self.batch_size

    def getpad(self):
        pad = self._current + self.batch_size - self._size
        return 0 if pad < 0 else pad

    def _get_batch(self):
        """
        Load data/label from dataset
        """
        batch_data = mx.nd.zeros((self.batch_size, 3, self._data_shape[0], self._data_shape[1]))
        batch_label = []
        for i in range(self.batch_size):
            if (self._current + i) >= self._size:
                if not self.is_train:
                    continue
                # use padding from middle in each epoch
                idx = (self._current + i + self._size // 2) % self._size
                index = self._index[idx]
            else:
                index = self._index[self._current + i]
            # index = self.debug_index
            im_path = self._imdb.image_path_from_index(index)
            with open(im_path, 'rb') as fp:
                img_content = fp.read()
            img = mx.img.imdecode(img_content)
            gt = self._imdb.label_from_index(index).copy() if self.is_train else None
            data, label = self._data_augmentation(img, gt)
            batch_data[i] = data
            if self.is_train:
                batch_label.append(label)
        self._data = {'data': batch_data}
        if self.is_train:
            self._label = {'label': mx.nd.array(np.array(batch_label))}
        else:
            self._label = {'label': None}

    def _data_augmentation(self, data, label):
        """
        perform data augmentations: crop, mirror, resize, sub mean, swap channels...
        """
        if self.is_train and self._rand_samplers:
            rand_crops = []
            for rs in self._rand_samplers:
                rand_crops += rs.sample(label)
            num_rand_crops = len(rand_crops)
            # randomly pick up one as input data
            if num_rand_crops > 0:
                index = int(np.random.uniform(0, 1) * num_rand_crops)
                width = data.shape[1]
                height = data.shape[0]
                crop = rand_crops[index][0]
                xmin = int(crop[0] * width)
                ymin = int(crop[1] * height)
                xmax = int(crop[2] * width)
                ymax = int(crop[3] * height)
                if xmin >= 0 and ymin >= 0 and xmax <= width and ymax <= height:
                    data = mx.img.fixed_crop(data, xmin, ymin, xmax-xmin, ymax-ymin)
                else:
                    # padding mode
                    new_width = xmax - xmin
                    new_height = ymax - ymin
                    offset_x = 0 - xmin
                    offset_y = 0 - ymin
                    data_bak = data
                    data = mx.nd.full((new_height, new_width, 3), 128, dtype='uint8')
                    data[offset_y:offset_y+height, offset_x:offset_x + width, :] = data_bak
                label = rand_crops[index][1]
        if self.is_train:
            interp_methods = [cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, \
                              cv2.INTER_NEAREST, cv2.INTER_LANCZOS4]
        else:
            interp_methods = [cv2.INTER_LINEAR]
        interp_method = interp_methods[int(np.random.uniform(0, 1) * len(interp_methods))]
        data = mx.img.imresize(data, self._data_shape[1], self._data_shape[0], interp_method)
        if self.is_train and self._rand_mirror:
            if np.random.uniform(0, 1) > 0.5:
                data = mx.nd.flip(data, axis=1)
                valid_mask = np.where(label[:, 0] > -1)[0]
                tmp = 1.0 - label[valid_mask, 1]
                label[valid_mask, 1] = 1.0 - label[valid_mask, 3]
                label[valid_mask, 3] = tmp
        data = mx.nd.transpose(data, (2,0,1))
        data = data.astype('float32')
        data = data - self._mean_pixels
        return data, label
<|endoftext|>