<commit_before># Copyright (C) 2010-2011 Mathijs de Bruin <mathijs@mathijsfietst.nl>
#
# This file is part of django-webshop.
#
# django-webshop is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

import logging
logger = logging.getLogger(__name__)

from decimal import Decimal

from django.db import models
from django.utils.translation import ugettext_lazy as _

from webshop.core.basemodels import AbstractPricedItemBase
from webshop.core.utils import get_model_from_string

from webshop.extensions.shipping.advanced.settings import \
    SHIPPING_METHOD_MODEL

from webshop.extensions.shipping.basemodels import \
    ShippedCartBase, ShippedCartItemBase, \
    ShippedOrderBase, ShippedOrderItemBase

class AutomaticShippingMixin(object):
    """
    Mixin class for shippable items for which the choice of method is
    automatic.
    """

    def get_shipping_method(self, **kwargs):
        """
        Return the shipping method used for the current item. This method
        should be overridden in subclasses. It should return `None` if
        shipping is not applicable for this item and hence, the shipping
        costs should be 0.
        """
        raise NotImplementedError

class CheapestShippingMixin(AutomaticShippingMixin):
    """ Shippable item which defaults to using the """

    def get_shipping_method(self, **kwargs):
        """
        Return the cheapest shipping method or an order or item.

        .. todo::
            This code is probably a bit too low-level.
        """

        assert 'order_methods' in kwargs or 'item_methods' in kwargs, \
            'We need either a restriction to order shipping methods or '+ \
            'item shipping methods in order to find the cheapest method.'

        shipping_method_class = get_model_from_string(SHIPPING_METHOD_MODEL)

        shipping_address = getattr(self, 'shipping_address', None)
        if not 'country' in kwargs and shipping_address:
            assert self.shipping_address.country

            country = self.shipping_address.country

            logger.debug(u'Using country %s to find cheapest shipping method for %s',
                         country, self)

            kwargs['country'] = country

        shipping_method = shipping_method_class.get_cheapest(**kwargs)

        logger.debug(u'Found shipping method %s for %s', shipping_method, self)

        return shipping_method

class CalculatedShippingItemMixin(object):
    def get_shipping_method(self, **kwargs):
        superclass = super(CalculatedShippingItemMixin, self)

        method = superclass.get_shipping_method(item_methods=True)

        return method

    def get_total_shipping_costs(self, **kwargs):
        method = self.get_shipping_method(**kwargs)

        if method:
            costs = method.get_cost()
            logger.info(u'Shipping method %s found for object %s with args %s',
                        method, self, kwargs)

        else:
            logger.info(u'No shipping method found for kwargs %s and object %s',
                        kwargs, self)

            costs = Decimal('0.00')

        assert isinstance(costs, Decimal)

        return costs

class CalculatedShippingOrderMixin(CalculatedShippingItemMixin):
    def get_shipping_method(self, **kwargs):
        superclass = super(CalculatedShippingItemMixin, self)

        method = superclass.get_shipping_method(order_methods=True)

        return method

    def get_order_shipping_costs(self, **kwargs):
        superclass = super(CalculatedShippingOrderMixin, self)
        return superclass.get_total_shipping_costs()

class PersistentShippedItemBase(models.Model):
    """
    Mixin class for `Order`'s and `OrderItem`'s for which the shipping method
    is stored persistently upon calling the `update_shipping` method.
    """

    class Meta:
        abstract = True

    shipping_method = models.ForeignKey(SHIPPING_METHOD_MODEL,
                                        verbose_name=_('shipping method'),
                                        null=True, blank=True)

    def update_shipping(self):
        """
        Call `update_shipping` on the superclass and get the shipping method,
        store the resulting `ShippingMethod` on the `shipping_method`
        property.
        """
        super(PersistentShippedItemBase, self).update_shipping()

        assert self.pk, 'Object not saved, need PK for assigning method'

        method = self.get_shipping_method()

        assert self.get_shipping_costs() == Decimal('0.00') or \
            (self.get_shipping_costs() and method)

        logger.debug(u'Storing shipping method %s for %s', method, self)

        self.shipping_method = method

class ShippedCartMixin(CalculatedShippingOrderMixin, CheapestShippingMixin, ShippedCartBase):
    """ Base class for shopping carts with shippable items. """
    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order. """
    #     superclass = super(ShippedCartMixin, self)
    #     return superclass.get_valid_shipping_methods(order_methods=True,
    #                                                  **kwargs)

class ShippedCartItemMixin(CalculatedShippingItemMixin, CheapestShippingMixin, ShippedCartItemBase):
    """ Base class for shopping cart items which are shippable. """
    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order item. """
    #     superclass = super(ShippedCartItemMixin, self)
    #     return superclass.get_valid_shipping_methods(item_methods=True,
    #                                                  product=self.product,
    #                                                  **kwargs)

# class ShippedOrderMixin(CalculatedShippingOrderMixin, CheapestShippingMixin, ShippedOrderBase):
class ShippedOrderMixin(PersistentShippedItemBase,
                        ShippedOrderBase,
                        CalculatedShippingOrderMixin,
                        CheapestShippingMixin):
    """ Base class for orders with a shipping_method. """

    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order. """
    #     superclass = super(ShippedOrderMixin, self)
    #     return superclass.get_valid_shipping_methods(order_methods=True,
    #                                                  **kwargs)

class ShippedOrderItemMixin(PersistentShippedItemBase,
                            CalculatedShippingItemMixin,
                            CheapestShippingMixin,
                            ShippedOrderItemBase):
    """
    Base class for orderitems which can have individual shipping costs
    applied to them.
    """

    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order item. """
    #     superclass = super(ShippedCartItemMixin, self)
    #     return superclass.get_valid_shipping_methods(item_methods=True,
    #                                                  product=self.product,
    #                                                  **kwargs)
<commit_msg>Make sure we pass the order and item prices for calculating shipping methods.<commit_after># Copyright (C) 2010-2011 Mathijs de Bruin <mathijs@mathijsfietst.nl>
#
# This file is part of django-webshop.
#
# django-webshop is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

import logging
logger = logging.getLogger(__name__)

from decimal import Decimal

from django.db import models
from django.utils.translation import ugettext_lazy as _

from webshop.core.basemodels import AbstractPricedItemBase
from webshop.core.utils import get_model_from_string

from webshop.extensions.shipping.advanced.settings import \
    SHIPPING_METHOD_MODEL

from webshop.extensions.shipping.basemodels import \
    ShippedCartBase, ShippedCartItemBase, \
    ShippedOrderBase, ShippedOrderItemBase

class AutomaticShippingMixin(object):
    """
    Mixin class for shippable items for which the choice of method is
    automatic.
    """

    def get_shipping_method(self, **kwargs):
        """
        Return the shipping method used for the current item. This method
        should be overridden in subclasses. It should return `None` if
        shipping is not applicable for this item and hence, the shipping
        costs should be 0.
        """
        raise NotImplementedError

class CheapestShippingMixin(AutomaticShippingMixin):
    """ Shippable item which defaults to using the """

    def get_shipping_method(self, **kwargs):
        """
        Return the cheapest shipping method or an order or item.

        .. todo::
            This code is probably a bit too low-level.
        """

        assert 'order_methods' in kwargs or 'item_methods' in kwargs, \
            'We need either a restriction to order shipping methods or '+ \
            'item shipping methods in order to find the cheapest method.'

        shipping_method_class = get_model_from_string(SHIPPING_METHOD_MODEL)

        shipping_address = getattr(self, 'shipping_address', None)
        if not 'country' in kwargs and shipping_address:
            assert self.shipping_address.country

            country = self.shipping_address.country

            logger.debug(u'Using country %s to find cheapest shipping method for %s',
                         country, self)

            kwargs['country'] = country

        shipping_method = shipping_method_class.get_cheapest(**kwargs)

        logger.debug(u'Found shipping method %s for %s', shipping_method, self)

        return shipping_method

class CalculatedShippingItemMixin(object):
    def get_shipping_method(self, **kwargs):
        superclass = super(CalculatedShippingItemMixin, self)

        price = self.get_price_without_shipping()

        method = superclass.get_shipping_method(item_methods=True,
                                                item_price=price)

        return method

    def get_total_shipping_costs(self, **kwargs):
        method = self.get_shipping_method(**kwargs)

        if method:
            costs = method.get_cost()
            logger.info(u'Shipping method %s found for object %s with args %s',
                        method, self, kwargs)

        else:
            logger.info(u'No shipping method found for kwargs %s and object %s',
                        kwargs, self)

            costs = Decimal('0.00')

        assert isinstance(costs, Decimal)

        return costs

class CalculatedShippingOrderMixin(CalculatedShippingItemMixin):
    def get_shipping_method(self, **kwargs):
        superclass = super(CalculatedShippingItemMixin, self)

        price = self.get_price_without_shipping()

        method = superclass.get_shipping_method(order_methods=True,
                                                order_price=price)

        return method

    def get_order_shipping_costs(self, **kwargs):
        superclass = super(CalculatedShippingOrderMixin, self)
        return superclass.get_total_shipping_costs()

class PersistentShippedItemBase(models.Model):
    """
    Mixin class for `Order`'s and `OrderItem`'s for which the shipping method
    is stored persistently upon calling the `update_shipping` method.
    """

    class Meta:
        abstract = True

    shipping_method = models.ForeignKey(SHIPPING_METHOD_MODEL,
                                        verbose_name=_('shipping method'),
                                        null=True, blank=True)

    def update_shipping(self):
        """
        Call `update_shipping` on the superclass and get the shipping method,
        store the resulting `ShippingMethod` on the `shipping_method`
        property.
        """

        super(PersistentShippedItemBase, self).update_shipping()

        assert self.pk, 'Object not saved, need PK for assigning method'

        method = self.get_shipping_method()

        assert self.get_shipping_costs() == Decimal('0.00') or \
            (self.get_shipping_costs() and method)

        logger.debug(u'Storing shipping method %s for %s', method, self)

        self.shipping_method = method

class ShippedCartMixin(CalculatedShippingOrderMixin, CheapestShippingMixin, ShippedCartBase):
    """ Base class for shopping carts with shippable items. """
    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order. """
    #     superclass = super(ShippedCartMixin, self)
    #     return superclass.get_valid_shipping_methods(order_methods=True,
    #                                                  **kwargs)

class ShippedCartItemMixin(CalculatedShippingItemMixin, CheapestShippingMixin, ShippedCartItemBase):
    """ Base class for shopping cart items which are shippable. """
    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order item. """
    #     superclass = super(ShippedCartItemMixin, self)
    #     return superclass.get_valid_shipping_methods(item_methods=True,
    #                                                  product=self.product,
    #                                                  **kwargs)

# class ShippedOrderMixin(CalculatedShippingOrderMixin, CheapestShippingMixin, ShippedOrderBase):
class ShippedOrderMixin(PersistentShippedItemBase,
                        ShippedOrderBase,
                        CalculatedShippingOrderMixin,
                        CheapestShippingMixin):
    """ Base class for orders with a shipping_method. """

    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order. """
    #     superclass = super(ShippedOrderMixin, self)
    #     return superclass.get_valid_shipping_methods(order_methods=True,
    #                                                  **kwargs)

class ShippedOrderItemMixin(PersistentShippedItemBase,
                            CalculatedShippingItemMixin,
                            CheapestShippingMixin,
                            ShippedOrderItemBase):
    """
    Base class for orderitems which can have individual shipping costs
    applied to them.
    """

    class Meta:
        abstract = True

    # def get_valid_shipping_methods(self, **kwargs):
    #     """ Return valid shipping_methods for the current order item. """
    #     superclass = super(ShippedCartItemMixin, self)
    #     return superclass.get_valid_shipping_methods(item_methods=True,
    #                                                  product=self.product,
    #                                                  **kwargs)
<|endoftext|>


<commit_before>from django.apps import AppConfig
from django.db.models import signals

class SlurmInvoicesConfig(AppConfig):
    name = 'nodeconductor_assembly_waldur.slurm_invoices'
    verbose_name = 'SLURM invoices'

    def ready(self):
        from nodeconductor_assembly_waldur.invoices import registrators
        from waldur_slurm import models as slurm_models
        from . import handlers, registrators as slurm_registrators

        registrators.RegistrationManager.add_registrator(
            slurm_models.Allocation,
            slurm_registrators.AllocationRegistrator
        )

        signals.post_save.connect(
            handlers.add_new_allocation_to_invoice,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.add_new_allocation_to_invoice',
        )

        signals.post_save.connect(
            handlers.terminate_invoice_when_allocation_cancelled,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.terminate_invoice_when_allocation_cancelled',
        )

        signals.pre_delete.connect(
            handlers.terminate_invoice_when_allocation_deleted,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.terminate_invoice_when_allocation_deleted',
        )

        signals.post_save.connect(
            handlers.update_invoice_item_on_allocation_usage_update,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.update_invoice_item_on_allocation_usage_update',
        )
<commit_msg>Rename SLURM invoices application for better Django dashoard menu item.<commit_after>from django.apps import AppConfig
from django.db.models import signals

class SlurmInvoicesConfig(AppConfig):
    name = 'nodeconductor_assembly_waldur.slurm_invoices'
    verbose_name = 'Batch packages'

    def ready(self):
        from nodeconductor_assembly_waldur.invoices import registrators
        from waldur_slurm import models as slurm_models
        from . import handlers, registrators as slurm_registrators

        registrators.RegistrationManager.add_registrator(
            slurm_models.Allocation,
            slurm_registrators.AllocationRegistrator
        )

        signals.post_save.connect(
            handlers.add_new_allocation_to_invoice,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.add_new_allocation_to_invoice',
        )

        signals.post_save.connect(
            handlers.terminate_invoice_when_allocation_cancelled,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.terminate_invoice_when_allocation_cancelled',
        )

        signals.pre_delete.connect(
            handlers.terminate_invoice_when_allocation_deleted,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.terminate_invoice_when_allocation_deleted',
        )

        signals.post_save.connect(
            handlers.update_invoice_item_on_allocation_usage_update,
            sender=slurm_models.Allocation,
            dispatch_uid='waldur_slurm.handlers.update_invoice_item_on_allocation_usage_update',
        )
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand

from xbrowse_server.base.models import Project, VCFFile
from xbrowse_server import sample_management

import os
import yaml
from xbrowse.utils import slugify

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('args', nargs='*')

    def handle(self, *args, **options):

        project_id = args[0]
        project = Project.objects.get(project_id=project_id)
        project_dir = os.path.abspath(args[1])
        project_yaml_file = os.path.join(project_dir, 'project.yaml')

        project_spec = yaml.load(open(project_yaml_file))

        # load in sample IDs that we'll use for the project
        sample_id_file = os.path.join(project_dir, project_spec['sample_id_list'])
        sample_ids = [l.strip('\n') for l in open(sample_id_file)]
        sample_ids = [slugify(s, separator='_') for s in sample_ids]
        sample_management.add_indiv_ids_to_project(project, sample_ids)

        # set meta info
        project.project_name = project_spec['project_name']
        project.save()

        # nicknames
        if 'nicknames' in project_spec:
            # todo
            pass

        # load individuals
        if 'ped_files' in project_spec:
            for relative_path in project_spec['ped_files']:
                ped_file_path = os.path.join(project_dir, relative_path)
                sample_management.update_project_from_fam(project, open(ped_file_path))
                # todo: add awesome-slugify to above

        # set VCF files
        if 'vcf_files' in project_spec:
            for relative_path in project_spec['vcf_files']:
                vcf_file_path = os.path.join(project_dir, relative_path)
                # todo: this should be a fn somewhere that add_vcf_to_project uses too
                vcf_file = VCFFile.objects.get_or_create(file_path=vcf_file_path)[0]
                sample_management.add_vcf_file_to_project(project, vcf_file)
<commit_msg>fix #217 - print help when loading non-existent project<commit_after>from django.core.management.base import BaseCommand

from xbrowse_server.base.models import Project, VCFFile
from xbrowse_server import sample_management

import os,sys
import yaml
from xbrowse.utils import slugify

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('args', nargs='*')

    def handle(self, *args, **options):

        project_id = args[0]
        try:
            project = Project.objects.get(project_id=project_id)

        except Project.DoesNotExist:
            print("\nError:")
            print("\nNo project could be found with id '%s'" % project_id)
            print("")
            print("Please use the add_project command first to add this project before loading it.")
            print("")
            sys.exit(1)

        project_dir = os.path.abspath(args[1])
        project_yaml_file = os.path.join(project_dir, 'project.yaml')

        project_spec = yaml.load(open(project_yaml_file))

        # load in sample IDs that we'll use for the project
        sample_id_file = os.path.join(project_dir, project_spec['sample_id_list'])
        sample_ids = [l.strip('\n') for l in open(sample_id_file)]
        sample_ids = [slugify(s, separator='_') for s in sample_ids]
        sample_management.add_indiv_ids_to_project(project, sample_ids)

        # set meta info
        project.project_name = project_spec['project_name']
        project.save()

        # nicknames
        if 'nicknames' in project_spec:
            # todo
            pass

        # load individuals
        if 'ped_files' in project_spec:
            for relative_path in project_spec['ped_files']:
                ped_file_path = os.path.join(project_dir, relative_path)
                sample_management.update_project_from_fam(project, open(ped_file_path))
                # todo: add awesome-slugify to above

        # set VCF files
        if 'vcf_files' in project_spec:
            for relative_path in project_spec['vcf_files']:
                vcf_file_path = os.path.join(project_dir, relative_path)
                # todo: this should be a fn somewhere that add_vcf_to_project uses too
                vcf_file = VCFFile.objects.get_or_create(file_path=vcf_file_path)[0]
                sample_management.add_vcf_file_to_project(project, vcf_file)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2017 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo.tests import common

class TestStockPickingShowReturn(common.TransactionCase):
    def setUp(self):
        super(TestStockPickingShowReturn, self).setUp()
        self.product = self.env['product.product'].create({
            'name': 'Test product',
        })
        picking_type = self.env.ref('stock.picking_type_internal')
        self.picking = self.env['stock.picking'].create({
            'picking_type_id': picking_type.id,
            'location_id': picking_type.default_location_src_id.id,
            'location_dest_id': picking_type.default_location_dest_id.id,
            'move_lines': [(0, 0, {
                'name': self.product.name,
                'product_id': self.product.id,
                'product_uom': self.product.uom_id.id,
            })],
        })

    def test_returned_ids_field(self):
        self.picking.force_assign()
        self.picking.do_transfer()
        wizard = self.env['stock.return.picking'].with_context(
            active_ids=self.picking.ids, active_id=self.picking.id).create({})
        wizard.create_returns()
        self.assertTrue(self.picking.returned_ids)
<commit_msg>[IMP] stock_*: Execute tests at the end<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2017 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo.tests import common

@common.at_install(False)
@common.post_install(True)
class TestStockPickingShowReturn(common.TransactionCase):
    def setUp(self):
        super(TestStockPickingShowReturn, self).setUp()
        self.product = self.env['product.product'].create({
            'name': 'Test product',
        })
        picking_type = self.env.ref('stock.picking_type_internal')
        self.picking = self.env['stock.picking'].create({
            'picking_type_id': picking_type.id,
            'location_id': picking_type.default_location_src_id.id,
            'location_dest_id': picking_type.default_location_dest_id.id,
            'move_lines': [(0, 0, {
                'name': self.product.name,
                'product_id': self.product.id,
                'product_uom': self.product.uom_id.id,
            })],
        })

    def test_returned_ids_field(self):
        self.picking.force_assign()
        self.picking.do_transfer()
        wizard = self.env['stock.return.picking'].with_context(
            active_ids=self.picking.ids, active_id=self.picking.id).create({})
        wizard.create_returns()
        self.assertTrue(self.picking.returned_ids)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#   Copyright 2014-2018 PUNCH Cyber Analytics Group
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from typing import Optional

from stoq.data_classes import ArchiverResponse, Payload, RequestMeta
from stoq.plugins import ArchiverPlugin

class DummyArchiver(ArchiverPlugin):
    def archive(
        self, payload: Payload, request_meta: RequestMeta
    ) -> Optional[ArchiverResponse]:
        return None

    def get(self, task: str) -> Optional[Payload]:
        return None
<commit_msg>Fix test signature value type for task<commit_after>#!/usr/bin/env python3

#   Copyright 2014-2018 PUNCH Cyber Analytics Group
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from typing import Optional

from stoq.plugins import ArchiverPlugin
from stoq.data_classes import ArchiverResponse, Payload, RequestMeta

class DummyArchiver(ArchiverPlugin):
    def archive(
        self, payload: Payload, request_meta: RequestMeta
    ) -> Optional[ArchiverResponse]:
        return None

    def get(self, task: ArchiverResponse) -> Optional[Payload]:
        return None
<|endoftext|>


<commit_before># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: miha@reciprocitylabs.com
# Maintained By: miha@reciprocitylabs.com

"""List of all error and warning messages for imports."""

WRONG_FILE_TYPE = (u"Line {line}: Wrong file type. Only .csv files are"
                   u" supported. Please upload a .csv file.")

MISSING_COLUMN = (u"Line {line}: Missing mandatory column{s} {column_names},"
                  u" when adding object.")

MISSING_CUSTOM_ATTRIBUTE_COLUMN = (u"Line {line}: Missing custom column"
                                   " {column_name}, when adding object.")

WRONG_OBJECT_TYPE = u"Line {line}: Unknown object type '{object_name}'"

UNKNOWN_COLUMN = (u"Line {line}: Attribute '{column_name}' does not"
                  u" exist. Column will be ignored.")

DELETE_UNKNOWN_OBJECT = (u"Line {line}: Object '{slug}' does not exist, so it"
                         u" can't be deleted.")

DUPLICATE_COLUMN = (u"Line {line}: Duplicate columns found {duplicates}."
                    u" Object block will be ignored.")

DUPLICATE_VALUE = (u"Line {line}: {column_name} '{value}' already exists."
                   u"Record will be ignored.")

DUPLICATE_VALUE_IN_CSV = (u"Lines {line_list} have same {column_name}"
                          u" '{value}'. Line{s} {ignore_lines} will be"
                          u" ignored.")

MAP_UNMAP_CONFLICT = (u"Line {line}: Object '{slug}' scheduled for mapping and"
                      u" unmapping at the same time. Mapping rule update will"
                      u" be ignored.")

UNKNOWN_OBJECT = (u"Line {line}: {object_type} '{slug}' doesn't exist, so it"
                  u" can't be mapped/unmapped.")

WHITESPACE_WARNING = (u"Line {line}: Field {column_name} contains multiple"
                      u"spaces together, that will be merged into one.")

UNKNOWN_USER_WARNING = (u"Line {line}: Specified user '{email}' does not "
                        u"exist. That user will be ignored.")

OWNER_MISSING = (u"Line {line}: {column_name} field does not contain a valid "
                 u"value. You will be assigned as object {column_name}.")

WRONG_MULTI_VALUE = (u"Line {line}: {column_name} contains invalid line. The"
                     u" value '{value}' will be ignored.")

WRONG_VALUE = (u"Line {line}: {column_name} contains invalid data. The value"
               u" will be ignored.")

WRONG_VALUE_ERROR = (u"Line {line}: {column_name} contains invalid data. The"
                     u" line will be ignored.")

WRONG_REQUIRED_VALUE = (u"Line {line}: Required field {column_name} contains"
                        u" invalid data '{value}'. The default value will be"
                        u" used.")

MISSING_VALUE_WARNING = (u"Line {line}: Field {column_name} is required. The "
                         u"default value '{default_value}' will be used.")

MISSING_VALUE_ERROR = (u"Line {line}: Field {column_name} is required. The "
                       u"line will be ignored.")

PERMISSION_ERROR = (u"Line {line}: You don't have permission to update/delete"
                    u" this record.")

MAPPING_PERMISSION_ERROR = (u"Line {line}: You don't have permission to update"
                            " mappings for {object_type}: {slug}.")

DELETE_NEW_OBJECT_ERROR = (u"Line {line}: Tried to create and delete the same"
                           " object {object_type}: {slug} in one import.")

DELETE_CASCADE_ERROR = (u"Line {line}: Cannot delete object {object_type}:"
                        " {slug} without deleting other objects")

UNKNOWN_ERROR = "Line {line}: Import failed due to unknown error."

REQUEST_INVALID_STATE = ("LINE {line}: Can not set Request state to Completed "
                         "or Verified via imports.")

UNKNOWN_DATE_FORMAT = (u"Line {line}: Field {column_name} contains invalid "
                       u"date format, use YYYY-MM-DD or MM/DD/YYYY. The line "
                       u"will be ignored.")

UNSUPPORTED_LIST_ERROR = (u"Line {line}: Field {column_name} does not support "
                          u"{value_type}.")
<commit_msg>Update import error messages<commit_after># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: miha@reciprocitylabs.com
# Maintained By: miha@reciprocitylabs.com

"""List of all error and warning messages for imports."""

WRONG_FILE_TYPE = (u"Line {line}: Wrong file type. Only .csv files are"
                   u" supported. Please upload a .csv file.")

MISSING_COLUMN = (u"Line {line}: Missing mandatory column{s} {column_names},"
                  u" when adding object.")

MISSING_CUSTOM_ATTRIBUTE_COLUMN = (u"Line {line}: Missing custom column"
                                   " {column_name}, when adding object.")

WRONG_OBJECT_TYPE = u"Line {line}: Unknown object type '{object_name}'"

UNKNOWN_COLUMN = (u"Line {line}: Attribute '{column_name}' does not"
                  u" exist. Column will be ignored.")

DELETE_UNKNOWN_OBJECT = (u"Line {line}: Object '{slug}' does not exist, so it"
                         u" can't be deleted.")

DUPLICATE_COLUMN = (u"Line {line}: Duplicate columns found {duplicates}."
                    u" Object block will be ignored.")

DUPLICATE_VALUE = (u"Line {line}: {column_name} '{value}' already exists."
                   u"Record will be ignored.")

DUPLICATE_VALUE_IN_CSV = (u"Lines {line_list} have same {column_name}"
                          u" '{value}'. Line{s} {ignore_lines} will be"
                          u" ignored.")

MAP_UNMAP_CONFLICT = (u"Line {line}: Object '{slug}' scheduled for mapping and"
                      u" unmapping at the same time. Mapping rule update will"
                      u" be ignored.")

UNKNOWN_OBJECT = (u"Line {line}: {object_type} '{slug}' doesn't exist, so it"
                  u" can't be mapped/unmapped.")

WHITESPACE_WARNING = (u"Line {line}: Field {column_name} contains multiple"
                      u"spaces together, that will be merged into one.")

UNKNOWN_USER_WARNING = (u"Line {line}: Specified user '{email}' does not "
                        u"exist. That user will be ignored.")

OWNER_MISSING = (u"Line {line}: {column_name} field does not contain a valid "
                 u"value. You will be assigned as object {column_name}.")

WRONG_MULTI_VALUE = (u"Line {line}: {column_name} contains invalid line. The"
                     u" value '{value}' will be ignored.")

WRONG_VALUE = (u"Line {line}: Field '{column_name}' contains invalid data. The"
               u" value will be ignored.")

WRONG_VALUE_ERROR = (u"Line {line}: Field '{column_name}' contains invalid "
                     u"data. The line will be ignored.")

WRONG_REQUIRED_VALUE = (u"Line {line}: Required field {column_name} contains"
                        u" invalid data '{value}'. The default value will be"
                        u" used.")

MISSING_VALUE_WARNING = (u"Line {line}: Field {column_name} is required. The "
                         u"default value '{default_value}' will be used.")

MISSING_VALUE_ERROR = (u"Line {line}: Field {column_name} is required. The "
                       u"line will be ignored.")

PERMISSION_ERROR = (u"Line {line}: You don't have permission to update/delete"
                    u" this record.")

MAPPING_PERMISSION_ERROR = (u"Line {line}: You don't have permission to update"
                            " mappings for {object_type}: {slug}.")

DELETE_NEW_OBJECT_ERROR = (u"Line {line}: Tried to create and delete the same"
                           " object {object_type}: {slug} in one import.")

DELETE_CASCADE_ERROR = (u"Line {line}: Cannot delete object {object_type}:"
                        " {slug} without deleting other objects")

UNKNOWN_ERROR = "Line {line}: Import failed due to unknown error."

REQUEST_INVALID_STATE = ("LINE {line}: Can not set Request state to Completed "
                         "or Verified via imports.")

UNKNOWN_DATE_FORMAT = (u"Line {line}: Field {column_name} contains invalid "
                       u"date format, use YYYY-MM-DD or MM/DD/YYYY. The line "
                       u"will be ignored.")

UNSUPPORTED_LIST_ERROR = (u"Line {line}: Field {column_name} does not support "
                          u"{value_type}.")
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2006-2015 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import urllib2

class MethodRequest(urllib2.Request):
    '''
    Used to create HEAD/PUT/DELETE/... requests with urllib2
    '''

    def set_method(self, method):
        self.method = method.upper()

    def get_method(self):
        return getattr(self, 'method', urllib2.Request.get_method(self))
<commit_msg>Minor style update<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2006-2015 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import urllib2

class MethodRequest(urllib2.Request):
    """
    Used to create HEAD/PUT/DELETE/... requests with urllib2
    """

    def set_method(self, method):
        self.method = method.upper()

    def get_method(self):
        return getattr(self, 'method', urllib2.Request.get_method(self))
<|endoftext|>


<commit_before>#!/usr/bin/python

import platform
import datetime
import time
import os
from pymongo import Connection
import datetime
import time
import hashlib
import codecs
import sys
import getopt
import ConfigParser

# Para leer donde esta instalado el mongodb
config = ConfigParser.RawConfigParser()
config.read('/opt/pdi/rpmControler/rpmControler.ini')

ip_mongo = config.get('mongo', 'ip')
port_mongo = config.getint('mongo','port')

def run(string, busqueda):

  #connect to mongodb

  connection = Connection(ip_mongo, port_mongo)

  # get database

  db = connection.rpms_database

  #get one collection

  collection = db.rpms_collection

  info_pcs = db.info_hosts

  rpms = db.rpms

  id_=""

  if busqueda == "IP":
    #print "option: IP"
    for host in info_pcs.find({"ips": string}):
      print "id=%(_id)s id_=%(id_)s name=%(name)s ips=%(ips)s date=%(date)s system=%(system)s release=%(release)s version=%(version)s distribution=%(distribution)s" % host
      id_=host["id_"]
    for record in rpms.find({"id_": id_}):
      # because record is a dict, we get you use lots of python magic
      print "id=%(_id)s id_=%(id_)s name=%(rpm)s date_process=%(date_process)s date_installed=%(date_installed)s" % record
  elif busqueda == "HOST":
    #print "option: HOST"
    for host in info_pcs.find({"name": string}):
      print "id=%(_id)s id_=%(id_)s name=%(name)s ips=%(ips)s date=%(date)s system=%(system)s release=%(release)s version=%(version)s distribution=%(distribution)s" % host
      id_=host["id_"]
    for record in rpms.find({"id_": id_}):
      # because record is a dict, we get you use lots of python magic
      print "id=%(_id)s id_=%(id_)s name=%(rpm)s date_process=%(date_process)s date_installed=%(date_installed)s" % record
  elif busqueda == "ONLYHOST":
    for host in info_pcs.find({"name": string}):
      print "id=%(_id)s id_=%(id_)s name=%(name)s ips=%(ips)s date=%(date)s system=%(system)s release=%(release)s version=%(version)s distribution=%(distribution)s" % host
      id_=host["id_"]
  elif busqueda == "ONLYRPM":
    #print "option: ONLYRPM"
    for host in info_pcs.find({"name": string}):
      #print "id=%(_id)s id_=%(id_)s name=%(name)s ips=%(ips)s date=%(date)s system=%(system)s release=%(release)s version=%(version)s distribution=%(distribution)s" % host
      id_=host["id_"]
    for record in rpms.find({"id_": id_}):
      # because record is a dict, we get you use lots of python magic
      print "id=%(_id)s id_=%(id_)s name=%(rpm)s date_process=%(date_process)s date_installed=%(date_installed)s" % record

def main():

  try:
    optsList, argsList = getopt.getopt(sys.argv[1:], "i:h:o:r:", ["ip=", "host=", "onlyhost", "onlyrpm"])

  except getopt.GetoptError, err:
    #Error handling for unknown or incorrect number of options
    print str(err)
    sys.exit(2)

  #dependiendo del parametro hago un find u otro

  for opt, arg in optsList:
    if opt in ('-i', '--ip'):
        ip_find = arg
        run(arg,"IP")
    elif opt in ('-h', '--host'):
        run(arg,"HOST")
    elif opt in ('-o', '--onlyhost'):
        run(arg,"ONLYHOST")
    elif opt in ('-r', '--onlyrpm'):
        run(arg,"ONLYRPM")
    else:
        print "Unhandled option"
        sys.exit(2)

main()
<commit_msg>refactor rpmFind<commit_after>#!/usr/bin/python

import platform
import datetime
import time
import os
from pymongo import Connection
import datetime
import time
import hashlib
import codecs
import sys
import getopt
import ConfigParser
import socket

# Para leer donde esta instalado el mongodb
config = ConfigParser.RawConfigParser()
config.read('/opt/pdi/rpmControler/rpmControler.ini')

ip_mongo = '127.0.0.1'
port_mongo = 27017

def find_collection():
  connection = Connection(ip_mongo, port_mongo)

  db = connection.rpmdb

  for collection_ in db.collection_names():
    if collection_ != "system.indexes":
      print collection_

def find_fqdn(fqdn):

  print "fqdn:",fqdn
  #fqdn = socket.getfqdn()

  #mongo = mongo_api.Info()
  #db = mongo_api.mongo_con(mongo.ip_mongo, mongo.port_mongo, fqdn)
  #db[collection].find({key:{'$regex':pattern}})
  #connect to mongodb

  connection = Connection(ip_mongo, port_mongo)

  # get database

  db = connection.rpmdb

  #get one collection

  #fqdn = socket.getfqdn()
  #fqdn = fqdn.replace("-", "_")
  #fqdn = fqdn.replace(".", "_")

  rpms = db[fqdn]

  for host in rpms.find({"fqdn": fqdn}):
    print "id=%(_id)s fqdn=%(fqdn)s system=%(system)s release=%(SO_release)s version=%(SO_version)s date=%(date)s distribution=%(SO_distribution)s" % host
  for record in rpms.find({"deleted":{'$exists': True}}):
    # because record is a dict, we get you use lots of python magic
    print "id=%(_id)s date=%(date)s version=%(version)s review_date=%(review_date)s name=%(name)s deleted=%(deleted)s release=%(release)s" % record

def find_ip(ip):
  print "ip:",ip
  connection = Connection(ip_mongo, port_mongo)

  # get database

  db = connection.rpmdb

  #get one collection

  for test in db.collection_names():
    print test

    rpms = db[test]

    for host in rpms.find({"ip": ip, "SO_release":{'$exists': True}}):
      print "id=%(_id)s fqdn=%(fqdn)s system=%(system)s release=%(SO_release)s version=%(SO_version)s date=%(date)s distribution=%(SO_distribution)s" % host
    for record in rpms.find({"ip": ip, "deleted":{'$exists': True}}):
      # because record is a dict, we get you use lots of python magic
      print "id=%(_id)s date=%(date)s version=%(version)s review_date=%(review_date)s name=%(name)s deleted=%(deleted)s release=%(release)s" % record

def main():

  try:
    optsList, argsList = getopt.getopt(sys.argv[1:], "l:i:h:")

  except getopt.GetoptError, err:
    #Error handling for unknown or incorrect number of options
    print str(err)
    sys.exit(2)

  #dependiendo del parametro hago un find u otro

  for opt, arg in optsList:
    if opt in ('-l', '--list'):
        find_collection()
    elif opt in ('-h', '--host'):
        find_fqdn(arg)
    elif opt in ('-i', '--ip'):
        find_ip(arg)
    else:
        print "Unhandled option"
        sys.exit(2)

main()
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import logging
import socket

from chrome_remote_control import tab_crash_exception
from chrome_remote_control import util
from chrome_remote_control import websocket

class InspectorException(Exception):
  pass

class InspectorBackend(object):
  def __init__(self, backend, descriptor):
    self._backend = backend
    self._descriptor = descriptor
    self._socket_url = descriptor['webSocketDebuggerUrl']
    self._socket = websocket.create_connection(
        descriptor['webSocketDebuggerUrl'])
    self._next_request_id = 0
    self._domain_handlers = {}
    self._cur_socket_timeout = 0

  def Close(self):
    for _, handlers in self._domain_handlers.items():
      _, will_close_handler = handlers
      will_close_handler()
    self._domain_handlers = {}
    self._socket.close()
    self._socket = None
    self._backend = None

  def DispatchNotifications(self, timeout=10):
    self._SetTimeout(timeout)
    try:
      data = self._socket.recv()
    except socket.error:
      if self._backend.DoesDebuggerUrlExist(self._socket_url):
        return
      raise tab_crash_exception.TabCrashException()

    res = json.loads(data)
    logging.debug('got [%s]', data)
    if 'method' not in res:
      return

    mname = res['method']
    dot_pos = mname.find('.')
    domain_name = mname[:dot_pos]
    if domain_name in self._domain_handlers:
      try:
        self._domain_handlers[domain_name][0](res)
      except Exception:
        import traceback
        traceback.print_exc()

  def SendAndIgnoreResponse(self, req):
    req['id'] = self._next_request_id
    self._next_request_id += 1
    self._socket.send(json.dumps(req))

  def _SetTimeout(self, timeout):
    if self._cur_socket_timeout != timeout:
      self._socket.settimeout(timeout)
      self._cur_socket_timeout = timeout

  def SyncRequest(self, req, timeout=10):
    # TODO(nduca): Listen to the timeout argument
    # pylint: disable=W0613
    self._SetTimeout(timeout)

    req['id'] = self._next_request_id
    self._next_request_id += 1
    self._socket.send(json.dumps(req))

    while True:
      try:
        data = self._socket.recv()
      except socket.error:
        if self._backend.DoesDebuggerUrlExist(self._socket_url):
          raise util.TimeoutException(
            "TimedOut waiting for reply. This is unusual.")
        raise tab_crash_exception.TabCrashException()

      res = json.loads(data)
      logging.debug('got [%s]', data)
      if 'method' in res:
        mname = res['method']
        dot_pos = mname.find('.')
        domain_name = mname[:dot_pos]
        if domain_name in self._domain_handlers:
          try:
            self._domain_handlers[domain_name][0](res)
          except Exception:
            import traceback
            traceback.print_exc()
        else:
          logging.debug('Unhandled inspector mesage: %s', data)
        continue

      if res['id'] != req['id']:
        logging.debug('Dropped reply: %s', json.dumps(res))
        continue
      return res

  def RegisterDomain(self,
      domain_name, notification_handler, will_close_handler):
    """Registers a given domain for handling notification methods.

    For example, given inspector_backend:
       def OnConsoleNotification(msg):
          if msg['method'] == 'Console.messageAdded':
             print msg['params']['message']
          return
       def OnConsoleClose(self):
          pass
       inspector_backend.RegisterDomain('Console',
                                        OnConsoleNotification, OnConsoleClose)
       """
    assert domain_name not in self._domain_handlers
    self._domain_handlers[domain_name] = (notification_handler,
                                          will_close_handler)

<commit_msg>Add a logging.debug for sending messages over devtools protocol.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import logging
import socket

from chrome_remote_control import tab_crash_exception
from chrome_remote_control import util
from chrome_remote_control import websocket

class InspectorException(Exception):
  pass

class InspectorBackend(object):
  def __init__(self, backend, descriptor):
    self._backend = backend
    self._descriptor = descriptor
    self._socket_url = descriptor['webSocketDebuggerUrl']
    self._socket = websocket.create_connection(
        descriptor['webSocketDebuggerUrl'])
    self._next_request_id = 0
    self._domain_handlers = {}
    self._cur_socket_timeout = 0

  def Close(self):
    for _, handlers in self._domain_handlers.items():
      _, will_close_handler = handlers
      will_close_handler()
    self._domain_handlers = {}
    self._socket.close()
    self._socket = None
    self._backend = None

  def DispatchNotifications(self, timeout=10):
    self._SetTimeout(timeout)
    try:
      data = self._socket.recv()
    except socket.error:
      if self._backend.DoesDebuggerUrlExist(self._socket_url):
        return
      raise tab_crash_exception.TabCrashException()

    res = json.loads(data)
    logging.debug('got [%s]', data)
    if 'method' not in res:
      return

    mname = res['method']
    dot_pos = mname.find('.')
    domain_name = mname[:dot_pos]
    if domain_name in self._domain_handlers:
      try:
        self._domain_handlers[domain_name][0](res)
      except Exception:
        import traceback
        traceback.print_exc()

  def SendAndIgnoreResponse(self, req):
    req['id'] = self._next_request_id
    self._next_request_id += 1
    data = json.dumps(req)
    self._socket.send(data)
    logging.debug('sent [%s]', data)

  def _SetTimeout(self, timeout):
    if self._cur_socket_timeout != timeout:
      self._socket.settimeout(timeout)
      self._cur_socket_timeout = timeout

  def SyncRequest(self, req, timeout=10):
    # TODO(nduca): Listen to the timeout argument
    # pylint: disable=W0613
    self._SetTimeout(timeout)
    self.SendAndIgnoreResponse(req)

    while True:
      try:
        data = self._socket.recv()
      except socket.error:
        if self._backend.DoesDebuggerUrlExist(self._socket_url):
          raise util.TimeoutException(
            "TimedOut waiting for reply. This is unusual.")
        raise tab_crash_exception.TabCrashException()

      res = json.loads(data)
      logging.debug('got [%s]', data)
      if 'method' in res:
        mname = res['method']
        dot_pos = mname.find('.')
        domain_name = mname[:dot_pos]
        if domain_name in self._domain_handlers:
          try:
            self._domain_handlers[domain_name][0](res)
          except Exception:
            import traceback
            traceback.print_exc()
        else:
          logging.debug('Unhandled inspector mesage: %s', data)
        continue

      if res['id'] != req['id']:
        logging.debug('Dropped reply: %s', json.dumps(res))
        continue
      return res

  def RegisterDomain(self,
      domain_name, notification_handler, will_close_handler):
    """Registers a given domain for handling notification methods.

    For example, given inspector_backend:
       def OnConsoleNotification(msg):
          if msg['method'] == 'Console.messageAdded':
             print msg['params']['message']
          return
       def OnConsoleClose(self):
          pass
       inspector_backend.RegisterDomain('Console',
                                        OnConsoleNotification, OnConsoleClose)
       """
    assert domain_name not in self._domain_handlers
    self._domain_handlers[domain_name] = (notification_handler,
                                          will_close_handler)

<|endoftext|>


<commit_before>#!/usr/bin/python
# archipelModuleHypervisorTest.py
#
# Copyright (C) 2010 Antoine Mercadal <antoine.mercadal@inframonde.eu>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# we need to import the package containing the class to surclass
import xmpp
from utils import *
import archipel
import uuid
import sqlite3
from apscheduler.scheduler import Scheduler

class TNActionScheduler:

    def __init__(self, entity, db_file):
        self.entity = entity
        self.scheduler = Scheduler()
        self.scheduler.start()

        self.database = sqlite3.connect(db_file, check_same_thread=False);
        self.database.execute("create table if not exists scheduler (entity_uuid text, job_uuid text, action text, year integer, month integer, day integer, hour integer, minute integer, second integer, comment text, params text)")
        self.database.commit()
        self.cursor = self.database.cursor()
        self.restore_jobs()

    def save_jobs(self, uid, action, year, month, day, hour, minute, second, comment, params=None):
        self.cursor.execute("INSERT INTO scheduler VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (self.entity.uuid, uid, action, year, month, day, hour, minute, second, comment, params,))
        self.database.commit()

    def restore_jobs(self):
        self.cursor.execute("SELECT * FROM scheduler WHERE entity_uuid=?", (self.entity.uuid,))
        for values in self.cursor:
            entity_uuid, job_uuid, action, year, month, day, hour, minute, second, comment, params = values
            str_date = "%s/%s/%s %s:%s:%s" % (year, month, day, hour, minute, second)
            self.scheduler.add_cron_job(self.__do_job, year=year, month=month, day=day, hour=hour, minute=minute, second=second, args=[action, job_uuid, str_date, comment])

    def process_iq(self, conn, iq):
        """
        this method is invoked when a ARCHIPEL_NS_VM_SCHEDULER IQ is received.

        it understands IQ of type:
            - jobs
            - schedule
            - unschedule

        @type conn: xmpp.Dispatcher
        @param conn: ths instance of the current connection that send the stanza
        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ
        """
        try:
            action = iq.getTag("query").getTag("archipel").getAttr("action")
            log.info("IQ RECEIVED: from: %s, type: %s, namespace: %s, action: %s" % (iq.getFrom(), iq.getType(), iq.getQueryNS(), action))
        except Exception as ex:
            reply = build_error_iq(self, ex, iq, ARCHIPEL_NS_ERROR_QUERY_NOT_WELL_FORMED)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed

        if action == "schedule":
            reply = self.iq_schedule(iq)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed
        elif action == "unschedule":
            reply = self.iq_unschedule(iq)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed
        elif action == "jobs":
            reply = self.iq_jobs(iq)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed

    def __do_job(self, action, uid, str_date, comment):
        if action == "create":
            self.entity.create()
        elif action == "shutdown":
            self.entity.shutdown()
        elif action == "destroy":
            self.entity.destroy()
        elif action == "suspend":
            self.entity.suspend()
        elif action == "resume":
            self.entity.resume()
        elif action == "migrate":
            pass

    def iq_schedule(self, iq):
        """
        Schedule a task.

        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ

        @rtype: xmpp.Protocol.Iq
        @return: a ready to send IQ containing the result of the action
        """
        try:
            reply = iq.buildReply("result")
            job = iq.getTag("query").getTag("archipel").getAttr("job")

            if not job in ("create", "shutdown", "destroy", "suspend", "resume", "reboot", "migrate"):
                raise Exception("job is not valid")

            year = iq.getTag("query").getTag("archipel").getAttr("year")
            month = iq.getTag("query").getTag("archipel").getAttr("month")
            day = iq.getTag("query").getTag("archipel").getAttr("day")
            hour = iq.getTag("query").getTag("archipel").getAttr("hour")
            minute = iq.getTag("query").getTag("archipel").getAttr("minute")
            second = iq.getTag("query").getTag("archipel").getAttr("second")
            comment = iq.getTag("query").getTag("archipel").getAttr("comment")
            uid = str(uuid.uuid1())

            str_date = "%s/%s/%s %s:%s:%s" % (year, month, day, hour, minute, second)
            self.scheduler.add_cron_job(self.__do_job, year=year, month=month, day=day, hour=hour, minute=minute, second=second, args=[job, uid, str_date, comment])
            self.save_jobs(uid, job, year, month, day, hour, minute, second, comment)

            self.entity.push_change("scheduler", "scheduled")
        except Exception as ex:
            reply = build_error_iq(self, ex, iq)
        return reply

    def iq_jobs(self, iq):
        """
        gets jobs

        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ

        @rtype: xmpp.Protocol.Iq
        @return: a ready to send IQ containing the result of the action
        """
        try:
            reply = iq.buildReply("result")
            nodes = [];
            for job in self.scheduler.jobs:
                job_node = xmpp.Node(tag="job", attrs={"action": str(job.args[0]), "uid": str(job.args[1]), "date": str(job.args[2]), "comment": job.args[3]})
                nodes.append(job_node)

            reply.setQueryPayload(nodes)
        except Exception as ex:
            reply = build_error_iq(self, ex, iq)
        return reply

    def iq_unschedule(self, iq):
        """
        gets jobs

        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ

        @rtype: xmpp.Protocol.Iq
        @return: a ready to send IQ containing the result of the action
        """
        try:
            reply = iq.buildReply("result")
            uid = iq.getTag("query").getTag("archipel").getAttr("uid")
            the_job = None
            for job in self.scheduler.jobs:
                if str(job.args[1]) == uid:
                    the_job = job;

            if not the_job:
                raise Exception("job with uid %s doesn't exists" % uid)

            self.scheduler.unschedule_job(the_job);
            self.entity.push_change("scheduler", "unscheduled")
        except Exception as ex:
            reply = build_error_iq(self, ex, iq)
        return reply

<commit_msg>push notification when job is executed<commit_after>#!/usr/bin/python
# archipelModuleHypervisorTest.py
#
# Copyright (C) 2010 Antoine Mercadal <antoine.mercadal@inframonde.eu>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# we need to import the package containing the class to surclass
import xmpp
from utils import *
import archipel
import uuid
import sqlite3
from apscheduler.scheduler import Scheduler

class TNActionScheduler:

    def __init__(self, entity, db_file):
        self.entity = entity
        self.scheduler = Scheduler()
        self.scheduler.start()

        self.database = sqlite3.connect(db_file, check_same_thread=False);
        self.database.execute("create table if not exists scheduler (entity_uuid text, job_uuid text, action text, year integer, month integer, day integer, hour integer, minute integer, second integer, comment text, params text)")
        self.database.commit()
        self.cursor = self.database.cursor()
        self.restore_jobs()

    def save_jobs(self, uid, action, year, month, day, hour, minute, second, comment, params=None):
        self.cursor.execute("INSERT INTO scheduler VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (self.entity.uuid, uid, action, year, month, day, hour, minute, second, comment, params,))
        self.database.commit()

    def restore_jobs(self):
        self.cursor.execute("SELECT * FROM scheduler WHERE entity_uuid=?", (self.entity.uuid,))
        for values in self.cursor:
            entity_uuid, job_uuid, action, year, month, day, hour, minute, second, comment, params = values
            str_date = "%s/%s/%s %s:%s:%s" % (year, month, day, hour, minute, second)
            self.scheduler.add_cron_job(self.__do_job, year=year, month=month, day=day, hour=hour, minute=minute, second=second, args=[action, job_uuid, str_date, comment])

    def process_iq(self, conn, iq):
        """
        this method is invoked when a ARCHIPEL_NS_VM_SCHEDULER IQ is received.

        it understands IQ of type:
            - jobs
            - schedule
            - unschedule

        @type conn: xmpp.Dispatcher
        @param conn: ths instance of the current connection that send the stanza
        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ
        """
        try:
            action = iq.getTag("query").getTag("archipel").getAttr("action")
            log.info("IQ RECEIVED: from: %s, type: %s, namespace: %s, action: %s" % (iq.getFrom(), iq.getType(), iq.getQueryNS(), action))
        except Exception as ex:
            reply = build_error_iq(self, ex, iq, ARCHIPEL_NS_ERROR_QUERY_NOT_WELL_FORMED)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed

        if action == "schedule":
            reply = self.iq_schedule(iq)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed
        elif action == "unschedule":
            reply = self.iq_unschedule(iq)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed
        elif action == "jobs":
            reply = self.iq_jobs(iq)
            conn.send(reply)
            raise xmpp.protocol.NodeProcessed

    def __do_job(self, action, uid, str_date, comment):
        if action == "create":
            self.entity.create()
        elif action == "shutdown":
            self.entity.shutdown()
        elif action == "destroy":
            self.entity.destroy()
        elif action == "suspend":
            self.entity.suspend()
        elif action == "resume":
            self.entity.resume()
        elif action == "migrate":
            pass
        self.cursor.execute("DELETE FROM scheduler WHERE job_uuid=?", (uid,))
        self.database.commit()
        self.entity.push_change("scheduler", "jobexecuted");

    def iq_schedule(self, iq):
        """
        Schedule a task.

        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ

        @rtype: xmpp.Protocol.Iq
        @return: a ready to send IQ containing the result of the action
        """
        try:
            reply = iq.buildReply("result")
            job = iq.getTag("query").getTag("archipel").getAttr("job")

            if not job in ("create", "shutdown", "destroy", "suspend", "resume", "reboot", "migrate"):
                raise Exception("job is not valid")

            year = iq.getTag("query").getTag("archipel").getAttr("year")
            month = iq.getTag("query").getTag("archipel").getAttr("month")
            day = iq.getTag("query").getTag("archipel").getAttr("day")
            hour = iq.getTag("query").getTag("archipel").getAttr("hour")
            minute = iq.getTag("query").getTag("archipel").getAttr("minute")
            second = iq.getTag("query").getTag("archipel").getAttr("second")
            comment = iq.getTag("query").getTag("archipel").getAttr("comment")
            uid = str(uuid.uuid1())

            str_date = "%s/%s/%s %s:%s:%s" % (year, month, day, hour, minute, second)
            self.scheduler.add_cron_job(self.__do_job, year=year, month=month, day=day, hour=hour, minute=minute, second=second, args=[job, uid, str_date, comment])
            self.save_jobs(uid, job, year, month, day, hour, minute, second, comment)

            self.entity.push_change("scheduler", "scheduled")
        except Exception as ex:
            reply = build_error_iq(self, ex, iq)
        return reply

    def iq_jobs(self, iq):
        """
        gets jobs

        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ

        @rtype: xmpp.Protocol.Iq
        @return: a ready to send IQ containing the result of the action
        """
        try:
            reply = iq.buildReply("result")
            nodes = [];
            for job in self.scheduler.jobs:
                job_node = xmpp.Node(tag="job", attrs={"action": str(job.args[0]), "uid": str(job.args[1]), "date": str(job.args[2]), "comment": job.args[3]})
                nodes.append(job_node)

            reply.setQueryPayload(nodes)
        except Exception as ex:
            reply = build_error_iq(self, ex, iq)
        return reply

    def iq_unschedule(self, iq):
        """
        gets jobs

        @type iq: xmpp.Protocol.Iq
        @param iq: the received IQ

        @rtype: xmpp.Protocol.Iq
        @return: a ready to send IQ containing the result of the action
        """
        try:
            reply = iq.buildReply("result")
            uid = iq.getTag("query").getTag("archipel").getAttr("uid")
            the_job = None
            for job in self.scheduler.jobs:
                if str(job.args[1]) == uid:
                    the_job = job;

            if not the_job:
                raise Exception("job with uid %s doesn't exists" % uid)

            self.scheduler.unschedule_job(the_job);
            self.entity.push_change("scheduler", "unscheduled")
        except Exception as ex:
            reply = build_error_iq(self, ex, iq)
        return reply

<|endoftext|>


<commit_before>import unittest

import mock
import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer.functions.activation import tanh
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

@testing.parameterize(*testing.product({
    'shape': [(3, 2), ()],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestTanh(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-.5, .5, self.shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-.1, .1, self.shape).astype(self.dtype)
        self.check_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_backward_options = {
                'dtype': numpy.float64, 'atol': 1e-4, 'rtol': 1e-3}

    def check_forward(self, x_data, use_cudnn='always'):
        x = chainer.Variable(x_data)
        with chainer.using_config('use_cudnn', use_cudnn):
            y = functions.tanh(x)
        self.assertEqual(y.data.dtype, self.dtype)
        y_expect = functions.tanh(chainer.Variable(self.x))
        testing.assert_allclose(y_expect.data, y.data)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), 'always')

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu_non_contiguous(self):
        self.check_forward(cuda.cupy.asfortranarray(cuda.to_gpu(self.x)),
                           'always')

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu_no_cudnn(self):
        self.check_forward(cuda.to_gpu(self.x), 'never')

    def check_backward(self, x_data, gy_data, use_cudnn='always'):
        with chainer.using_config('use_cudnn', use_cudnn):
            gradient_check.check_backward(
                functions.tanh, x_data, gy_data,
                **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu_non_contiguous(self):
        self.check_backward(cuda.cupy.asfortranarray(cuda.to_gpu(self.x)),
                            cuda.cupy.asfortranarray(cuda.to_gpu(self.gy)))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu_no_cudnn(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy), 'never')

@testing.parameterize(*testing.product({
    'use_cudnn': ['always', 'auto', 'never'],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
@attr.cudnn
class TestTanhCudnnCall(unittest.TestCase):

    def setUp(self):
        self.x = cuda.cupy.random.uniform(-1, 1, (2, 3)).astype(self.dtype)
        self.gy = cuda.cupy.random.uniform(-1, 1, (2, 3)).astype(self.dtype)
        with chainer.using_config('use_cudnn', self.use_cudnn):
            self.expect = chainer.should_use_cudnn('==always')

    def forward(self):
        x = chainer.Variable(self.x)
        return functions.tanh(x)

    def test_call_cudnn_forward(self):
        with chainer.using_config('use_cudnn', self.use_cudnn):
            default_func = cuda.cupy.cudnn.activation_forward
            with mock.patch('cupy.cudnn.activation_forward') as func:
                func.side_effect = default_func
                self.forward()
                self.assertEqual(func.called, self.expect)

    def test_call_cudnn_backward(self):
        with chainer.using_config('use_cudnn', self.use_cudnn):
            y = self.forward()
            y.grad = self.gy
            default_func = cuda.cupy.cudnn.activation_backward
            with mock.patch('cupy.cudnn.activation_backward') as func:
                func.side_effect = default_func
                y.backward()
                self.assertEqual(func.called, self.expect)

@testing.parameterize(*testing.product({
    'shape': [(3, 2), ()],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestTanhGrad(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-.5, .5, self.shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.ggx = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)

    def check_backward(self, x_data, y_data, gy_data, ggx_data):
        def f(y, gy):
            return tanh.TanhGrad(x_data).apply((y, gy))[0]

        gradient_check.check_backward(
            f, (y_data, gy_data), ggx_data, dtype='d', atol=1e-4, rtol=1e-4)

    def test_backward_cpu(self):
        y = numpy.array(numpy.tanh(self.x))
        self.check_backward(self.x, y, self.gy, self.ggx)

testing.run_module(__name__, __file__)
<commit_msg>Add test for double backward<commit_after>import unittest

import mock
import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer.functions.activation import tanh
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

@testing.parameterize(*testing.product({
    'shape': [(3, 2), ()],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestTanh(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-.5, .5, self.shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-.5, .5, self.shape).astype(self.dtype)
        self.ggx = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.check_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_backward_options = {
                'dtype': numpy.float64, 'atol': 1e-4, 'rtol': 1e-3}

    def check_forward(self, x_data, use_cudnn='always'):
        x = chainer.Variable(x_data)
        with chainer.using_config('use_cudnn', use_cudnn):
            y = functions.tanh(x)
        self.assertEqual(y.data.dtype, self.dtype)
        y_expect = functions.tanh(chainer.Variable(self.x))
        testing.assert_allclose(y_expect.data, y.data)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), 'always')

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu_non_contiguous(self):
        self.check_forward(cuda.cupy.asfortranarray(cuda.to_gpu(self.x)),
                           'always')

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu_no_cudnn(self):
        self.check_forward(cuda.to_gpu(self.x), 'never')

    def check_backward(self, x_data, gy_data, use_cudnn='always'):
        with chainer.using_config('use_cudnn', use_cudnn):
            gradient_check.check_backward(
                functions.tanh, x_data, gy_data,
                **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu_non_contiguous(self):
        self.check_backward(cuda.cupy.asfortranarray(cuda.to_gpu(self.x)),
                            cuda.cupy.asfortranarray(cuda.to_gpu(self.gy)))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu_no_cudnn(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy), 'never')

    def check_double_backward(self, x_data, gy_data, ggx_data):
        gradient_check.check_double_backward(
            chainer.functions.tanh,  x_data, gy_data, ggx_data, dtype='d')

    @condition.retry(3)
    def test_double_backward_cpu(self):
        self.check_double_backward(self.x, self.gy, self.ggx)

    @attr.gpu
    @condition.retry(3)
    def test_double_backward_gpu(self):
        self.check_double_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), cuda.to_gpu(self.ggx))

@testing.parameterize(*testing.product({
    'use_cudnn': ['always', 'auto', 'never'],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
@attr.cudnn
class TestTanhCudnnCall(unittest.TestCase):

    def setUp(self):
        self.x = cuda.cupy.random.uniform(-1, 1, (2, 3)).astype(self.dtype)
        self.gy = cuda.cupy.random.uniform(-1, 1, (2, 3)).astype(self.dtype)
        with chainer.using_config('use_cudnn', self.use_cudnn):
            self.expect = chainer.should_use_cudnn('==always')

    def forward(self):
        x = chainer.Variable(self.x)
        return functions.tanh(x)

    def test_call_cudnn_forward(self):
        with chainer.using_config('use_cudnn', self.use_cudnn):
            default_func = cuda.cupy.cudnn.activation_forward
            with mock.patch('cupy.cudnn.activation_forward') as func:
                func.side_effect = default_func
                self.forward()
                self.assertEqual(func.called, self.expect)

    def test_call_cudnn_backward(self):
        with chainer.using_config('use_cudnn', self.use_cudnn):
            y = self.forward()
            y.grad = self.gy
            default_func = cuda.cupy.cudnn.activation_backward
            with mock.patch('cupy.cudnn.activation_backward') as func:
                func.side_effect = default_func
                y.backward()
                self.assertEqual(func.called, self.expect)

@testing.parameterize(*testing.product({
    'shape': [(3, 2), ()],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestTanhGrad(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-.5, .5, self.shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.ggx = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)

    def check_backward(self, x_data, y_data, gy_data, ggx_data):
        def f(y, gy):
            return tanh.TanhGrad(x_data).apply((y, gy))[0]

        gradient_check.check_backward(
            f, (y_data, gy_data), ggx_data, dtype='d', atol=1e-4, rtol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        y = numpy.array(numpy.tanh(self.x))
        self.check_backward(self.x, y, self.gy, self.ggx)

    @attr.gpu
    @condition.retry(3)
    def test_backward_cpu(self):
        y = numpy.array(numpy.tanh(self.x))
        self.check_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(y), cuda.to_gpu(self.gy),
            cuda.to_gpu(self.ggx))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>"""
Run a local Storm topology.

Note: If you have "org.apache.storm" in your uberjar-exclusions list in your
project.clj, this will fail. Temporarily remove it to use `sparse run`.
"""

from __future__ import absolute_import, print_function

from argparse import RawDescriptionHelpFormatter
from tempfile import NamedTemporaryFile

from fabric.api import local
from ruamel import yaml

from ..util import (get_env_config, get_topology_definition,
                    get_topology_from_file, local_storm_version,
                    storm_lib_version)
from .common import (add_ackers, add_debug, add_environment, add_name,
                     add_options, add_workers, resolve_options)
from .jar import jar_for_deploy

def run_local_topology(name=None, env_name=None, time=0, options=None):
    """Run a topology locally using Flux and `storm jar`."""
    name, topology_file = get_topology_definition(name)
    env_name, env_config = get_env_config(env_name)
    topology_class = get_topology_from_file(topology_file)

    storm_options = resolve_options(options, env_config, topology_class, name)
    if storm_options['topology.acker.executors'] != 0:
        storm_options['topology.acker.executors'] = 1
    storm_options['topology.workers'] = 1

    # Check Storm version is the same
    local_version = local_storm_version()
    project_version = storm_lib_version()
    if local_version != project_version:
        raise ValueError('Local Storm version, {}, is not the same as the '
                         'version in your project.clj, {}. The versions must '
                         'match.'.format(local_version, project_version))

    # Prepare a JAR that has Storm dependencies packaged
    topology_jar = jar_for_deploy(simple_jar=False)

    if time <= 0:
        time = 9223372036854775807  # Max long value in Java

    # Write YAML file
    with NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as yaml_file:
        topology_flux_dict = topology_class.to_flux_dict(name)
        topology_flux_dict['config'] = storm_options
        yaml.safe_dump(topology_flux_dict, yaml_file)
        cmd = ('storm jar {jar} org.apache.storm.flux.Flux --local --no-splash '
               '--sleep {time} {yaml}'.format(jar=topology_jar,
                                              time=time,
                                              yaml=yaml_file.name))
        local(cmd)

def subparser_hook(subparsers):
    """ Hook to add subparser for this command. """
    subparser = subparsers.add_parser('run',
                                      description=__doc__,
                                      help=main.__doc__,
                                      formatter_class=RawDescriptionHelpFormatter)
    subparser.set_defaults(func=main)
    add_ackers(subparser)
    add_debug(subparser)
    add_environment(subparser)
    add_name(subparser)
    add_options(subparser)
    subparser.add_argument('-t', '--time',
                           default=0,
                           type=int,
                           help='Time (in seconds) to keep local cluster '
                                'running. If time <= 0, run indefinitely. '
                                '(default: %(default)s)')
    add_workers(subparser)

def main(args):
    """Run the local topology with the given arguments"""
    run_local_topology(name=args.name, time=args.time, options=args.options)
<commit_msg>Fix issue where sparse run wasn't using environment when set<commit_after>"""
Run a local Storm topology.

Note: If you have "org.apache.storm" in your uberjar-exclusions list in your
project.clj, this will fail. Temporarily remove it to use `sparse run`.
"""

from __future__ import absolute_import, print_function

from argparse import RawDescriptionHelpFormatter
from tempfile import NamedTemporaryFile

from fabric.api import local
from ruamel import yaml

from ..util import (get_env_config, get_topology_definition,
                    get_topology_from_file, local_storm_version,
                    storm_lib_version)
from .common import (add_ackers, add_debug, add_environment, add_name,
                     add_options, add_workers, resolve_options)
from .jar import jar_for_deploy

def run_local_topology(name=None, env_name=None, time=0, options=None):
    """Run a topology locally using Flux and `storm jar`."""
    name, topology_file = get_topology_definition(name)
    env_name, env_config = get_env_config(env_name)
    topology_class = get_topology_from_file(topology_file)

    storm_options = resolve_options(options, env_config, topology_class, name)
    if storm_options['topology.acker.executors'] != 0:
        storm_options['topology.acker.executors'] = 1
    storm_options['topology.workers'] = 1

    # Check Storm version is the same
    local_version = local_storm_version()
    project_version = storm_lib_version()
    if local_version != project_version:
        raise ValueError('Local Storm version, {}, is not the same as the '
                         'version in your project.clj, {}. The versions must '
                         'match.'.format(local_version, project_version))

    # Prepare a JAR that has Storm dependencies packaged
    topology_jar = jar_for_deploy(simple_jar=False)

    if time <= 0:
        time = 9223372036854775807  # Max long value in Java

    # Write YAML file
    with NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as yaml_file:
        topology_flux_dict = topology_class.to_flux_dict(name)
        topology_flux_dict['config'] = storm_options
        yaml.safe_dump(topology_flux_dict, yaml_file)
        cmd = ('storm jar {jar} org.apache.storm.flux.Flux --local --no-splash '
               '--sleep {time} {yaml}'.format(jar=topology_jar,
                                              time=time,
                                              yaml=yaml_file.name))
        local(cmd)

def subparser_hook(subparsers):
    """ Hook to add subparser for this command. """
    subparser = subparsers.add_parser('run',
                                      description=__doc__,
                                      help=main.__doc__,
                                      formatter_class=RawDescriptionHelpFormatter)
    subparser.set_defaults(func=main)
    add_ackers(subparser)
    add_debug(subparser)
    add_environment(subparser)
    add_name(subparser)
    add_options(subparser)
    subparser.add_argument('-t', '--time',
                           default=0,
                           type=int,
                           help='Time (in seconds) to keep local cluster '
                                'running. If time <= 0, run indefinitely. '
                                '(default: %(default)s)')
    add_workers(subparser)

def main(args):
    """Run the local topology with the given arguments"""
    run_local_topology(name=args.name, time=args.time, options=args.options,
                       env_name=args.environment)
<|endoftext|>


<commit_before>from django.http import HttpResponse
from django.template import RequestContext
from django.shortcuts import render_to_response
from django.db.models import Avg, Count

import simplejson

from mbta.budgetcalc.models import Category, Optiongroup, Option, Submission

def index(request):
    """
    Get everything or save submission.
    """

    if request.method == 'GET':
        options = Option.objects.select_related().all()
        return render_to_response('budgetcalc/index.html', locals(), context_instance=RequestContext(request))
    elif request.method == 'POST':

        email = request.POST.get('email').lower()
        budget = float(request.POST.get('filled'))

        if Submission.objects.filter(email__icontains=email).exists():
            # update existing
            submission = Submission.objects.get(email=email)
            submission.budget = budget
            submission.options.clear()
        else:
            # create new
            submission = Submission(
                email=email,
                budget=budget
            )

        submission.save()

        options = simplejson.loads(request.POST['options'])
        for option in options:
            submission.options.add(Option.objects.get(pk=option))

        # some stats
        from django.db.models import Avg
        stats = Submission.objects.aggregate(Avg('budget'), Count('budget'))
        stats['options'] = dict()
        options = Option.objects.annotate(num_submissions=Count('selected_options'))
        for option in options:
            stats['options'][option.id] = option.num_submissions

        return HttpResponse(
                simplejson.dumps(stats),
                status=201,
                mimetype='application/json'
            )

<commit_msg>removes obsolete module import<commit_after>from django.http import HttpResponse
from django.template import RequestContext
from django.shortcuts import render_to_response
from django.db.models import Avg, Count

import simplejson

from mbta.budgetcalc.models import Category, Optiongroup, Option, Submission

def index(request):
    """
    Get everything or save submission.
    """

    if request.method == 'GET':
        options = Option.objects.select_related().all()
        return render_to_response('budgetcalc/index.html', locals(), context_instance=RequestContext(request))
    elif request.method == 'POST':

        email = request.POST.get('email').lower()
        budget = float(request.POST.get('filled'))

        if Submission.objects.filter(email__icontains=email).exists():
            # update existing
            submission = Submission.objects.get(email=email)
            submission.budget = budget
            submission.options.clear()
        else:
            # create new
            submission = Submission(
                email=email,
                budget=budget
            )

        submission.save()

        options = simplejson.loads(request.POST['options'])
        for option in options:
            submission.options.add(Option.objects.get(pk=option))

        # some stats
        stats = Submission.objects.aggregate(Avg('budget'), Count('budget'))
        stats['options'] = dict()
        options = Option.objects.annotate(num_submissions=Count('selected_options'))
        for option in options:
            stats['options'][option.id] = option.num_submissions

        return HttpResponse(
                simplejson.dumps(stats),
                status=201,
                mimetype='application/json'
            )

<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import os
import os.path
import xml.dom.minidom

for param in os.environ.keys():
    print "%20s %s" % (param,os.environ[param])

if os.environ.get("TRAVIS_SECURE_ENV_VARS") == "false":
  print "No secure env vars available, skipping deployment"
  sys.exit()

if os.environ.get("TRAVIS_BRANCH") != "master":
  print "[" + os.environ.get("TRAVIS_BRANCH") + "] is not the master branch, skipping deployment"
  sys.exit()

if os.environ.get("TRAVIS_PULL_REQUEST") == "true":
  print "This is a pull request, skipping deployment"
  sys.exit()

cwd = os.getcwd()
print cwd
m2 = xml.dom.minidom.Document()
settings = m2.createElement("settings")
m2.appendChild(settings)

serversNodes = settings.getElementsByTagName("servers")
if not serversNodes:
  serversNode = m2.createElement("servers")
  settings.appendChild(serversNode)
else:
  serversNode = serversNodes[0]

sonatypeServerNode = m2.createElement("server")
sonatypeServerId = m2.createElement("id")
sonatypeServerUser = m2.createElement("username")
sonatypeServerPass = m2.createElement("password")

idNode = m2.createTextNode("sonatype-nexus-snapshots")
userNode = m2.createTextNode(os.environ["SONATYPE_USERNAME"])
passNode = m2.createTextNode(os.environ["SONATYPE_PASSWORD"])

sonatypeServerId.appendChild(idNode)
sonatypeServerUser.appendChild(userNode)
sonatypeServerPass.appendChild(passNode)

sonatypeServerNode.appendChild(sonatypeServerId)
sonatypeServerNode.appendChild(sonatypeServerUser)
sonatypeServerNode.appendChild(sonatypeServerPass)

serversNode.appendChild(sonatypeServerNode)

m2Str = m2.toxml()
f = open(cwd + '/settings.xml', 'w')
f.write(m2Str)
f.close()
<commit_msg>Removed debug output to deploy script<commit_after>#!/usr/bin/env python
import sys
import os
import os.path
import xml.dom.minidom

if os.environ.get("TRAVIS_SECURE_ENV_VARS") == "false":
  print "No secure env vars available, skipping deployment"
  sys.exit()

if os.environ.get("TRAVIS_PULL_REQUEST") == "true":
  print "This is a pull request, skipping deployment"
  sys.exit()

cwd = os.getcwd()
print cwd
m2 = xml.dom.minidom.Document()
settings = m2.createElement("settings")
m2.appendChild(settings)

serversNodes = settings.getElementsByTagName("servers")
if not serversNodes:
  serversNode = m2.createElement("servers")
  settings.appendChild(serversNode)
else:
  serversNode = serversNodes[0]

sonatypeServerNode = m2.createElement("server")
sonatypeServerId = m2.createElement("id")
sonatypeServerUser = m2.createElement("username")
sonatypeServerPass = m2.createElement("password")

idNode = m2.createTextNode("sonatype-nexus-snapshots")
userNode = m2.createTextNode(os.environ["SONATYPE_USERNAME"])
passNode = m2.createTextNode(os.environ["SONATYPE_PASSWORD"])

sonatypeServerId.appendChild(idNode)
sonatypeServerUser.appendChild(userNode)
sonatypeServerPass.appendChild(passNode)

sonatypeServerNode.appendChild(sonatypeServerId)
sonatypeServerNode.appendChild(sonatypeServerUser)
sonatypeServerNode.appendChild(sonatypeServerPass)

serversNode.appendChild(sonatypeServerNode)

m2Str = m2.toxml()
f = open(cwd + '/settings.xml', 'w')
f.write(m2Str)
f.close()
<|endoftext|>


<commit_before># The all important loader
from strictyaml.parser import load

# Validators
from strictyaml.validators import Optional
from strictyaml.validators import Validator
from strictyaml.validators import OrValidator
from strictyaml.validators import Any
from strictyaml.validators import Scalar
from strictyaml.validators import Enum
from strictyaml.validators import Str
from strictyaml.validators import Int
from strictyaml.validators import Bool
from strictyaml.validators import Float
from strictyaml.validators import Decimal
from strictyaml.validators import Map
from strictyaml.validators import MapPattern
from strictyaml.validators import Seq
from strictyaml.validators import UniqueSeq
from strictyaml.validators import FixedSeq
from strictyaml.validators import Datetime
from strictyaml.validators import EmptyNone
from strictyaml.validators import EmptyDict
from strictyaml.validators import EmptyList
from strictyaml.validators import CommaSeparated

# Base exception from ruamel.yaml (all exceptions inherit from this)
from ruamel.yaml import YAMLError

# Exceptions
from strictyaml.exceptions import StrictYAMLError
from strictyaml.exceptions import YAMLValidationError

# Disallowed token exceptions
from strictyaml.exceptions import DisallowedToken

from strictyaml.exceptions import TagTokenDisallowed
from strictyaml.exceptions import FlowMappingDisallowed
from strictyaml.exceptions import AnchorTokenDisallowed
from strictyaml.exceptions import DuplicateKeysDisallowed
<commit_msg>FEATURE : YAML object importable directly from module.<commit_after># The all important loader
from strictyaml.parser import load

# YAML object
from strictyaml.representation import YAML

# Validators
from strictyaml.validators import Optional
from strictyaml.validators import Validator
from strictyaml.validators import OrValidator
from strictyaml.validators import Any
from strictyaml.validators import Scalar
from strictyaml.validators import Enum
from strictyaml.validators import Str
from strictyaml.validators import Int
from strictyaml.validators import Bool
from strictyaml.validators import Float
from strictyaml.validators import Decimal
from strictyaml.validators import Map
from strictyaml.validators import MapPattern
from strictyaml.validators import Seq
from strictyaml.validators import UniqueSeq
from strictyaml.validators import FixedSeq
from strictyaml.validators import Datetime
from strictyaml.validators import EmptyNone
from strictyaml.validators import EmptyDict
from strictyaml.validators import EmptyList
from strictyaml.validators import CommaSeparated

# Base exception from ruamel.yaml (all exceptions inherit from this)
from ruamel.yaml import YAMLError

# Exceptions
from strictyaml.exceptions import StrictYAMLError
from strictyaml.exceptions import YAMLValidationError

# Disallowed token exceptions
from strictyaml.exceptions import DisallowedToken

from strictyaml.exceptions import TagTokenDisallowed
from strictyaml.exceptions import FlowMappingDisallowed
from strictyaml.exceptions import AnchorTokenDisallowed
from strictyaml.exceptions import DuplicateKeysDisallowed
<|endoftext|>