<commit_before>"""
This module contains controller application to manage
a set of OpenFlow switches
"""
from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_3 as ofp
from ryu.ofproto import ofproto_v1_3_parser as parser

from fabric.network import Network
import fabric.packet as pkt
import fabric.flows as fl

IDLE_TIMEOUT = 10  #: Timeout for dynamic flows
DEF_PRI = 0
LOW_PRI = 1
HIGH_PRI = 2

class NetworkManager(app_manager.RyuApp):
    """
    Core class of this package, NetworkManager is a top-level Ryu application
    def features.reply():
        pass
    def port_status():
        pass

    """

    OFP_VERSIONS = [ofp.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(NetworkManager, self).__init__(*args, **kwargs)
        self.net = Network()  #: Init the Network instance

    @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
    def _handle_switch_features(self, ev):
        """
        Handle new switches connecting to the network
        """
        pass

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _handle_packet_in(self, ev):

        def packet_in(ev):
            p=pkt.parse(ev.msg.data)
            if p["dl_type"]

            ....PASS

        def reply_to_arp(dp):
            pass

        def send_lldp(dp):
            pass

        Handle unknown flow seen on the switches

        pass

<commit_msg>Update app.py<commit_after>"""
This module contains controller application to manage
a set of OpenFlow switches
"""
from ryu.base import app_manager
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.ofproto import ofproto_v1_3 as ofp
from ryu.ofproto import ofproto_v1_3_parser as parser

from fabric.network import Network
import fabric.packet as pkt
import fabric.flows as fl

IDLE_TIMEOUT = 10  #: Timeout for dynamic flows
DEF_PRI = 0
LOW_PRI = 1
HIGH_PRI = 2

class NetworkManager(app_manager.RyuApp):
    """
    Core class of this package, NetworkManager is a top-level Ryu application
    """

    def features.reply():
        pass

    def port_status():
        pass

    OFP_VERSIONS = [ofp.OFP_VERSION]

    def __init__(self, *args, **kwargs):
        super(NetworkManager, self).__init__(*args, **kwargs)
        self.net = Network()  #: Init the Network instance

    @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER)
    def _handle_switch_features(self, ev):
        """
        Handle new switches connecting to the network
        """
        pass

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def _handle_packet_in(self, ev):

        def packet_in(ev):
            p=pkt.parse(ev.msg.data)
            if p["dl_type"]

            ....PASS

        def reply_to_arp(dp):
            pass

        def send_lldp(dp):
            pass

        Handle unknown flow seen on the switches

        pass

<|endoftext|>


<commit_before>"""
Types used on a per-build basis.
"""

__author__ = "Eldar Abusalimov"
__date__ = "2012-11-09"

__all__ = ["Context"]

from collections import defaultdict
from collections import deque
from collections import MutableSet
from contextlib import contextmanager
from functools import partial
from itertools import chain
from itertools import izip
from itertools import izip_longest
from itertools import product

from core import *
from constraints import *
from expr import *
from util import singleton
from util import NotifyingMixin
import pdag

import logs as log

class Context(object):
    """docstring for Context"""

    def __init__(self):
        super(Context, self).__init__()
        self._modules = {}
        self._job_queue = deque()
        self._reent_locked = False

    def freeze(self):
        if hasattr(self, '_pdag'):
            raise RuntimeError("'freeze' must be called only once.")

        # def iter_atoms():
        #     for module_domain in self._module.itervalues():

        self._pdag = pdag.Pdag(*iter_atoms())

    def post(self, fxn):
        with self.reent_lock(): # to flush the queue on block exit
            self._job_queue.append(fxn)

    @contextmanager
    def reent_lock(self):
        was_locked = self._reent_locked
        self._reent_locked = True

        try:
            yield
        finally:
            if not was_locked:
                self._job_queue_flush()
            self._reent_locked = was_locked

    def _job_queue_flush(self):
        queue = self._job_queue

        while queue:
            fxn = queue.popleft()
            fxn()

    def consider(self, module, option=None, value=Ellipsis):
        domain = self.domain_for(module)
        if option is not None:
            domain.consider_option(option, value)

    def register(self, instance):
        self.domain_for(instance._module).register(instance)

    def domain_for(self, module, option=None):
        try:
            domain = self._modules[module]
        except KeyError:
            with self.reent_lock():
                domain = self._modules[module] = ModuleDomain(self, module)

        if option is not None:
            domain = domain.domain_for(option)

        return domain

    def atom_for(self, module, option=None, value=Ellipsis, negated=False):
        cache = self._atom_cache
        cache_key = module, option, value, negated

        try:
            return cache[cache_key]
        except KeyError:
            pass

        if negated:
            ret = pdag.Not(self.atom_for(module, option, value, negated=False))
        else:
            domain = self.domain_for(module, option)
            ret = domain.atom if option is None else domain.atom_for(value)

        cache[cache_key] = ret

        return ret

    def pnode_from(self, expr):
        pass

class DomainBase(object):
    """docstring for DomainBase"""

    context = property(attrgetter('_context'))

    def __init__(self, context):
        super(DomainBase, self).__init__()
        self._context = context

class ModuleDomain(DomainBase):
    """docstring for ModuleDomain"""

    module = property(attrgetter('_module'))
    atom = property(attrgetter('_atom'))

    def __init__(self, context, module):
        super(ModuleDomain, self).__init__(context)

        self._module = module
        self._atom = module._atom_type()

        self._instances = {} # { optuple : InstanceDomain }
        self._options = module._options._make(OptionDomain(option)
                                              for option in module._options)

        self._instantiate_product(self._options)

    def init_pnode(self):
        presence_pnode = pdag.EqGroup(self._atom,
            *(option_domain.pnode for option_domain in self._options))
        instances_pnode = pdag.And(
            *(instance.pnode
              for instance_set in self._instances.itervalues()
              for instance in instance_set))

    def _instantiate_product(self, iterables):
        make_optuple = self._options._make
        instances = self._instances

        for new_tuple in product(*iterables):
            new_optuple = make_optuple(new_tuple)

            assert new_optuple not in instances
            instances[new_optuple] = InstanceDomain(self._context, new_optuple)

    def consider_option(self, option, value):
        domain_to_extend = getattr(self._options, option)
        if value in domain_to_extend:
            return

        log.debug('mybuild: extending %r with %r', domain_to_extend, value)
        domain_to_extend.add(value)

        self._instantiate_product(
            option_domain if option_domain is not domain_to_extend else (value,)
            for option_domain in self._options)

    def domain_for(self, option):
        return getattr(self._options, option)

@Module.register_attr('_atom_type')
class ModuleAtom(pdag.Atom):
    __slots__ = ()

    def __str__(self):
        return self._module_name

    @classmethod
    def _new_type(cls, module_type, *ignored):
        return type('ModuleAtom_M%s' % (module_type._module_name,),
                    (cls, module_type),
                    dict(__slots__=()))

class NotifyingSet(MutableSet, NotifyingMixin):
    """Set with notification support. Backed by a dictionary."""

    def __init__(self, values):
        super(NotifyingSet, self).__init__()
        self._dict = {}

        self |= values

    def _create_value_for(self, key):
        pass

    def add(self, value):
        if value in self:
            return
        self._dict[value] = self._create_value_for(value)

        self._notify(value)

    def discard(self, value):
        if value not in self:
            return
        raise NotImplementedError

    def __iter__(self):
        return iter(self._dict)
    def __len__(self):
        return len(self._dict)
    def __contains__(self, value):
        return value in self._dict

    def __str__(self):
        return '<%s: %s>' % (type(self).__name__, self._dict.keys())

class OptionDomain(NotifyingSet):
    """docstring for OptionDomain"""

    def __init__(self, option):
        self._option = option
        self.pnode = pdag.AtMostOne()

        super(OptionDomain, self).__init__(option._values)

    def atom_for(self, value):
        if value not in self:
            self.add(value)
        return self._dict[value]

    def _create_value_for(self, value):
        atom = self._option._atom_type(value)
        return self.pnode._new_operand(atom)

@Option.register_attr('_atom_type')
class OptionValueAtom(pdag.Atom):
    __slots__ = '_value'

    def __init__(self, value):
        super(OptionValueAtom, self).__init__()
        self._value = value

    def __str__(self):
        return '(%s.%s==%s)' % (self._module_name,
                                self._option_name, self._value)

    @classmethod
    def _new_type(cls, option_type):
        return type('OptionAtom_M%s_O%s' % (option_type._module_name,
                                            option_type._option_name),
                    (cls, option_type),
                    dict(__slots__=()))

@Module.register_attr('_instance_type')
class Instance(Module.Type):
    """docstring for Instance"""

    class _InstanceProxy(object):
        """docstring for _InstanceProxy"""
        __slots__ = '_owner_instance', '_optuple'

        def __init__(self, owner_instance, optuple):
            super(Instance._InstanceProxy, self).__init__()
            self._owner_instance = owner_instance
            self._optuple = optuple

        def __nonzero__(self):
            return self._owner_instance._decide(self._optuple)

        def __getattr__(self, attr):
            return self._owner_instance._decide_option(self._optuple, attr)

    def __init__(self, domain, optuple, constraints):
        """Private constructor. Use '_post_new' instead."""
        super(Instance, self).__init__()

        self._domain = domain
        self._optuple = optuple
        self._constraints = constraints

        with log.debug("mybuild: new %r", self):
            try:
                self._init_fxn(*optuple)
            except InstanceError as e:
                log.debug("mybuild: unviable %r: %s", self, e)
                raise e
            else:
                log.debug("mybuild: succeeded %r", self)

    @classmethod
    def _post_new(cls, domain, optuple, _constraints=None):
        if _constraints is None:
            _constraints = Constraints()
            # _constraints.constrain_mslice(optuple)

        def new():
            try:
                instance = cls(domain, optuple, _constraints)
            except InstanceError:
                pass
            else:
                domain.register(instance)

        domain.post(new)

    def ask(self, mslice):
        optuple = mslice._to_optuple()
        exprify_eval(optuple, self._domain.consider)
        return self._InstanceProxy(self, optuple)

    @singleton
    class _build_visitor(ExprVisitor):

        def visit(self, expr, constraints):
            expr = exprify(expr)
            # log.debug('mybuild: visit [%r] %r with %r',
            #           type(expr), expr, constraints)
            return ExprVisitor.visit(self,
                exprify_eval(expr, constraints.check), constraints)

        def visit_bool(self, expr, constraints):
            return (constraints,) if expr else ()

        def visit_Atom(self, expr, constraints, negated=False):
            try:
                expr.eval(constraints.constrain, negated=negated)
            except ConstraintError:
                return self.visit_bool(False, constraints)
            else:
                return self.visit_bool(True, constraints)

        def visit_Not(self, expr, constraints):
            return self.visit_Atom(expr.atom, constraints, negated=True)

        def visit_Or(self, expr, constraints):
            # disjuncion of disjunctions: (C1|C2) | ... | (CK|...|CN)
            # nothing special to do here, expand parens by flattening the list
            return tuple(chain.from_iterable(
                self.visit(e, constraints.new_branch()) for e in expr.operands))

        def visit_And(self, expr, constraints):
            # Important assumption:
            # 'expr.operands' first yields atomic expressions (atoms and
            # negations), and then compounds (disjunctions in this case).
            # This allows us to defer branching the 'constraints' as much as
            # possible, until it becomes really necessary.

            # conjuction of disjunctions: (C1|C2) & ... & (CK|...|CN)
            cj_of_djs = (self.visit(e, constraints) for e in expr.operands)

            # expand parens by merging constraint dicts to get the resulting
            # disjunction: C1&...&CK | C1&...&CN | C2&...&CK | C2&...&CN | ...
            def iter_multiply(djs):
                def uniquify_filter(djs):
                    """
                    Removes duplicates, filters out the parent constraints,
                    and yields non-empty iterables.
                    """
                    for dj in djs:
                        if not dj:
                            raise ConstraintError
                        if len(dj) == 1:
                            if dj[0] is not constraints:
                                yield dj
                        else:
                            constraints_by_id = dict((id(c), c) for c in dj)
                            del constraints_by_id[constraints]
                            if constraints_by_id:
                                yield constraints_by_id.itervalues()

                try:
                    djs_list = sorted(uniquify_filter(djs), key=len)
                except ConstraintError:
                    return

                if not djs_list:
                    yield constraints
                    return

                for new_cj in product(*djs_list):
                    try:
                        yield constraints.merge_children(new_cj)
                    except ConstraintError:
                        pass

            return tuple(iter_multiply(cj_of_djs))

    def _log_build_choices(self, name, choices):
        length = len(choices)
        log.debug('mybuild: got %d %s choice%s: %r',
            length, name, 's' if length != 1 else '', choices)

    def constrain(self, expr):
        expr = exprify_eval(expr, self._domain.consider)

        with log.debug('mybuild: constrain %r', expr):
            choices = self._build_visitor.visit(expr, self._constraints)
            self._log_build_choices('constrain', choices)

            self._constraints = self._take_one_spawn_rest(choices)

    def _decide(self, expr):
        expr = exprify(expr)

        with log.debug('mybuild: deciding bool(%r)', expr):
            visit = self._build_visitor.visit

            yes_choices = visit(expr, self._constraints.new_branch())
            self._log_build_choices('"yes"', yes_choices)

            no_choices = visit(~expr, self._constraints.new_branch())
            self._log_build_choices('"no"', no_choices)

            ret = bool(yes_choices)
            choices = no_choices, yes_choices

            self._constraints = self._take_one_spawn_rest(choices[ret])
            self._spawn_all(choices[not ret])

            log.debug('mybuild: return %s', ret)
            return ret

    def _decide_option(self, optuple, option):
        module = optuple._module

        with log.debug('mybuild: deciding %r.%s', optuple, option):
            if not hasattr(optuple, option):
                raise AttributeError("'%s' module has no attribute '%s'" %
                    (module._name, option))

            try:# to get an already constrained exact value (if any).
                # This is the way of how spawned instances get an option
                # which caused the spawning.
                ret_value = self._constraints.get(module, option)
            except ConstraintError:
                pass
            else:
                log.debug('mybuild: return %r', ret_value)
                return ret_value

            # Option without the module itself is meaningless. Fail-fast way
            # for case when the whole module has been previously excluded.
            self.constrain(module)

            octx = self._domain.domain_for(module, option)

            octx.subscribe(partial(self._branch_and_spawn,
                                   self._constraints, module, option))
            # after that one shouldn't touch self._constraints anymore
            self._constraints.freeze()

            def constrain_all(values):
                constrain = self._constraints.constrain
                for value in values:
                    try:
                        yield constrain(module, option, value, branch=True)
                    except ConstraintError:
                        pass

            self._constraints = self._take_one_spawn_rest(constrain_all(octx))

            ret_value = self._constraints.get(module, option) # must not throw

            log.debug('mybuild: return %r', ret_value)
            return ret_value

    def _branch_and_spawn(self, constraints, module, option, value):
        log.debug('mybuild: branch %r with %r by %r.%s = %r',
            self._optuple, constraints, module, option, value)

        try:
            constraints = constraints.constrain(module, option, value,
                branch=True)
        except ConstraintError as e:
            log.debug('mybuild: branch error: %s', e)
            pass
        else:
            log.debug('mybuild: branch OK')
            self._spawn(constraints)

    def _take_one_spawn_rest(self, constraints_iterable):
        constraints_it = iter(constraints_iterable)

        try: # Retrieve the first one (if any) to return it.
            ret_constraints = constraints_it.next()
        except StopIteration:
            raise InstanceError('No viable choice to take')
        else:
            log.debug('mybuild: take %r', ret_constraints)

        # Spawn for the rest ones.
        self._spawn_all(constraints_it)

        return ret_constraints

    def _spawn_all(self, constraints_iterable):
        for constraints in constraints_iterable:
            self._spawn(constraints)

    def _spawn(self, constraints):
        log.debug('mybuild: spawn %r', constraints)
        self._post_new(self._domain, self._optuple, constraints)

    def __repr__(self):
        return '<Instance %r with %r>' % (self._optuple, self._constraints)

    @classmethod
    def _new_type(cls, module_type, fxn):
        return type('Instance_M%s' % (module_type._module_name,),
                    (cls, module_type),
                    dict(__slots__=(), _init_fxn=fxn))

def build(conf_module):
    domain = Context()
    conf_domain = domain.domain_for(conf_module)

    assert len(conf_domain._instances) == 1
    conf_instance_set = conf_domain._instances.itervalues().next()

    assert len(conf_instance_set) == 1
    conf_instance = iter(conf_instance_set).next()

    constraints = conf_instance._constraints # XXX

    # flat_constr = constraints.branch().flatten()
    # for m in flat_constr._dict:
    #     ctx = self._modules[m]
    #     for mslice, inst_set in ctx._instances:
    #         if flat_constr.check_mslice(mslice) is not False:

    # try:
    #     constraints = Constraints.merge(instance._constraints
    #                                     for instance in domain._instances)
    # except Exception, e:
    #     raise e

if __name__ == '__main__':
    from mybuild import module, option

    log.zones = {'mybuild'}
    log.verbose = True
    log.init_log()

    @module
    def conf(self):
        self.constrain(m1)

    @module
    def m1(self):
        pass

    build(conf)

<commit_msg>throw out old implementation of instance<commit_after>"""
Types used on a per-build basis.
"""

__author__ = "Eldar Abusalimov"
__date__ = "2012-11-09"

__all__ = ["Context"]

from collections import defaultdict
from collections import deque
from collections import MutableSet
from contextlib import contextmanager
from functools import partial
from itertools import chain
from itertools import izip
from itertools import izip_longest
from itertools import product

from core import *
from constraints import *
from expr import *
from util import singleton
from util import NotifyingMixin
import pdag

import logs as log

class Context(object):
    """docstring for Context"""

    def __init__(self):
        super(Context, self).__init__()
        self._modules = {}
        self._job_queue = deque()
        self._reent_locked = False

    def freeze(self):
        if hasattr(self, '_pdag'):
            raise RuntimeError("'freeze' must be called only once.")

        # def iter_atoms():
        #     for module_domain in self._module.itervalues():

        self._pdag = pdag.Pdag(*iter_atoms())

    def post(self, fxn):
        with self.reent_lock(): # to flush the queue on block exit
            self._job_queue.append(fxn)

    @contextmanager
    def reent_lock(self):
        was_locked = self._reent_locked
        self._reent_locked = True

        try:
            yield
        finally:
            if not was_locked:
                self._job_queue_flush()
            self._reent_locked = was_locked

    def _job_queue_flush(self):
        queue = self._job_queue

        while queue:
            fxn = queue.popleft()
            fxn()

    def consider(self, module, option=None, value=Ellipsis):
        domain = self.domain_for(module)
        if option is not None:
            domain.consider_option(option, value)

    def register(self, instance):
        self.domain_for(instance._module).register(instance)

    def domain_for(self, module, option=None):
        try:
            domain = self._modules[module]
        except KeyError:
            with self.reent_lock():
                domain = self._modules[module] = ModuleDomain(self, module)

        if option is not None:
            domain = domain.domain_for(option)

        return domain

    def atom_for(self, module, option=None, value=Ellipsis, negated=False):
        cache = self._atom_cache
        cache_key = module, option, value, negated

        try:
            return cache[cache_key]
        except KeyError:
            pass

        if negated:
            ret = pdag.Not(self.atom_for(module, option, value, negated=False))
        else:
            domain = self.domain_for(module, option)
            ret = domain.atom if option is None else domain.atom_for(value)

        cache[cache_key] = ret

        return ret

    def pnode_from(self, expr):
        pass

class DomainBase(object):
    """docstring for DomainBase"""

    context = property(attrgetter('_context'))

    def __init__(self, context):
        super(DomainBase, self).__init__()
        self._context = context

class ModuleDomain(DomainBase):
    """docstring for ModuleDomain"""

    module = property(attrgetter('_module'))
    atom = property(attrgetter('_atom'))

    def __init__(self, context, module):
        super(ModuleDomain, self).__init__(context)

        self._module = module
        self._atom = module._atom_type()

        self._instances = {} # { optuple : InstanceDomain }
        self._options = module._options._make(OptionDomain(option)
                                              for option in module._options)

        self._instantiate_product(self._options)

    def init_pnode(self):
        presence_pnode = pdag.EqGroup(self._atom,
            *(option_domain.pnode for option_domain in self._options))
        instances_pnode = pdag.And(
            *(instance.pnode
              for instance_set in self._instances.itervalues()
              for instance in instance_set))

    def _instantiate_product(self, iterables):
        make_optuple = self._options._make
        instances = self._instances

        for new_tuple in product(*iterables):
            new_optuple = make_optuple(new_tuple)

            assert new_optuple not in instances
            instances[new_optuple] = InstanceDomain(self._context, new_optuple)

    def consider_option(self, option, value):
        domain_to_extend = getattr(self._options, option)
        if value in domain_to_extend:
            return

        log.debug('mybuild: extending %r with %r', domain_to_extend, value)
        domain_to_extend.add(value)

        self._instantiate_product(
            option_domain if option_domain is not domain_to_extend else (value,)
            for option_domain in self._options)

    def domain_for(self, option):
        return getattr(self._options, option)

@Module.register_attr('_atom_type')
class ModuleAtom(pdag.Atom):
    __slots__ = ()

    def __str__(self):
        return self._module_name

    @classmethod
    def _new_type(cls, module_type, *ignored):
        return type('ModuleAtom_M%s' % (module_type._module_name,),
                    (cls, module_type),
                    dict(__slots__=()))

class NotifyingSet(MutableSet, NotifyingMixin):
    """Set with notification support. Backed by a dictionary."""

    def __init__(self, values):
        super(NotifyingSet, self).__init__()
        self._dict = {}

        self |= values

    def _create_value_for(self, key):
        pass

    def add(self, value):
        if value in self:
            return
        self._dict[value] = self._create_value_for(value)

        self._notify(value)

    def discard(self, value):
        if value not in self:
            return
        raise NotImplementedError

    def __iter__(self):
        return iter(self._dict)
    def __len__(self):
        return len(self._dict)
    def __contains__(self, value):
        return value in self._dict

    def __str__(self):
        return '<%s: %s>' % (type(self).__name__, self._dict.keys())

class OptionDomain(NotifyingSet):
    """docstring for OptionDomain"""

    def __init__(self, option):
        self._option = option
        self.pnode = pdag.AtMostOne()

        super(OptionDomain, self).__init__(option._values)

    def atom_for(self, value):
        if value not in self:
            self.add(value)
        return self._dict[value]

    def _create_value_for(self, value):
        atom = self._option._atom_type(value)
        return self.pnode._new_operand(atom)

@Option.register_attr('_atom_type')
class OptionValueAtom(pdag.Atom):
    __slots__ = '_value'

    def __init__(self, value):
        super(OptionValueAtom, self).__init__()
        self._value = value

    def __str__(self):
        return '(%s.%s==%s)' % (self._module_name,
                                self._option_name, self._value)

    @classmethod
    def _new_type(cls, option_type):
        return type('OptionAtom_M%s_O%s' % (option_type._module_name,
                                            option_type._option_name),
                    (cls, option_type),
                    dict(__slots__=()))

def build(conf_module):
    domain = Context()
    conf_domain = domain.domain_for(conf_module)

    assert len(conf_domain._instances) == 1
    conf_instance_set = conf_domain._instances.itervalues().next()

    assert len(conf_instance_set) == 1
    conf_instance = iter(conf_instance_set).next()

    constraints = conf_instance._constraints # XXX

    # flat_constr = constraints.branch().flatten()
    # for m in flat_constr._dict:
    #     ctx = self._modules[m]
    #     for mslice, inst_set in ctx._instances:
    #         if flat_constr.check_mslice(mslice) is not False:

    # try:
    #     constraints = Constraints.merge(instance._constraints
    #                                     for instance in domain._instances)
    # except Exception, e:
    #     raise e

if __name__ == '__main__':
    from mybuild import module, option

    log.zones = {'mybuild'}
    log.verbose = True
    log.init_log()

    @module
    def conf(self):
        self.constrain(m1)

    @module
    def m1(self):
        pass

    build(conf)

<|endoftext|>


<commit_before>import config

from flask import Flask
import flask_sqlalchemy
sa = flask_sqlalchemy.sqlalchemy

db = flask_sqlalchemy.SQLAlchemy()

import string
from itertools import islice
from datetime import datetime

class SensTable(object):
    def __init__(self):
        pass

    def __len__(self):
        cnt = self.fridge_table().count()
        return db.session.execute(cnt).first()[0]

    def __getitem__(self, key):
        if isinstance(key, slice):
            if (key.start is not None and not isinstance(key.start, int)) \
               or (key.stop is not None and not isinstance(key.stop, int)):
                raise TypeError('Start and end indices must be numeric')

            start = key.start
            stop = key.stop
            step = key.step if key.step else 1
            if step < 0:
                raise ValueError("Negative step sizes are not supported")
            limit = None
            reverse = False

            if start is None:
                if stop is None:
                  return self[0::step]
                if stop < 0:
                    count = len(self)
                    stop = count+stop-1
                return self[0:stop:step]
            elif start < 0:
                if stop is not None:
                    count = len(self)
                    start = count+start
                    if stop < 0:
                        stop = count+stop-1
                    return self[start:stop:step]

                # Otherwise select in reverse order and swap order in python
                res = self.fridge_table().select(limit=abs(start))
                res = res.order_by(self.fridge_table().c.Time.desc())
                reverse = True
            else: # start > 0
                if stop is None:
                    pass # This is the same as limit=None
                elif stop < 0:
                    count = len(self)
                    stop = count+stop-1
                    return self[start:stop:step]
                else:
                    limit = stop-start
                    if limit < 0:
                        return iter(())

                res = self.fridge_table().select(offset=start, limit=limit)
                res = res.order_by(self.fridge_table().c.Time.asc())

            res = db.session.execute(res)

            if reverse:
                res = reversed(tuple(res))

            return islice(res, None, None, step)
        elif isinstance(key, int):
            if key < 0:
                offset = abs(key) - 1
            else:
                offset = key
            res = self.fridge_table().select(offset=offset, limit=1)
            if key >= 0:
                res = res.order_by(self.fridge_table().c.Time.asc())
            else:
                res = res.order_by(self.fridge_table().c.Time.desc())
            res = db.session.execute(res)
            res = res.first()
            if res is None:
                raise KeyError("Index out of range")
            return res
        else:
            raise TypeError('List indices must be int')

    def __setitem__(self, key, value):
        if not isinstance(key, int):
            raise TypeError('List index must be int')
        if key < 0:
            key = len(self)+key

        if key >= len(self):
            raise IndexError('List index out of range')
        res = self[key]
        stmt = self.fridge_table().update().where(self.fridge_table().c.Time==res['Time']).values(**value)
        db.session.execute(stmt)
        db.session.commit()

    def __delitem__(self, key):
        items = self[key]
        if isinstance(key, int):
            items = [items]
        items = list(items)
        for i in items:
            stmt = self.fridge_table().delete().where(self.fridge_table().c.Time==i['Time'])
            db.session.execute(stmt)
        db.session.commit()

    def __iter__(self):
        return iter(self[:])

    def __reversed__(self):
        return iter(self[::-1])

    def append(self, **values):
        if "Time" in values:
            time = values["Time"]
            del values["Time"]
        else:
            time = datetime.now()

        try:
            ins = self.fridge_table().insert().values(Time=time, **values)
            db.session.execute(ins)
            db.session.commit()
        except sa.exc.CompileError:
            raise KeyError("Invalid column name")
        except sa.exc.IntegrityError:
            # This occurs if we try and add a duplicate timestamp
            # We can fail quietly here
            return False

    def remove(self):
        stmt = self.fridge_table().delete().where(fridge.table().c.Time == i['Time'])
        db.session.execute(stmt)
        db.session.commit()

    def hourly_avg(self, sensor):
        columns = self.fridge_table().columns
        dategroup = sa.func.date_trunc('hour', sa.func.timezone('UTC', columns.Time)).label("Time")
        dategroup.type = sa.DateTime()
        if sensor not in columns:
            raise KeyError("Sensor not found")
        sensor_c = sa.func.avg(columns[sensor]).label(sensor)
        query = sa.select((dategroup, sensor_c)).group_by(dategroup).order_by(dategroup.asc())
        return db.session.execute(query)

    def range(self, start, stop):
        query = self.fridge_table().select().where(self.fridge_table().columns.Time.between(start, stop)).order_by(self.fridge_table().columns.Time.asc())
        return db.session.execute(query)

class SensorMeta(flask_sqlalchemy._BoundDeclarativeMeta, sa.sql.visitors.VisitableType):
    pass

class Sensors(db.Model, db.Column, metaclass=SensorMeta):
    id = db.Column(db.Integer(), primary_key=True)
    fridge_id = db.Column(db.Integer(), db.ForeignKey('Fridges.id'))
    column_name = db.Column(db.String(1024))
    display_name = db.Column("name", db.String(1024))
    view_order = db.Column(db.Integer())
    visible = db.Column(db.Integer())
    __tablename__ = "Sensors"

    def __init__(self, display_name, column_name, fridge, view_order=1, visible=1):
        if isinstance(fridge, Fridges):
            self.fridge_id = fridge.id
        else:
            self.fridge_id = int(fridge)
        self.display_name = name
        self.column_name = name
        self.view_order = view_order
        self.visible = visible

        db.Column.__init__(self, name, Float)

    def __repr__(self):
        return "<Sensor %r>" % (self.display_name)

    @classmethod
    def get_sensor(cls, name, fridge, add=False):
        if isinstance(fridge, Fridges):
            fridge_id = fridge.id
        else:
            fridge_id = int(fridge)

        sensor = cls.query.filter_by(column_name=name, fridge_id=fridge_id)
        if sensor.count() == 0 and add:
            sensor = Sensors(name, fridge)
            db.session.add(sensor)
            return sensor
        elif sensor.count() == 0 and not add:
            raise KeyError("Sensor not found")
        return sensor.first()

class Sensors_Supplementary(db.Model):
    id = db.Column(db.Integer(), primary_key=True)
    fridge_suppl_id = db.Column(db.Integer(), db.ForeignKey('Fridges_Supplementary.id'))
    column_name = db.Column(db.String(1024))
    display_name = db.Column("name", db.String(1024))
    view_order = db.Column(db.Integer())
    visible = db.Column(db.Integer())
    __tablename__ = "Sensors_Supplementary"

    def __init__(self, name, fridge_suppl, column_name=None, view_order=1, visible=1):
        if isinstance(fridge_suppl, Fridges_Supplementary):
            self.fridge_suppl_id = fridge_suppl.id
        else:
            self.fridge_suppl_id = int(fridge_suppl)
        self.column_name = name
        if column_name:
            self.column_name = column_name
        else:
            self.column_name = name
        self.view_order = view_order
        self.visible = visible

    def __repr__(self):
        return "<Supplementary_Sensor %r>" % (self.column_name)

    @classmethod
    def get_sensor(cls, name, fridge, add=False):
        if isinstance(fridge, Fridges_Supplementary):
            suppl_id = fridge.id
        else:
            suppl_id = int(fridge)

        sensor = cls.query.filter_by(column_name=name, fridge_suppl_id=suppl_id)
        if sensor.count() == 0 and add:
            sensor = Sensors_Supplementary(name, fridge, name)
            db.session.add(sensor)
            return sensor
        elif sensor.count() == 0 and not add:
            raise KeyError("Unknown Sensor")
        return sensor.first()

class Fridges_Supplementary(db.Model, SensTable):
    id = db.Column(db.Integer(), primary_key=True)
    fridge_id = db.Column(db.Integer(), db.ForeignKey('Fridges.id'))
    table_name = db.Column(db.String(1024))
    name = db.Column(db.String(1024))
    label = db.Column(db.String(1024))
    comment = db.Column(db.UnicodeText())
    sensors = db.relationship('Sensors_Supplementary', backref='fridge_suppl', lazy='dynamic')
    __tablename__ = "Fridges_Supplementary"

    def __init__(self, name, table_name, label, sensors, comment=None):
        self.table_name = table_name
        self.name = name
        self.label = label
        for sensor in sensors:
            self.sensors.append(Sensors_Supplementary.get_sensor(sensor, self, True))
        self.comment = comment

    def get_sensor(self, sensor):
        return Sensors_Supplementary.get_sensor(sensor, self, False)

    def __repr__(self):
        return "<Supplementary for %r: %r>" % (self.fridge, self.table_name)

    def fridge_table(self):
        # Sanitize name by replacing all spacial characters
        if self.table_name in db.metadata.tables:
            return db.metadata.tables[self.table_name]
        elif not hasattr(self, '_suppl_table') or self._suppl_table is None:
            self._suppl_table = db.Table(self.table_name, db.metadata,
                                         db.Column('Time', db.TIMESTAMP, primary_key=True),
                                         *[db.Column(sensor.column_name, db.Float) for sensor in self.sensors])
            return self._suppl_table

class Fridges(db.Model, SensTable):
    id = db.Column(db.Integer(), primary_key=True)
    name = db.Column(db.String(255), unique=True)
    comment = db.Column(db.UnicodeText())
    sensors = db.relationship('Sensors', backref='fridge', lazy='dynamic')
    supplementary = db.relationship('Fridges_Supplementary', backref='fridge', lazy='dynamic')
    __tablename__ = "Fridges"

    # Cache of existing table definitions
    __tables__ = {}

    def __init__(self, name, sensors, comment=None):
        self.name = name
        for sensor in sensors:
            self.sensors.append(Sensors.get_sensor(sensor, self, True))
        self.comment = comment if comment is not None else ""
        self._fridge_table = None

    def __repr__(self):
        return "<Fridge %r>" % self.name

    def get_sensor(self, sensor):
        return Sensors.get_sensor(sensor, self, False)

    def fridge_table(self):
        # Sanitize name by replacing all spacial characters
        name = "".join(c if c in string.ascii_letters+string.digits else "_" for c in self.name)
        if name in db.metadata.tables:
            return db.metadata.tables[name]
        elif not hasattr(self, '_fridge_table') or self._fridge_table is None:
            self._fridge_table = db.Table(name, db.metadata,
                                   db.Column('Time', db.TIMESTAMP, primary_key=True),
                                   *[db.Column(sensor.column_name, db.Float) for sensor in self.sensors])
        return self._fridge_table

    @classmethod
    def get_fridge(cls, name):
        # Sanitize name by replacing all spacial characters
        name = "".join(c if c in string.ascii_letters+string.digits else " " for c in name)
        res = Fridges.query.filter(Fridges.name==name).first()
        if res is None:
            raise KeyError("Unable to find fridge")
        return res

if __name__ == "__main__":
    from app import app
    # Bind db
    db.app = app
    db.init_app(app)

    # Bootstrap table and data creation
    db.create_all()

    # Generate a default list of fridges, and thermometers on those fridges
    thermometers = ["MC_PT", "MC_Speer", "50mK_RuO", "Still_RuO", "4K_RuO"]
    fridges = ["Big_Fridge"]
    fridge_models = []

    # Create tables for those fridges
    for fridge in fridges:
        fridge_model = Fridges(name=fridge, thermometers=thermometers)
        fridge_table = fridge_model.fridge_table()
        fridge_models.append(fridge_model)

    # Then, create all the new tables
    db.create_all()
    # And finally, add all models
    for fridge in fridge_models:
        db.session.add(fridge)
    db.session.commit()

<commit_msg>Fix some initialization errors<commit_after>import config

from flask import Flask
import flask_sqlalchemy
sa = flask_sqlalchemy.sqlalchemy

db = flask_sqlalchemy.SQLAlchemy()

import string
from itertools import islice
from datetime import datetime

class SensTable(object):
    def __init__(self):
        pass

    def __len__(self):
        cnt = self.fridge_table().count()
        return db.session.execute(cnt).first()[0]

    def __getitem__(self, key):
        if isinstance(key, slice):
            if (key.start is not None and not isinstance(key.start, int)) \
               or (key.stop is not None and not isinstance(key.stop, int)):
                raise TypeError('Start and end indices must be numeric')

            start = key.start
            stop = key.stop
            step = key.step if key.step else 1
            if step < 0:
                raise ValueError("Negative step sizes are not supported")
            limit = None
            reverse = False

            if start is None:
                if stop is None:
                  return self[0::step]
                if stop < 0:
                    count = len(self)
                    stop = count+stop-1
                return self[0:stop:step]
            elif start < 0:
                if stop is not None:
                    count = len(self)
                    start = count+start
                    if stop < 0:
                        stop = count+stop-1
                    return self[start:stop:step]

                # Otherwise select in reverse order and swap order in python
                res = self.fridge_table().select(limit=abs(start))
                res = res.order_by(self.fridge_table().c.Time.desc())
                reverse = True
            else: # start > 0
                if stop is None:
                    pass # This is the same as limit=None
                elif stop < 0:
                    count = len(self)
                    stop = count+stop-1
                    return self[start:stop:step]
                else:
                    limit = stop-start
                    if limit < 0:
                        return iter(())

                res = self.fridge_table().select(offset=start, limit=limit)
                res = res.order_by(self.fridge_table().c.Time.asc())

            res = db.session.execute(res)

            if reverse:
                res = reversed(tuple(res))

            return islice(res, None, None, step)
        elif isinstance(key, int):
            if key < 0:
                offset = abs(key) - 1
            else:
                offset = key
            res = self.fridge_table().select(offset=offset, limit=1)
            if key >= 0:
                res = res.order_by(self.fridge_table().c.Time.asc())
            else:
                res = res.order_by(self.fridge_table().c.Time.desc())
            res = db.session.execute(res)
            res = res.first()
            if res is None:
                raise KeyError("Index out of range")
            return res
        else:
            raise TypeError('List indices must be int')

    def __setitem__(self, key, value):
        if not isinstance(key, int):
            raise TypeError('List index must be int')
        if key < 0:
            key = len(self)+key

        if key >= len(self):
            raise IndexError('List index out of range')
        res = self[key]
        stmt = self.fridge_table().update().where(self.fridge_table().c.Time==res['Time']).values(**value)
        db.session.execute(stmt)
        db.session.commit()

    def __delitem__(self, key):
        items = self[key]
        if isinstance(key, int):
            items = [items]
        items = list(items)
        for i in items:
            stmt = self.fridge_table().delete().where(self.fridge_table().c.Time==i['Time'])
            db.session.execute(stmt)
        db.session.commit()

    def __iter__(self):
        return iter(self[:])

    def __reversed__(self):
        return iter(self[::-1])

    def append(self, **values):
        if "Time" in values:
            time = values["Time"]
            del values["Time"]
        else:
            time = datetime.now()

        try:
            ins = self.fridge_table().insert().values(Time=time, **values)
            db.session.execute(ins)
            db.session.commit()
        except sa.exc.CompileError:
            raise KeyError("Invalid column name")
        except sa.exc.IntegrityError:
            # This occurs if we try and add a duplicate timestamp
            # We can fail quietly here
            return False

    def remove(self):
        stmt = self.fridge_table().delete().where(fridge.table().c.Time == i['Time'])
        db.session.execute(stmt)
        db.session.commit()

    def hourly_avg(self, sensor):
        columns = self.fridge_table().columns
        dategroup = sa.func.date_trunc('hour', sa.func.timezone('UTC', columns.Time)).label("Time")
        dategroup.type = sa.DateTime()
        if sensor not in columns:
            raise KeyError("Sensor not found")
        sensor_c = sa.func.avg(columns[sensor]).label(sensor)
        query = sa.select((dategroup, sensor_c)).group_by(dategroup).order_by(dategroup.asc())
        return db.session.execute(query)

    def range(self, start, stop):
        query = self.fridge_table().select().where(self.fridge_table().columns.Time.between(start, stop)).order_by(self.fridge_table().columns.Time.asc())
        return db.session.execute(query)

class SensorMeta(flask_sqlalchemy._BoundDeclarativeMeta, sa.sql.visitors.VisitableType):
    pass

class Sensors(db.Model, db.Column, metaclass=SensorMeta):
    id = db.Column(db.Integer(), primary_key=True)
    fridge_id = db.Column(db.Integer(), db.ForeignKey('Fridges.id'))
    column_name = db.Column(db.String(1024))
    display_name = db.Column("name", db.String(1024))
    view_order = db.Column(db.Integer())
    visible = db.Column(db.Integer())
    __tablename__ = "Sensors"

    def __init__(self, display_name, column_name, fridge, view_order=1, visible=1):
        if isinstance(fridge, Fridges):
            self.fridge_id = fridge.id
        else:
            self.fridge_id = int(fridge)
        self.display_name = display_name
        self.column_name = column_name
        self.view_order = view_order
        self.visible = visible

        db.Column.__init__(self, column_name, db.Float)

    def __repr__(self):
        return "<Sensor %r>" % (self.display_name)

    @classmethod
    def get_sensor(cls, name, fridge, add=False):
        if isinstance(fridge, Fridges):
            fridge_id = fridge.id
        else:
            fridge_id = int(fridge)

        sensor = cls.query.filter_by(column_name=name, fridge_id=fridge_id)
        if sensor.count() == 0 and add:
            safe_name = name.replace(' ', '_')
            sensor = Sensors(name, safe_name, fridge)
            db.session.add(sensor)
            return sensor
        elif sensor.count() == 0 and not add:
            raise KeyError("Sensor not found")
        return sensor.first()

class Sensors_Supplementary(db.Model):
    id = db.Column(db.Integer(), primary_key=True)
    fridge_suppl_id = db.Column(db.Integer(), db.ForeignKey('Fridges_Supplementary.id'))
    column_name = db.Column(db.String(1024))
    display_name = db.Column("name", db.String(1024))
    view_order = db.Column(db.Integer())
    visible = db.Column(db.Integer())
    __tablename__ = "Sensors_Supplementary"

    def __init__(self, name, fridge_suppl, column_name=None, view_order=1, visible=1):
        if isinstance(fridge_suppl, Fridges_Supplementary):
            self.fridge_suppl_id = fridge_suppl.id
        else:
            self.fridge_suppl_id = int(fridge_suppl)
        self.column_name = name
        if column_name:
            self.column_name = column_name
        else:
            self.column_name = name
        self.view_order = view_order
        self.visible = visible

    def __repr__(self):
        return "<Supplementary_Sensor %r>" % (self.column_name)

    @classmethod
    def get_sensor(cls, name, fridge, add=False):
        if isinstance(fridge, Fridges_Supplementary):
            suppl_id = fridge.id
        else:
            suppl_id = int(fridge)

        sensor = cls.query.filter_by(column_name=name, fridge_suppl_id=suppl_id)
        if sensor.count() == 0 and add:
            sensor = Sensors_Supplementary(name, fridge, name)
            db.session.add(sensor)
            return sensor
        elif sensor.count() == 0 and not add:
            raise KeyError("Unknown Sensor")
        return sensor.first()

class Fridges_Supplementary(db.Model, SensTable):
    id = db.Column(db.Integer(), primary_key=True)
    fridge_id = db.Column(db.Integer(), db.ForeignKey('Fridges.id'))
    table_name = db.Column(db.String(1024))
    name = db.Column(db.String(1024))
    label = db.Column(db.String(1024))
    comment = db.Column(db.UnicodeText())
    sensors = db.relationship('Sensors_Supplementary', backref='fridge_suppl', lazy='dynamic')
    __tablename__ = "Fridges_Supplementary"

    def __init__(self, name, table_name, label, sensors, comment=None):
        self.table_name = table_name
        self.name = name
        self.label = label
        for sensor in sensors:
            self.sensors.append(Sensors_Supplementary.get_sensor(sensor, self, True))
        self.comment = comment

    def get_sensor(self, sensor):
        return Sensors_Supplementary.get_sensor(sensor, self, False)

    def __repr__(self):
        return "<Supplementary for %r: %r>" % (self.fridge, self.table_name)

    def fridge_table(self):
        # Sanitize name by replacing all spacial characters
        if self.table_name in db.metadata.tables:
            return db.metadata.tables[self.table_name]
        elif not hasattr(self, '_suppl_table') or self._suppl_table is None:
            self._suppl_table = db.Table(self.table_name, db.metadata,
                                         db.Column('Time', db.TIMESTAMP, primary_key=True),
                                         *[db.Column(sensor.column_name, db.Float) for sensor in self.sensors])
            return self._suppl_table

class Fridges(db.Model, SensTable):
    id = db.Column(db.Integer(), primary_key=True)
    name = db.Column(db.String(255), unique=True)
    comment = db.Column(db.UnicodeText())
    sensors = db.relationship('Sensors', backref='fridge', lazy='dynamic')
    supplementary = db.relationship('Fridges_Supplementary', backref='fridge', lazy='dynamic')
    __tablename__ = "Fridges"

    # Cache of existing table definitions
    __tables__ = {}

    def __init__(self, name, sensors, comment=None):
        self.name = name
        for sensor in sensors:
            self.sensors.append(Sensors.get_sensor(sensor, self, True))
        self.comment = comment if comment is not None else ""
        self._fridge_table = None

    def __repr__(self):
        return "<Fridge %r>" % self.name

    def get_sensor(self, sensor):
        return Sensors.get_sensor(sensor, self, False)

    def fridge_table(self):
        # Sanitize name by replacing all spacial characters
        name = "".join(c if c in string.ascii_letters+string.digits else "_" for c in self.name)
        if name in db.metadata.tables:
            return db.metadata.tables[name]
        elif not hasattr(self, '_fridge_table') or self._fridge_table is None:
            self._fridge_table = db.Table(name, db.metadata,
                                   db.Column('Time', db.TIMESTAMP, primary_key=True),
                                   *[db.Column(sensor.column_name, db.Float) for sensor in self.sensors])
        return self._fridge_table

    @classmethod
    def get_fridge(cls, name):
        # Sanitize name by replacing all spacial characters
        name = "".join(c if c in string.ascii_letters+string.digits else " " for c in name)
        res = Fridges.query.filter(Fridges.name==name).first()
        if res is None:
            raise KeyError("Unable to find fridge")
        return res

if __name__ == "__main__":
    from app import app
    # Bind db
    db.app = app
    db.init_app(app)

    # Bootstrap table and data creation
    db.create_all()

    # Generate a default list of fridges, and thermometers on those fridges
    thermometers = ["MC_PT", "MC_Speer", "50mK_RuO", "Still_RuO", "4K_RuO"]
    fridges = ["Big_Fridge"]
    fridge_models = []

    # Create tables for those fridges
    for fridge in fridges:
        fridge_model = Fridges(name=fridge, sensors=thermometers)
        fridge_table = fridge_model.fridge_table()
        fridge_models.append(fridge_model)

    # Then, create all the new tables
    db.create_all()
    # And finally, add all models
    for fridge in fridge_models:
        db.session.add(fridge)
    db.session.commit()

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import logging

from flask.ext.script import (  # pylint: disable=E0611,F0401
                            Command, Group, Option)
from progress.bar import ShadyBar

from warehouse import db, redis, script
from warehouse import utils
from warehouse.packages import diff, store
from warehouse.packages.models import Project, FileType
from warehouse.synchronize.fetchers import PyPIFetcher

REDIS_SINCE_KEY = "warehouse:since"
REDIS_SYNC_LOCK_KEY = "warehouse:sync:lock:{project}"

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class DummyBar(object):
    def iter(self, iterable):
        for item in iterable:
            yield item

def synchronize_project(project, fetcher, download=None):
    key = REDIS_SYNC_LOCK_KEY.format(project=project)

    with redis.lock(key, timeout=60 * 10):
        logger.info("Synchronizing '%s' from pypi.python.org", project)

        project = store.project(project)
        versions = fetcher.versions(project.name)

        for ver in versions:
            logger.debug(
                "Synchronizing version '%s' of '%s' from pypi.python.org",
                ver,
                project.name,
            )

            release = fetcher.release(project.name, ver)
            version = store.version(project, release)

            dists = fetcher.distributions(project.name, version.version)
            dists = list(dists)

            for dist in dists:
                logger.debug(
                    "Synchronizing '%s' from version '%s' of '%s' "
                        "from pypi.python.org",
                    dist["filename"],
                    version.version,
                    project.name,
                )

                distribution = store.distribution(version, dist)

                if distribution.hashes is None:
                    current_hashes = {}
                else:
                    current_hashes = distribution.hashes

                # Handle the True/False/None logic of download, and if
                #   download is None check if our stored hash matches
                #   the hash from PyPI
                if download or (
                            download is None and
                            dist["md5_digest"] != current_hashes.get("md5")
                        ):
                    file_data = fetcher.file(dist["url"])
                    store.distribution_file(distribution, file_data)

                    if distribution.type == FileType.source:
                        store.setuptools_requires(
                            version,
                            distribution.filename,
                            file_data,
                        )

            # Yank distributions that no longer exist in PyPI
            logger.debug("Diffing distributions of '%s' version '%s'",
                    project.name,
                    version.version,
                )
            diff.distributions(version, [x["filename"] for x in dists])

        # Yank versions that no longer exist in PyPI
        logger.debug("Diffing versions of '%s'", project.name)
        diff.versions(project, versions)

        # Commit our changes
        db.session.commit()

def synchronize_classifiers(fetcher):
    # Sync the Classifiers
    for classifier in fetcher.classifiers():
        store.classifier(classifier)

    # Commit the classifiers
    db.session.commit()

def synchronize_by_journals(since=None, fetcher=None, progress=True,
        download=None):
    if fetcher is None:
        fetcher = PyPIFetcher()

    # Grab the current datetime
    current = fetcher.current()

    # Synchronize all the classifiers with PyPI
    synchronize_classifiers(fetcher)

    # Grab the journals since `since`
    journals = fetcher.journals(since=since)

    # Storage for projects that have been updated or deleted
    updated = set()
    deleted = set()

    # Check if we have anything to process before attempting to
    if journals:
        if progress:
            bar = ShadyBar("Processing Journals", max=len(journals))
        else:
            bar = DummyBar()

        for journal in bar.iter(journals):
            if (journal.action.lower() == "remove" and
                    journal.version is None):
                # Delete the entire project
                if journal.name not in deleted:
                    updated.discard(journal.name)
                    deleted.add(journal.name)

                    # Actually yank the project
                    Project.yank(journal.name, synchronize=False)
                    db.session.commit()
            else:
                # Process the update
                if journal.name not in updated:
                    deleted.discard(journal.name)
                    updated.add(journal.name)

                    # Actually synchronize the project
                    synchronize_project(journal.name,
                        fetcher,
                        download=download,
                    )

    logger.info(
        "Finished processing journals at %s; updated %s and deleted %s",
        current, len(updated), len(deleted),
    )

    return current

def synchronize_by_projects(projects=None, fetcher=None, progress=True,
        download=None):
    if fetcher is None:
        fetcher = PyPIFetcher()

    # Grab the current datetime
    current = fetcher.current()

    # Synchronize all the classifiers with PyPI
    synchronize_classifiers(fetcher)

    if not projects:
        # Grab a list of projects from PyPI
        projects = fetcher.projects()

        # We are not synchronizing a subset of projects, so we can check for
        #   any deletions (if required) and yank them.
        diff.projects(projects)

        # Commit our yanked projects
        db.session.commit()

    if progress:
        bar = ShadyBar("Processing Projects", max=len(projects))
    else:
        bar = DummyBar()

    for project in bar.iter(projects):
        synchronize_project(project, fetcher, download=download)

    return current

class Synchronize(Command):
    """
    Synchronizes Warehouse with PyPI.
    """

    # pylint: disable=W0232

    option_list = [
        Option("projects", nargs="*", help="list of projects to synchronize"),
        Option("--repeat-every",
            type=int,
            dest="repeat",
            default=False,
            help="repeat the synchronization with the selected options every "
                "REPEAT seconds",
        ),
        Option("--no-progress",
            action="store_false",
            dest="progress",
            help="do not display a progress bar",
        ),
        Option("--no-store",
            action="store_false",
            dest="store_since",
            default=True,
            help="do not store the synchronization time",
        ),
        Group(
            Option("--full",
                action="store_true",
                dest="full",
                default=False,
                help="force a full synchronization instead of differential",
            ),
            Option("--since",
                type=int,
                default=None,
                help="synchronize since SINCE",
            ),
            exclusive=True,
        ),
        Group(
            Option("--force-download",
                action="store_true",
                dest="download",
                help="force downloading packages even if the hashes match",
                default=None,
            ),
            Option("--no-download",
                action="store_false",
                dest="download",
                help="disable downloading of files even if hashes mismatch",
            ),
            exclusive=True,
        ),
    ]

    def run(self, projects=None, progress=True, download=None,
            since=None, full=False, store_since=True, repeat=False):
        # This is a hack to normalize the incoming projects to unicode
        projects = [x.decode("utf-8") for x in projects]

        if projects:
            logger.info("Will synchronize %s from pypi.python.org", projects)
        else:
            logger.info("will synchronize all projects from pypi.python.org")

        # record if we should be grabbing since from redis
        fetch_since = not since

        for _ in utils.repeat_every(
                    seconds=repeat if repeat else 0,
                    times=None if repeat else 1,
                ):
            if full or projects:
                # We are preforming a full synchronization, or by a list of
                #   projects
                synced = synchronize_by_projects(projects,
                            progress=progress,
                            download=download,
                        )
            else:
                if fetch_since:
                    # Grab the since key from redis
                    fetched = redis.get(REDIS_SINCE_KEY)
                    since = int(fetched) if not fetched is None else None

                # We are preforming a standard journal based synchronization
                synced = synchronize_by_journals(since,
                        progress=progress,
                        download=download,
                    )

            # Save our synchronization time in redis
            if store_since:
                redis.set(REDIS_SINCE_KEY, synced)

            # Output the time we started the sync
            logger.info(
                "Synchronization from pypi.python.org completed at %s",
                synced,
            )

script.add_command("sync", Synchronize())
<commit_msg>Deduplicate some logging<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import logging

from flask.ext.script import (  # pylint: disable=E0611,F0401
                            Command, Group, Option)
from progress.bar import ShadyBar

from warehouse import db, redis, script
from warehouse import utils
from warehouse.packages import diff, store
from warehouse.packages.models import Project, FileType
from warehouse.synchronize.fetchers import PyPIFetcher

REDIS_SINCE_KEY = "warehouse:since"
REDIS_SYNC_LOCK_KEY = "warehouse:sync:lock:{project}"

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

class DummyBar(object):
    def iter(self, iterable):
        for item in iterable:
            yield item

def synchronize_project(project, fetcher, download=None):
    key = REDIS_SYNC_LOCK_KEY.format(project=project)

    with redis.lock(key, timeout=60 * 10):
        logger.info("Synchronizing '%s' from pypi.python.org", project)

        project = store.project(project)
        versions = fetcher.versions(project.name)

        for ver in versions:
            logger.debug(
                "Synchronizing version '%s' of '%s' from pypi.python.org",
                ver,
                project.name,
            )

            release = fetcher.release(project.name, ver)
            version = store.version(project, release)

            dists = fetcher.distributions(project.name, version.version)
            dists = list(dists)

            for dist in dists:
                logger.debug(
                    "Synchronizing '%s' from version '%s' of '%s' "
                        "from pypi.python.org",
                    dist["filename"],
                    version.version,
                    project.name,
                )

                distribution = store.distribution(version, dist)

                if distribution.hashes is None:
                    current_hashes = {}
                else:
                    current_hashes = distribution.hashes

                # Handle the True/False/None logic of download, and if
                #   download is None check if our stored hash matches
                #   the hash from PyPI
                if download or (
                            download is None and
                            dist["md5_digest"] != current_hashes.get("md5")
                        ):
                    file_data = fetcher.file(dist["url"])
                    store.distribution_file(distribution, file_data)

                    if distribution.type == FileType.source:
                        store.setuptools_requires(
                            version,
                            distribution.filename,
                            file_data,
                        )

            # Yank distributions that no longer exist in PyPI
            logger.debug("Diffing distributions of '%s' version '%s'",
                    project.name,
                    version.version,
                )
            diff.distributions(version, [x["filename"] for x in dists])

        # Yank versions that no longer exist in PyPI
        logger.debug("Diffing versions of '%s'", project.name)
        diff.versions(project, versions)

        # Commit our changes
        db.session.commit()

def synchronize_classifiers(fetcher):
    # Sync the Classifiers
    for classifier in fetcher.classifiers():
        store.classifier(classifier)

    # Commit the classifiers
    db.session.commit()

def synchronize_by_journals(since=None, fetcher=None, progress=True,
        download=None):
    if fetcher is None:
        fetcher = PyPIFetcher()

    # Grab the current datetime
    current = fetcher.current()

    # Synchronize all the classifiers with PyPI
    synchronize_classifiers(fetcher)

    # Grab the journals since `since`
    journals = fetcher.journals(since=since)

    # Storage for projects that have been updated or deleted
    updated = set()
    deleted = set()

    # Check if we have anything to process before attempting to
    if journals:
        if progress:
            bar = ShadyBar("Processing Journals", max=len(journals))
        else:
            bar = DummyBar()

        for journal in bar.iter(journals):
            if (journal.action.lower() == "remove" and
                    journal.version is None):
                # Delete the entire project
                if journal.name not in deleted:
                    updated.discard(journal.name)
                    deleted.add(journal.name)

                    # Actually yank the project
                    Project.yank(journal.name, synchronize=False)
                    db.session.commit()
            else:
                # Process the update
                if journal.name not in updated:
                    deleted.discard(journal.name)
                    updated.add(journal.name)

                    # Actually synchronize the project
                    synchronize_project(journal.name,
                        fetcher,
                        download=download,
                    )

    logger.info(
        "Finished processing journals at %s; updated %s and deleted %s",
        current, len(updated), len(deleted),
    )

    return current

def synchronize_by_projects(projects=None, fetcher=None, progress=True,
        download=None):
    if fetcher is None:
        fetcher = PyPIFetcher()

    # Grab the current datetime
    current = fetcher.current()

    # Synchronize all the classifiers with PyPI
    synchronize_classifiers(fetcher)

    if not projects:
        # Grab a list of projects from PyPI
        projects = fetcher.projects()

        # We are not synchronizing a subset of projects, so we can check for
        #   any deletions (if required) and yank them.
        diff.projects(projects)

        # Commit our yanked projects
        db.session.commit()

    if progress:
        bar = ShadyBar("Processing Projects", max=len(projects))
    else:
        bar = DummyBar()

    for project in bar.iter(projects):
        synchronize_project(project, fetcher, download=download)

    logger.info("Finished processing projects at %s", current)

    return current

class Synchronize(Command):
    """
    Synchronizes Warehouse with PyPI.
    """

    # pylint: disable=W0232

    option_list = [
        Option("projects", nargs="*", help="list of projects to synchronize"),
        Option("--repeat-every",
            type=int,
            dest="repeat",
            default=False,
            help="repeat the synchronization with the selected options every "
                "REPEAT seconds",
        ),
        Option("--no-progress",
            action="store_false",
            dest="progress",
            help="do not display a progress bar",
        ),
        Option("--no-store",
            action="store_false",
            dest="store_since",
            default=True,
            help="do not store the synchronization time",
        ),
        Group(
            Option("--full",
                action="store_true",
                dest="full",
                default=False,
                help="force a full synchronization instead of differential",
            ),
            Option("--since",
                type=int,
                default=None,
                help="synchronize since SINCE",
            ),
            exclusive=True,
        ),
        Group(
            Option("--force-download",
                action="store_true",
                dest="download",
                help="force downloading packages even if the hashes match",
                default=None,
            ),
            Option("--no-download",
                action="store_false",
                dest="download",
                help="disable downloading of files even if hashes mismatch",
            ),
            exclusive=True,
        ),
    ]

    def run(self, projects=None, progress=True, download=None,
            since=None, full=False, store_since=True, repeat=False):
        # This is a hack to normalize the incoming projects to unicode
        projects = [x.decode("utf-8") for x in projects]

        if projects:
            logger.info("Will synchronize %s from pypi.python.org", projects)
        else:
            logger.info("will synchronize all projects from pypi.python.org")

        # record if we should be grabbing since from redis
        fetch_since = not since

        for _ in utils.repeat_every(
                    seconds=repeat if repeat else 0,
                    times=None if repeat else 1,
                ):
            if full or projects:
                # We are preforming a full synchronization, or by a list of
                #   projects
                synced = synchronize_by_projects(projects,
                            progress=progress,
                            download=download,
                        )
            else:
                if fetch_since:
                    # Grab the since key from redis
                    fetched = redis.get(REDIS_SINCE_KEY)
                    since = int(fetched) if not fetched is None else None

                # We are preforming a standard journal based synchronization
                synced = synchronize_by_journals(since,
                        progress=progress,
                        download=download,
                    )

            # Save our synchronization time in redis
            if store_since:
                redis.set(REDIS_SINCE_KEY, synced)

script.add_command("sync", Synchronize())
<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE
from __future__ import print_function, absolute_import, division

import pytest
import astropy.units as u

from ._testing_data import header
from ..io import find_beam_width, find_beam_properties

def test_load_beam():

    beam_header = header.copy()
    beam_header["BMAJ"] = 1.0

    beamwidth = find_beam_width(beam_header)

    assert beamwidth == 1.0 * u.deg

@pytest.mark.parametrize(('major', 'minor', 'pa'), [(1.0, 0.5, 10),
                                                    (1.0, 'skip', 10),
                                                    (1.0, 0.5, 'skip')])
def test_load_beam_props(major, minor, pa):

    beam_header = header.copy()
    beam_header["BMAJ"] = major
    if minor != 'skip':
        beam_header["BMIN"] = minor
    if pa != 'skip':
        beam_header["BPA"] = pa

    bmaj, bmin, bpa = find_beam_properties(beam_header)

    assert bmaj == major * u.deg

    if minor == 'skip':
        assert bmin == major * u.deg
    else:
        assert bmin == minor * u.deg

    if pa == "skip":
        assert bpa == 0 * u.deg
    else:
        assert bpa == pa * u.deg

@pytest.mark.xfail(raises=ValueError)
def test_load_beam_fail():

    find_beam_width(header)
<commit_msg>Also xfail on TypeErrors<commit_after># Licensed under an MIT open source license - see LICENSE
from __future__ import print_function, absolute_import, division

import pytest
import astropy.units as u

from ._testing_data import header
from ..io import find_beam_width, find_beam_properties

def test_load_beam():

    beam_header = header.copy()
    beam_header["BMAJ"] = 1.0

    beamwidth = find_beam_width(beam_header)

    assert beamwidth == 1.0 * u.deg

@pytest.mark.parametrize(('major', 'minor', 'pa'), [(1.0, 0.5, 10),
                                                    (1.0, 'skip', 10),
                                                    (1.0, 0.5, 'skip')])
def test_load_beam_props(major, minor, pa):

    beam_header = header.copy()
    beam_header["BMAJ"] = major
    if minor != 'skip':
        beam_header["BMIN"] = minor
    if pa != 'skip':
        beam_header["BPA"] = pa

    bmaj, bmin, bpa = find_beam_properties(beam_header)

    assert bmaj == major * u.deg

    if minor == 'skip':
        assert bmin == major * u.deg
    else:
        assert bmin == minor * u.deg

    if pa == "skip":
        assert bpa == 0 * u.deg
    else:
        assert bpa == pa * u.deg

@pytest.mark.xfail(raises=(ValueError, TypeError))
def test_load_beam_fail():

    find_beam_width(header)
<|endoftext|>


<commit_before>from django.conf import settings
from django.db import models
from django.utils import timezone
from django.contrib.contenttypes.fields import GenericForeignKey

from django.contrib.contenttypes.models import ContentType

import jsonfield

from .signals import event_logged

class Log(models.Model):

    user = models.ForeignKey(
        getattr(settings, "AUTH_USER_MODEL", "auth.User"),
        null=True,
        on_delete=models.SET_NULL
    )
    timestamp = models.DateTimeField(default=timezone.now, db_index=True)
    action = models.CharField(max_length=50, db_index=True)
    content_type = models.ForeignKey(ContentType, null=True)
    object_id = models.PositiveIntegerField(null=True)
    obj = GenericForeignKey("content_type", "object_id")
    extra = jsonfield.JSONField()

    @property
    def template_fragment_name(self):
        return "eventlog/{}.html".format(self.action.lower())

    class Meta:
        ordering = ["-timestamp"]

def log(user, action, extra=None, obj=None, dateof=None):
    if (user is not None and not user.is_authenticated()):
        user = None
    if extra is None:
        extra = {}
    content_type = None
    object_id = None
    if obj is not None:
        content_type = ContentType.objects.get_for_model(obj)
        object_id = obj.pk
    event = Log.objects.create(
        user=user,
        action=action,
        extra=extra,
        content_type=content_type,
        object_id=object_id,
        timestamp=dateof
    )
    event_logged.send(sender=Log, event=event)
    return event
<commit_msg>default<commit_after>from django.conf import settings
from django.db import models
from django.utils import timezone
from django.contrib.contenttypes.fields import GenericForeignKey

from django.contrib.contenttypes.models import ContentType

import jsonfield
import datetime

from .signals import event_logged

class Log(models.Model):

    user = models.ForeignKey(
        getattr(settings, "AUTH_USER_MODEL", "auth.User"),
        null=True,
        on_delete=models.SET_NULL
    )
    timestamp = models.DateTimeField(default=timezone.now, db_index=True)
    action = models.CharField(max_length=50, db_index=True)
    content_type = models.ForeignKey(ContentType, null=True)
    object_id = models.PositiveIntegerField(null=True)
    obj = GenericForeignKey("content_type", "object_id")
    extra = jsonfield.JSONField()

    @property
    def template_fragment_name(self):
        return "eventlog/{}.html".format(self.action.lower())

    class Meta:
        ordering = ["-timestamp"]

def log(user, action, extra=None, obj=None, dateof=None):
    if (user is not None and not user.is_authenticated()):
        user = None
    if extra is None:
        extra = {}
    content_type = None
    object_id = None
    if obj is not None:
        content_type = ContentType.objects.get_for_model(obj)
        object_id = obj.pk
    if dateof is None:
        dateof = datetime.datetime.utcnow()

    event = Log.objects.create(
        user=user,
        action=action,
        extra=extra,
        content_type=content_type,
        object_id=object_id,
        timestamp=dateof
    )
    event_logged.send(sender=Log, event=event)
    return event
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
fabrik.api
----------------
This module implements the fabrik api.
"""

import time

from fabric.api import run
from fabric.state import env
from fabric.decorators import task, runs_once
from fabric.context_managers import cd
from fabric.contrib.files import exists

from utils import run_task
from hooks import run_hook, has_hook
import paths
from .logger import logger

def report(msg, err=None):
    logger.error(msg)

    if err:
        logger.error(err)

    if hasattr(env, "raise_errors") and env.raise_errors:
        raise Exception(msg)

@runs_once
def init_tasks():
    """
    Performs basic setup before any of the tasks are run. All tasks needs to
    run this before continuing. It only fires once.
    """

    # Make sure exist are set
    if "exists" not in env:
        env.exists = exists

    if "run" not in env:
        env.run = run

    if "cd" not in env:
        env.cd = cd

    if "max_releases" not in env:
        env.max_releases = 5

    if "public_path" in env:
        public_path = env.public_path.rstrip("/")
        env.public_path = public_path

    run_hook("init_tasks")

@task
def setup():
    """
    Creates shared and upload directory then fires setup to recipes.
    """

    init_tasks()

    run_hook("before_setup")

    # Create shared folder
    env.run("mkdir -p %s" % (paths.get_shared_path()))
    env.run("chmod 755 %s" % (paths.get_shared_path()))

    # Create backup folder
    env.run("mkdir -p %s" % (paths.get_backup_path()))
    env.run("chmod 750 %s" % (paths.get_backup_path()))

    # Create uploads folder
    env.run("mkdir -p %s" % (paths.get_upload_path()))
    env.run("chmod 775 %s" % (paths.get_upload_path()))

    run_hook("setup")
    run_hook("after_setup")

@task
def deploy():
    """
    Performs a deploy by invoking copy, then generating next release name and
    invoking necessary hooks.
    """

    init_tasks()

    if not has_hook("copy"):
        return report("No copy method has been defined")

    if not env.exists(paths.get_shared_path()):
        return report("You need to run setup before running deploy")

    run_hook("before_deploy")

    release_name = int(time.time()*1000)
    release_path = paths.get_releases_path(release_name)

    env.current_release = release_path

    try:
        run_hook("copy")
    except Exception as e:
        return report("Error occurred on copy. Aborting deploy", err=e)

    if not env.exists(paths.get_source_path(release_name)):
        return report("Source path not found '%s'" %
                      paths.get_source_path(release_name))

    try:
        run_hook("deploy")
    except Exception as e:
        message = "Error occurred on deploy, starting rollback..."

        logger.error(message)
        logger.error(e)

        run_task("rollback")
        return report("Error occurred on deploy")

    # Symlink current folder
    paths.symlink(paths.get_source_path(release_name),
                  paths.get_current_path())

    # Clean older releases
    if "max_releases" in env:
        cleanup_releases(int(env.max_releases))

    run_hook("after_deploy")

    if "public_path" in env:
        paths.symlink(paths.get_source_path(release_name), env.public_path)

    logger.info("Deploy complete")

@task
def rollback():
    """
    Rolls back to previous release
    """

    init_tasks()

    run_hook("before_rollback")

    # Remove current version
    current_release = paths.get_current_release_path()
    if current_release:
        env.run("rm -rf %s" % current_release)

    # Restore previous version
    old_release = paths.get_current_release_name()
    if old_release:
        paths.symlink(paths.get_source_path(old_release),
                      paths.get_current_path())

    run_hook("rollback")
    run_hook("after_rollback")

    logger.info("Rollback complete")

@task
def cleanup_releases(limit=5):
    """
    Removes older releases.
    """

    init_tasks()

    max_versions = limit + 1

    env.run("ls -dt %s/*/ | tail -n +%s | xargs rm -rf" % (
        paths.get_releases_path(),
        max_versions)
    )

@task
def debug():
    """
    Outputs debug information, needs to run before the task".

    Example:
        fab prod debug deploy.
    """

    from fabric.network import ssh

    init_tasks()

    ssh.util.log_to_file("fabrik-debug.log", 10)

@task
def test():
    """
    Prints environment information.
    """

    init_tasks()

    env.run("cat /etc/*-release")  # List linux dist info
<commit_msg>Made raise_errors behaviour True by default<commit_after># -*- coding: utf-8 -*-

"""
fabrik.api
----------------
This module implements the fabrik api.
"""

import time

from fabric.api import run
from fabric.state import env
from fabric.decorators import task, runs_once
from fabric.context_managers import cd
from fabric.contrib.files import exists

from utils import run_task
from hooks import run_hook, has_hook
import paths
from .logger import logger

def report(msg, err=None):
    logger.error(msg)

    if err:
        logger.error(err)

    if getattr(env, "raise_errors", default=True):
        raise Exception(msg)

@runs_once
def init_tasks():
    """
    Performs basic setup before any of the tasks are run. All tasks needs to
    run this before continuing. It only fires once.
    """

    # Make sure exist are set
    if "exists" not in env:
        env.exists = exists

    if "run" not in env:
        env.run = run

    if "cd" not in env:
        env.cd = cd

    if "max_releases" not in env:
        env.max_releases = 5

    if "public_path" in env:
        public_path = env.public_path.rstrip("/")
        env.public_path = public_path

    run_hook("init_tasks")

@task
def setup():
    """
    Creates shared and upload directory then fires setup to recipes.
    """

    init_tasks()

    run_hook("before_setup")

    # Create shared folder
    env.run("mkdir -p %s" % (paths.get_shared_path()))
    env.run("chmod 755 %s" % (paths.get_shared_path()))

    # Create backup folder
    env.run("mkdir -p %s" % (paths.get_backup_path()))
    env.run("chmod 750 %s" % (paths.get_backup_path()))

    # Create uploads folder
    env.run("mkdir -p %s" % (paths.get_upload_path()))
    env.run("chmod 775 %s" % (paths.get_upload_path()))

    run_hook("setup")
    run_hook("after_setup")

@task
def deploy():
    """
    Performs a deploy by invoking copy, then generating next release name and
    invoking necessary hooks.
    """

    init_tasks()

    if not has_hook("copy"):
        return report("No copy method has been defined")

    if not env.exists(paths.get_shared_path()):
        return report("You need to run setup before running deploy")

    run_hook("before_deploy")

    release_name = int(time.time()*1000)
    release_path = paths.get_releases_path(release_name)

    env.current_release = release_path

    try:
        run_hook("copy")
    except Exception as e:
        return report("Error occurred on copy. Aborting deploy", err=e)

    if not env.exists(paths.get_source_path(release_name)):
        return report("Source path not found '%s'" %
                      paths.get_source_path(release_name))

    try:
        run_hook("deploy")
    except Exception as e:
        message = "Error occurred on deploy, starting rollback..."

        logger.error(message)
        logger.error(e)

        run_task("rollback")
        return report("Error occurred on deploy")

    # Symlink current folder
    paths.symlink(paths.get_source_path(release_name),
                  paths.get_current_path())

    # Clean older releases
    if "max_releases" in env:
        cleanup_releases(int(env.max_releases))

    run_hook("after_deploy")

    if "public_path" in env:
        paths.symlink(paths.get_source_path(release_name), env.public_path)

    logger.info("Deploy complete")

@task
def rollback():
    """
    Rolls back to previous release
    """

    init_tasks()

    run_hook("before_rollback")

    # Remove current version
    current_release = paths.get_current_release_path()
    if current_release:
        env.run("rm -rf %s" % current_release)

    # Restore previous version
    old_release = paths.get_current_release_name()
    if old_release:
        paths.symlink(paths.get_source_path(old_release),
                      paths.get_current_path())

    run_hook("rollback")
    run_hook("after_rollback")

    logger.info("Rollback complete")

@task
def cleanup_releases(limit=5):
    """
    Removes older releases.
    """

    init_tasks()

    max_versions = limit + 1

    env.run("ls -dt %s/*/ | tail -n +%s | xargs rm -rf" % (
        paths.get_releases_path(),
        max_versions)
    )

@task
def debug():
    """
    Outputs debug information, needs to run before the task".

    Example:
        fab prod debug deploy.
    """

    from fabric.network import ssh

    init_tasks()

    ssh.util.log_to_file("fabrik-debug.log", 10)

@task
def test():
    """
    Prints environment information.
    """

    init_tasks()

    env.run("cat /etc/*-release")  # List linux dist info
<|endoftext|>


<commit_before># pip install PyMySQL

import pymysql
pymysql.install_as_MySQLdb()<commit_msg>fix mysql import for python2<commit_after># pip install PyMySQL

import sys
if sys.version_info.major == 3:
    import pymysql
    pymysql.install_as_MySQLdb()<|endoftext|>


<commit_before>from thinglang.compiler.buffer import CompilationBuffer
from thinglang.compiler.context import CompilationContext
from thinglang.compiler.opcodes import OpcodeAssignStatic, OpcodeAssignLocal, OpcodePopLocal, OpcodePopMember
from thinglang.lexer.operators.assignment import LexicalAssignment
from thinglang.lexer.values.identifier import Identifier
from thinglang.parser.constructs.cast_operation import CastOperation
from thinglang.parser.nodes.base_node import BaseNode
from thinglang.parser.rule import ParserRule
from thinglang.parser.values.access import Access
from thinglang.utils.type_descriptors import ValueType

class AssignmentOperation(BaseNode):
    """
    Represents all flavors of assignment operations, static-to-local, static-to-member, local-to-local, member-to-member, etc...
    """

    DECELERATION = object()  # This assignment declares a local variable, and assigns an initial value to it
    REASSIGNMENT = object()  # This assignment refers to an existing local variable or to a member variable

    def __init__(self, intent, name, value, type_name=None):
        super(AssignmentOperation, self).__init__([name, value, type_name])
        self.intent, self.name, self.value = intent, name, value

        if type_name is not None:
            assert self.name.type is None
            self.name.type = type_name

    def compile(self, context: CompilationBuffer):

        is_local = isinstance(self.name, Identifier)

        if is_local:
            target = context.resolve(self.name)

            if self.value.STATIC:
                ref = context.append_static(self.value.serialize())
                return context.append(OpcodeAssignStatic.from_reference(target, ref), self.source_ref)
            elif isinstance(self.value, Identifier):
                ref = context.resolve(self.value)
                return context.append(OpcodeAssignLocal.from_references(target, ref), self.source_ref)

        value_ref = self.value.compile(context)
        target = self.name

        cast_placeholder = context.current_index + 1

        if isinstance(target, Identifier):
            target_ref = context.resolve(target)
            context.append(OpcodePopLocal.from_reference(target_ref), self.source_ref)
        elif isinstance(target, Access):
            if target.extensions:
                target_ref = target.compile(context, pop_last=True)
            else:
                target_ref = context.resolve(target)
                context.append(OpcodePopMember.from_reference(target_ref), self.source_ref)
        else:
            raise Exception('Cannot pull up {}'.format(target))

        if value_ref.type.untyped != target_ref.type.untyped:
            optional = context.optional()
            CastOperation.create(source=value_ref.type, destination=target_ref.type).deriving_from(self).compile(optional)
            context.insert(cast_placeholder, optional)

    @property
    def type(self):
        return self.name.type

    def __repr__(self):
        return f'Assignment({self.name} = {self.value})'

    ASSIGNMENT_TARGET = Identifier, Access

    @staticmethod
    @ParserRule.mark
    def assignment_declaration(type_name: Identifier, name: Identifier, _: LexicalAssignment, value: ValueType):
        return AssignmentOperation(AssignmentOperation.DECELERATION, name, value, type_name)

    @staticmethod
    @ParserRule.mark
    def assignment_reassignment(name: ASSIGNMENT_TARGET, _: LexicalAssignment, value: ValueType):
        return AssignmentOperation(AssignmentOperation.REASSIGNMENT, name, value)
<commit_msg>Add in place modifiers such as +=, -= etc...<commit_after>from thinglang.compiler.buffer import CompilationBuffer
from thinglang.compiler.context import CompilationContext
from thinglang.compiler.opcodes import OpcodeAssignStatic, OpcodeAssignLocal, OpcodePopLocal, OpcodePopMember
from thinglang.lexer.operators.assignment import LexicalAssignment
from thinglang.lexer.operators.binary import LexicalBinaryOperation
from thinglang.lexer.values.identifier import Identifier
from thinglang.parser.constructs.cast_operation import CastOperation
from thinglang.parser.nodes.base_node import BaseNode
from thinglang.parser.rule import ParserRule
from thinglang.parser.values.access import Access
from thinglang.parser.values.binary_operation import BinaryOperation
from thinglang.utils.type_descriptors import ValueType

class AssignmentOperation(BaseNode):
    """
    Represents all flavors of assignment operations, static-to-local, static-to-member, local-to-local, member-to-member, etc...
    """

    DECELERATION = object()  # This assignment declares a local variable, and assigns an initial value to it
    REASSIGNMENT = object()  # This assignment refers to an existing local variable or to a member variable

    def __init__(self, intent, name, value, type_name=None):
        super(AssignmentOperation, self).__init__([name, value, type_name])
        self.intent, self.name, self.value = intent, name, value

        if type_name is not None:
            assert self.name.type is None
            self.name.type = type_name

    def compile(self, context: CompilationBuffer):

        is_local = isinstance(self.name, Identifier)

        if is_local:
            target = context.resolve(self.name)

            if self.value.STATIC:
                ref = context.append_static(self.value.serialize())
                return context.append(OpcodeAssignStatic.from_reference(target, ref), self.source_ref)
            elif isinstance(self.value, Identifier):
                ref = context.resolve(self.value)
                return context.append(OpcodeAssignLocal.from_references(target, ref), self.source_ref)

        value_ref = self.value.compile(context)
        target = self.name

        cast_placeholder = context.current_index + 1

        if isinstance(target, Identifier):
            target_ref = context.resolve(target)
            context.append(OpcodePopLocal.from_reference(target_ref), self.source_ref)
        elif isinstance(target, Access):
            if target.extensions:
                target_ref = target.compile(context, pop_last=True)
            else:
                target_ref = context.resolve(target)
                context.append(OpcodePopMember.from_reference(target_ref), self.source_ref)
        else:
            raise Exception('Cannot pull up {}'.format(target))

        if value_ref.type.untyped != target_ref.type.untyped:
            optional = context.optional()
            CastOperation.create(source=value_ref.type, destination=target_ref.type).deriving_from(self).compile(optional)
            context.insert(cast_placeholder, optional)

    @property
    def type(self):
        return self.name.type

    def __repr__(self):
        return f'Assignment({self.name} = {self.value})'

    ASSIGNMENT_TARGET = Identifier, Access

    @staticmethod
    @ParserRule.mark
    def assignment_declaration(type_name: Identifier, name: Identifier, _: LexicalAssignment, value: ValueType):
        return AssignmentOperation(AssignmentOperation.DECELERATION, name, value, type_name)

    @staticmethod
    @ParserRule.mark
    def assignment_reassignment(name: ASSIGNMENT_TARGET, _: LexicalAssignment, value: ValueType):
        return AssignmentOperation(AssignmentOperation.REASSIGNMENT, name, value)

    @staticmethod
    @ParserRule.mark
    def in_place_modifier(name: ASSIGNMENT_TARGET, operator: LexicalBinaryOperation, _: LexicalAssignment, val: ValueType):
        return AssignmentOperation(AssignmentOperation.DECELERATION, name, BinaryOperation(operator, name, val))
<|endoftext|>