<commit_before>#!/usr/bin/python

from cmd import Cmd
import traceback
import subprocess
import sys

class CLI(Cmd):
    prompt = "MaxiNet> "

    helpStr = """You may also type '<hostname> <command>' to execute commands on maxinet hosts
\texample: h1 ifconfig h1-eth1"""

    def __init__(self,experiment,plocals,pglobals):
        self.experiment = experiment
        self.plocals = plocals
        self.pglobals = pglobals
        Cmd.__init__(self)
        while True:
            try:
                self.cmdloop()
                break
            except KeyboardInterrupt:
                print "\nInterrupt"

    def emptyline(self):
        pass

    def do_help(self,s):
        "Print help"
        Cmd.do_help(self,s)
        if s is "":
            print self.helpStr

    def do_hosts(self,s):
        "Print all hostnames"
        h=""
        for host in self.experiment.hosts:
            h=h+" "+host.name
        print h

    def do_switches(self,s):
        "Print all switchnames"
        h=""
        for switch in self.experiment.switches:
            h=h+" "+self.experiment.get(switch).name
        print h

    def do_pingall(self,s):
        "Do ping between all hosts (or between one and all other hosts if host is given as parameter)"
        sent=0.0
        received=0.0
        if(len(s)==0):
            for host in self.experiment.hosts:
                for target in self.experiment.hosts:
                    if(target==host):
                        continue
                    sys.stdout.write(host.name +" -> "+ target.name)
                    sent+=1.0
                    if(host.pexec("ping -c1 "+target.IP())[2]!=0):
                        print " X"
                    else:
                        received +=1.0
                        print ""
        else:
            host = self.experiment.get(s)
            if(host==None):
                print "Error: Node "+s+" does not exist"
            else:
                for target in self.experiment.hosts:
                    if(target==host):
                        continue
                    sys.stdout.write(host.name +" -> "+ target.name)
                    sent+=1.0
                    if(host.pexec("ping -c1 "+target.IP())[2]!=0):
                        print " X"
                    else:
                        received+=1.0
                        print ""
        print "*** Results: %.2f%% dropped (%d/%d received)" % ((1.0-received/sent), int(received), int(sent))

    def do_ip(self,s):
        "Print ip of host"
        node = self.experiment.get(s)
        if(node==None):
            print "Error: Node "+s+" does not exist"
        else:
            print node.IP()

    def do_py(self,s):
        "Execute Python command"
        cmd = s
        main = __import__("__main__")
        try:
            exec(cmd, self.pglobals, self.plocals)
        except Exception, e:
            traceback.print_exc()

    def do_exit(self,s):
        "Exit"
        return "exited by user command"

    def do_quit(self,s):
        "Exit"
        return self.do_exit(s)

    def default(self,s):
        node = s[:s.find(" ")]
        cmd = s[s.find(" ")+1:]
        if(self.experiment.get(node)==None):
            print "Error: Node "+s+" does not exist"
        else:
            pid = self.experiment.get_worker(self.experiment.get(node)).run_cmd("ps ax | grep \"bash -ms mininet:"+node+"\" -m1 | awk '{print $1}'").strip()
            if self.experiment.get_worker(self.experiment.get(node)).tunnelX11(node):
                rcmd = "ssh -q -X -t "+self.experiment.get_worker(self.experiment.get(node)).hn()+" sudo mnexec -a "+pid+" "+cmd
            else:
                rcmd = "ssh -q -t "+self.experiment.get_worker(self.experiment.get(node)).hn()+" sudo mnexec -a "+pid+" "+cmd
            subprocess.call(rcmd,shell=True)
<commit_msg>set xauthority of root to maxinet user<commit_after>#!/usr/bin/python

from cmd import Cmd
import traceback
import subprocess
import sys

class CLI(Cmd):
    prompt = "MaxiNet> "

    helpStr = """You may also type '<hostname> <command>' to execute commands on maxinet hosts
\texample: h1 ifconfig h1-eth1"""

    def __init__(self,experiment,plocals,pglobals):
        self.experiment = experiment
        self.plocals = plocals
        self.pglobals = pglobals
        Cmd.__init__(self)
        while True:
            try:
                self.cmdloop()
                break
            except KeyboardInterrupt:
                print "\nInterrupt"

    def emptyline(self):
        pass

    def do_help(self,s):
        "Print help"
        Cmd.do_help(self,s)
        if s is "":
            print self.helpStr

    def do_hosts(self,s):
        "Print all hostnames"
        h=""
        for host in self.experiment.hosts:
            h=h+" "+host.name
        print h

    def do_switches(self,s):
        "Print all switchnames"
        h=""
        for switch in self.experiment.switches:
            h=h+" "+self.experiment.get(switch).name
        print h

    def do_pingall(self,s):
        "Do ping between all hosts (or between one and all other hosts if host is given as parameter)"
        sent=0.0
        received=0.0
        if(len(s)==0):
            for host in self.experiment.hosts:
                for target in self.experiment.hosts:
                    if(target==host):
                        continue
                    sys.stdout.write(host.name +" -> "+ target.name)
                    sent+=1.0
                    if(host.pexec("ping -c1 "+target.IP())[2]!=0):
                        print " X"
                    else:
                        received +=1.0
                        print ""
        else:
            host = self.experiment.get(s)
            if(host==None):
                print "Error: Node "+s+" does not exist"
            else:
                for target in self.experiment.hosts:
                    if(target==host):
                        continue
                    sys.stdout.write(host.name +" -> "+ target.name)
                    sent+=1.0
                    if(host.pexec("ping -c1 "+target.IP())[2]!=0):
                        print " X"
                    else:
                        received+=1.0
                        print ""
        print "*** Results: %.2f%% dropped (%d/%d received)" % ((1.0-received/sent), int(received), int(sent))

    def do_ip(self,s):
        "Print ip of host"
        node = self.experiment.get(s)
        if(node==None):
            print "Error: Node "+s+" does not exist"
        else:
            print node.IP()

    def do_py(self,s):
        "Execute Python command"
        cmd = s
        main = __import__("__main__")
        try:
            exec(cmd, self.pglobals, self.plocals)
        except Exception, e:
            traceback.print_exc()

    def do_exit(self,s):
        "Exit"
        return "exited by user command"

    def do_quit(self,s):
        "Exit"
        return self.do_exit(s)

    def default(self,s):
        node = s[:s.find(" ")]
        cmd = s[s.find(" ")+1:]
        if(self.experiment.get(node)==None):
            print "Error: Node "+s+" does not exist"
        else:
            pid = self.experiment.get_worker(self.experiment.get(node)).run_cmd("ps ax | grep \"bash -ms mininet:"+node+"\" -m1 | awk '{print $1}'").strip()
            if self.experiment.get_worker(self.experiment.get(node)).tunnelX11(node):
                user = subprocess.check_output("ssh -q -t "+self.experiment.get_worker(self.experiment.get(node)).hn()+" echo $USER", shell=True).strip()
                rcmd = "ssh -q -X -t "+self.experiment.get_worker(self.experiment.get(node)).hn()+" sudo  XAUTHORITY=/home/"+user+"/.Xauthority mnexec -a "+pid+" "+cmd
            else:
                rcmd = "ssh -q -t "+self.experiment.get_worker(self.experiment.get(node)).hn()+" sudo mnexec -a "+pid+" "+cmd
            subprocess.call(rcmd,shell=True)
<|endoftext|>


<commit_before>from .util import collect, dshape
from .internal_utils import remove
from .coretypes import *

# https://github.com/ContinuumIO/datashape/blob/master/docs/source/types.rst

__all__ = ['isdimension', 'ishomogeneous', 'istabular', 'isfixed', 'isscalar']

dimension_types = (Fixed, Var, Ellipsis, int)

def isunit(ds):
    """ Is this dshape a single dtype?

    >>> isunit('int')
    True
    >>> isunit('?int')
    True
    >>> isunit('{name: string, amount: int}')
    False
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]
    if isinstance(ds, Option):
        ds = ds.ty
    return isinstance(ds, Unit)

def isrecord(ds):
    """ Is this dshape a record type?

    >>> isrecord('{name: string, amount: int}')
    True
    >>> isrecord('int')
    False
    >>> isrecord('?{name: string, amount: int}')
    True
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]
    if isinstance(ds, Option):
        ds = ds.ty
    return isinstance(ds, Record)

def isdimension(ds):
    """ Is a component a dimension?

    >>> isdimension(Fixed(10))
    True
    >>> isdimension(Var())
    True
    >>> isdimension(int32)
    False
    """
    return isinstance(ds, dimension_types)

def ishomogeneous(ds):
    """ Does datashape contain only one dtype?

    >>> ishomogeneous(int32)
    True
    >>> ishomogeneous('var * 3 * string')
    True
    >>> ishomogeneous('var * {name: string, amount: int}')
    False
    """
    ds = dshape(ds)
    return len(set(remove(isdimension, collect(isunit, ds)))) == 1

def _dimensions(ds):
    """ Number of dimensions of datashape

    Interprets records as dimensional

    >>> _dimensions(int32)
    0
    >>> _dimensions(10 * int32)
    1
    >>> _dimensions('var * 10 * int')
    2
    >>> _dimensions('var * {name: string, amount: int}')
    2
    """
    ds = dshape(ds)
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]
    if isinstance(ds, Option):
        return _dimensions(ds.ty)
    if isinstance(ds, Record):
        return 1 + max(map(_dimensions, ds.types))
    if isinstance(ds, DataShape) and isdimension(ds[0]):
        return 1 + _dimensions(ds.subshape[0])
    if isunit(ds):
        return 0
    raise NotImplementedError('Can not compute dimensions for %s' % ds)

def isfixed(ds):
    """ Contains no variable dimensions

    >>> isfixed('10 * int')
    True
    >>> isfixed('var * int')
    False
    >>> isfixed('10 * {name: string, amount: int}')
    True
    >>> isfixed('10 * {name: string, amounts: var * int}')
    False
    """
    ds = dshape(ds)
    if isinstance(ds[0], TypeVar):
        return None  # don't know
    if isinstance(ds[0], Var):
        return False
    if isinstance(ds[0], Record):
        return all(map(isfixed, ds[0].types))
    if len(ds) > 1:
        return isfixed(ds.subarray(1))
    return True

def istabular(ds):
    """ Can be represented by a two dimensional with fixed columns

    >>> istabular('var * 3 * int')
    True
    >>> istabular('var * {name: string, amount: int}')
    True
    >>> istabular('var * 10 * 3 * int')
    False
    >>> istabular('10 * var * int')
    False
    """
    ds = dshape(ds)
    return _dimensions(ds) == 2 and isfixed(ds.subarray(1))

def isscalar(ds):
    """ Has no dimensions

    >>> isscalar('int')
    True
    >>> isscalar('3 * int')
    False
    >>> isscalar('{name: string, amount: int}')
    True
    """
    ds = dshape(ds)
    return not ds.shape
<commit_msg>add iscollection predicate<commit_after>from .util import collect, dshape
from .internal_utils import remove
from .coretypes import *

# https://github.com/ContinuumIO/datashape/blob/master/docs/source/types.rst

__all__ = ['isdimension', 'ishomogeneous', 'istabular', 'isfixed', 'isscalar']

dimension_types = (Fixed, Var, Ellipsis, int)

def isunit(ds):
    """ Is this dshape a single dtype?

    >>> isunit('int')
    True
    >>> isunit('?int')
    True
    >>> isunit('{name: string, amount: int}')
    False
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]
    if isinstance(ds, Option):
        ds = ds.ty
    return isinstance(ds, Unit)

def isrecord(ds):
    """ Is this dshape a record type?

    >>> isrecord('{name: string, amount: int}')
    True
    >>> isrecord('int')
    False
    >>> isrecord('?{name: string, amount: int}')
    True
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]
    if isinstance(ds, Option):
        ds = ds.ty
    return isinstance(ds, Record)

def isdimension(ds):
    """ Is a component a dimension?

    >>> isdimension(Fixed(10))
    True
    >>> isdimension(Var())
    True
    >>> isdimension(int32)
    False
    """
    return isinstance(ds, dimension_types)

def ishomogeneous(ds):
    """ Does datashape contain only one dtype?

    >>> ishomogeneous(int32)
    True
    >>> ishomogeneous('var * 3 * string')
    True
    >>> ishomogeneous('var * {name: string, amount: int}')
    False
    """
    ds = dshape(ds)
    return len(set(remove(isdimension, collect(isunit, ds)))) == 1

def _dimensions(ds):
    """ Number of dimensions of datashape

    Interprets records as dimensional

    >>> _dimensions(int32)
    0
    >>> _dimensions(10 * int32)
    1
    >>> _dimensions('var * 10 * int')
    2
    >>> _dimensions('var * {name: string, amount: int}')
    2
    """
    ds = dshape(ds)
    if isinstance(ds, str):
        ds = dshape(ds)
    if isinstance(ds, DataShape) and len(ds) == 1:
        ds = ds[0]
    if isinstance(ds, Option):
        return _dimensions(ds.ty)
    if isinstance(ds, Record):
        return 1 + max(map(_dimensions, ds.types))
    if isinstance(ds, DataShape) and isdimension(ds[0]):
        return 1 + _dimensions(ds.subshape[0])
    if isunit(ds):
        return 0
    raise NotImplementedError('Can not compute dimensions for %s' % ds)

def isfixed(ds):
    """ Contains no variable dimensions

    >>> isfixed('10 * int')
    True
    >>> isfixed('var * int')
    False
    >>> isfixed('10 * {name: string, amount: int}')
    True
    >>> isfixed('10 * {name: string, amounts: var * int}')
    False
    """
    ds = dshape(ds)
    if isinstance(ds[0], TypeVar):
        return None  # don't know
    if isinstance(ds[0], Var):
        return False
    if isinstance(ds[0], Record):
        return all(map(isfixed, ds[0].types))
    if len(ds) > 1:
        return isfixed(ds.subarray(1))
    return True

def istabular(ds):
    """ Can be represented by a two dimensional with fixed columns

    >>> istabular('var * 3 * int')
    True
    >>> istabular('var * {name: string, amount: int}')
    True
    >>> istabular('var * 10 * 3 * int')
    False
    >>> istabular('10 * var * int')
    False
    """
    ds = dshape(ds)
    return _dimensions(ds) == 2 and isfixed(ds.subarray(1))

def isscalar(ds):
    """ Has no dimensions

    >>> isscalar('int')
    True
    >>> isscalar('3 * int')
    False
    >>> isscalar('{name: string, amount: int}')
    True
    """
    ds = dshape(ds)
    return not ds.shape

def iscollection(ds):
    """ Is a collection of items, has dimension

    >>> iscollection('5 * int32')
    True
    >>> iscollection('int32')
    False
    """
    if isinstance(ds, str):
        ds = dshape(ds)
    return isdimension(ds[0])
<|endoftext|>


<commit_before>from __future__ import print_function
from autobahn.twisted.websocket import WebSocketClientProtocol, WebSocketClientFactory
from twisted.internet.protocol import ReconnectingClientFactory

class LiveStream(WebSocketClientProtocol):  # pragma: no cover
    """ Internal class used to call the websocket callbacks. """

    def onOpen(self):
        self.factory.datasift['send_message'] = self.sendMessage
        self.factory.datasift['on_open']()

    def onClose(self, wasClean, code, reason):
        self.factory.datasift['on_close'](wasClean, code, reason)

    def onMessage(self, msg, binary):
        self.factory.datasift['on_message'](msg, binary)
    def onPing(self, payload):
        self.recieved = datetime.now()
        #self.sendPong(payload=payload)

class LiveStreamFactory(ReconnectingClientFactory, WebSocketClientFactory):  # pragma: no cover
    """ Internal class used to implement the WebSocketClientFactory used in :class:`~datasift.client.Client` with reconnection"""

    protocol = LiveStream

    maxDelay = 320
    delay = 1

    def startedConnecting(self, connector):
        pass

    def clientConnectionLost(self, connector, reason):
        ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
<commit_msg>Added comment on why it looks broken<commit_after>from __future__ import print_function
from autobahn.twisted.websocket import WebSocketClientProtocol, WebSocketClientFactory
from twisted.internet.protocol import ReconnectingClientFactory

class LiveStream(WebSocketClientProtocol):  # pragma: no cover
    """ Internal class used to call the websocket callbacks. """

    def onOpen(self):
        self.factory.datasift['send_message'] = self.sendMessage
        self.factory.datasift['on_open']()

    def onClose(self, wasClean, code, reason):
        self.factory.datasift['on_close'](wasClean, code, reason)

    def onMessage(self, msg, binary):
        self.factory.datasift['on_message'](msg, binary)

    def onPing(self, payload):
        self.recieved = datetime.now()
        # Intentionally commented out for the beta, revert when suitable
        #self.sendPong(payload=payload)

class LiveStreamFactory(ReconnectingClientFactory, WebSocketClientFactory):  # pragma: no cover
    """ Internal class used to implement the WebSocketClientFactory used in :class:`~datasift.client.Client` with reconnection"""

    protocol = LiveStream

    maxDelay = 320
    delay = 1

    def startedConnecting(self, connector):
        pass

    def clientConnectionLost(self, connector, reason):
        ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Wrapper around Server Density API
=================================

.. versionadded:: 2014.7.0
'''

# Import Python libs
from __future__ import absolute_import
import json
import logging

# Import 3rd-party libs
import requests
import salt.ext.six as six
from salt.ext.six.moves import map  # pylint: disable=import-error,no-name-in-module,redefined-builtin

from salt.exceptions import CommandExecutionError

log = logging.getLogger(__name__)

def get_sd_auth(val, sd_auth_pillar_name='serverdensity'):
    '''
    Returns requested Server Density authentication value from pillar.

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.get_sd_auth <val>
    '''
    sd_pillar = __pillar__.get(sd_auth_pillar_name)
    log.debug('Server Density Pillar: {0}'.format(sd_pillar))
    if not sd_pillar:
        log.error('Cloud not load {0} pillar'.format(sd_auth_pillar_name))
        raise CommandExecutionError(
            '{0} pillar is required for authentication'.format(sd_auth_pillar_name)
        )

    try:
        return sd_pillar[val]
    except KeyError:
        log.error('Cloud not find value {0} in pillar'.format(val))
        raise CommandExecutionError('{0} value was not found in pillar'.format(val))

def _clean_salt_variables(params, variable_prefix="__"):
    '''
    Pops out variables from params which starts with `variable_prefix`.
    '''
    list(list(map(params.pop, [k for k in params if k.startswith(variable_prefix)])))
    return params

def create(name, **params):
    '''
    Function to create device in Server Density. For more info, see the `API
    docs`__.

    .. __: https://apidocs.serverdensity.com/Inventory/Devices/Creating

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.create lama
        salt '*' serverdensity_device.create rich_lama group=lama_band installedRAM=32768
    '''
    log.debug('Server Density params: {0}'.format(params))
    params = _clean_salt_variables(params)

    params['name'] = name
    api_response = requests.post(
        'https://api.serverdensity.io/inventory/devices/',
        params={'token': get_sd_auth('api_token')},
        data=params
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error('Could not parse API Response content: {0}'.format(api_response.content))
            raise CommandExecutionError(
                'Failed to create, API Response: {0}'.format(api_response)
            )
    else:
        return None

def delete(device_id):
    '''
    Delete a device from Server Density. For more information, see the `API
    docs`__.

    .. __: https://apidocs.serverdensity.com/Inventory/Devices/Deleting

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.delete 51f7eafcdba4bb235e000ae4
    '''
    api_response = requests.delete(
        'https://api.serverdensity.io/inventory/devices/' + device_id,
        params={'token': get_sd_auth('api_token')}
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error('Could not parse API Response content: {0}'.format(api_response.content))
            raise CommandExecutionError(
                'Failed to create, API Response: {0}'.format(api_response)
            )
    else:
        return None

def ls(**params):
    '''
    List devices in Server Density

    Results will be filtered by any params passed to this function. For more
    information, see the API docs on listing_ and searching_.

    .. _listing: https://apidocs.serverdensity.com/Inventory/Devices/Listing
    .. _searching: https://apidocs.serverdensity.com/Inventory/Devices/Searching

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.ls
        salt '*' serverdensity_device.ls name=lama
        salt '*' serverdensity_device.ls name=lama group=lama_band installedRAM=32768
    '''
    params = _clean_salt_variables(params)

    endpoint = 'devices'

    # Change endpoint if there are params to filter by:
    if params:
        endpoint = 'resources'

    # Convert all ints to strings:
    for key, val in six.iteritems(params):
        params[key] = str(val)

    api_response = requests.get(
        'https://api.serverdensity.io/inventory/{0}'.format(endpoint),
        params={'token': get_sd_auth('api_token'), 'filter': json.dumps(params)}
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error(
                'Could not parse Server Density API Response content: {0}'
                .format(api_response.content)
            )
            raise CommandExecutionError(
                'Failed to create, Server Density API Response: {0}'
                .format(api_response)
            )
    else:
        return None

def update(device_id, **params):
    '''
    Updates device information in Server Density. For more information see the
    `API docs`__.

    .. __: https://apidocs.serverdensity.com/Inventory/Devices/Updating

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.update 51f7eafcdba4bb235e000ae4 name=lama group=lama_band
        salt '*' serverdensity_device.update 51f7eafcdba4bb235e000ae4 name=better_lama group=rock_lamas swapSpace=512
    '''
    params = _clean_salt_variables(params)

    api_response = requests.put(
        'https://api.serverdensity.io/inventory/devices/' + device_id,
        params={'token': get_sd_auth('api_token')},
        data=params
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error(
                'Could not parse Server Density API Response content: {0}'
                .format(api_response.content)
            )
            raise CommandExecutionError(
                'Failed to create, API Response: {0}'.format(api_response)
            )
    else:
        return None

def install_agent(agent_key):
    '''
    Function downloads Server Density installation agent, and installs sd-agent
    with agent_key.

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.install_agent c2bbdd6689ff46282bdaa07555641498
    '''
    work_dir = '/tmp/'
    account_url = get_sd_auth('account_url')

    __salt__['cmd.run'](
        cmd='curl https://www.serverdensity.com/downloads/agent-install.sh -o install.sh',
        cwd=work_dir
    )
    __salt__['cmd.run'](cmd='chmod +x install.sh', cwd=work_dir)

    return __salt__['cmd.run'](
        cmd='./install.sh -a {account_url} -k {agent_key}'.format(
            account_url=account_url, agent_key=agent_key),
        cwd=work_dir
    )
<commit_msg>typo fix in log s/Cloud/Could/<commit_after># -*- coding: utf-8 -*-
'''
Wrapper around Server Density API
=================================

.. versionadded:: 2014.7.0
'''

# Import Python libs
from __future__ import absolute_import
import json
import logging

# Import 3rd-party libs
import requests
import salt.ext.six as six
from salt.ext.six.moves import map  # pylint: disable=import-error,no-name-in-module,redefined-builtin

from salt.exceptions import CommandExecutionError

log = logging.getLogger(__name__)

def get_sd_auth(val, sd_auth_pillar_name='serverdensity'):
    '''
    Returns requested Server Density authentication value from pillar.

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.get_sd_auth <val>
    '''
    sd_pillar = __pillar__.get(sd_auth_pillar_name)
    log.debug('Server Density Pillar: {0}'.format(sd_pillar))
    if not sd_pillar:
        log.error('Could not load {0} pillar'.format(sd_auth_pillar_name))
        raise CommandExecutionError(
            '{0} pillar is required for authentication'.format(sd_auth_pillar_name)
        )

    try:
        return sd_pillar[val]
    except KeyError:
        log.error('Could not find value {0} in pillar'.format(val))
        raise CommandExecutionError('{0} value was not found in pillar'.format(val))

def _clean_salt_variables(params, variable_prefix="__"):
    '''
    Pops out variables from params which starts with `variable_prefix`.
    '''
    list(list(map(params.pop, [k for k in params if k.startswith(variable_prefix)])))
    return params

def create(name, **params):
    '''
    Function to create device in Server Density. For more info, see the `API
    docs`__.

    .. __: https://apidocs.serverdensity.com/Inventory/Devices/Creating

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.create lama
        salt '*' serverdensity_device.create rich_lama group=lama_band installedRAM=32768
    '''
    log.debug('Server Density params: {0}'.format(params))
    params = _clean_salt_variables(params)

    params['name'] = name
    api_response = requests.post(
        'https://api.serverdensity.io/inventory/devices/',
        params={'token': get_sd_auth('api_token')},
        data=params
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error('Could not parse API Response content: {0}'.format(api_response.content))
            raise CommandExecutionError(
                'Failed to create, API Response: {0}'.format(api_response)
            )
    else:
        return None

def delete(device_id):
    '''
    Delete a device from Server Density. For more information, see the `API
    docs`__.

    .. __: https://apidocs.serverdensity.com/Inventory/Devices/Deleting

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.delete 51f7eafcdba4bb235e000ae4
    '''
    api_response = requests.delete(
        'https://api.serverdensity.io/inventory/devices/' + device_id,
        params={'token': get_sd_auth('api_token')}
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error('Could not parse API Response content: {0}'.format(api_response.content))
            raise CommandExecutionError(
                'Failed to create, API Response: {0}'.format(api_response)
            )
    else:
        return None

def ls(**params):
    '''
    List devices in Server Density

    Results will be filtered by any params passed to this function. For more
    information, see the API docs on listing_ and searching_.

    .. _listing: https://apidocs.serverdensity.com/Inventory/Devices/Listing
    .. _searching: https://apidocs.serverdensity.com/Inventory/Devices/Searching

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.ls
        salt '*' serverdensity_device.ls name=lama
        salt '*' serverdensity_device.ls name=lama group=lama_band installedRAM=32768
    '''
    params = _clean_salt_variables(params)

    endpoint = 'devices'

    # Change endpoint if there are params to filter by:
    if params:
        endpoint = 'resources'

    # Convert all ints to strings:
    for key, val in six.iteritems(params):
        params[key] = str(val)

    api_response = requests.get(
        'https://api.serverdensity.io/inventory/{0}'.format(endpoint),
        params={'token': get_sd_auth('api_token'), 'filter': json.dumps(params)}
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error(
                'Could not parse Server Density API Response content: {0}'
                .format(api_response.content)
            )
            raise CommandExecutionError(
                'Failed to create, Server Density API Response: {0}'
                .format(api_response)
            )
    else:
        return None

def update(device_id, **params):
    '''
    Updates device information in Server Density. For more information see the
    `API docs`__.

    .. __: https://apidocs.serverdensity.com/Inventory/Devices/Updating

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.update 51f7eafcdba4bb235e000ae4 name=lama group=lama_band
        salt '*' serverdensity_device.update 51f7eafcdba4bb235e000ae4 name=better_lama group=rock_lamas swapSpace=512
    '''
    params = _clean_salt_variables(params)

    api_response = requests.put(
        'https://api.serverdensity.io/inventory/devices/' + device_id,
        params={'token': get_sd_auth('api_token')},
        data=params
    )
    log.debug('Server Density API Response: {0}'.format(api_response))
    log.debug('Server Density API Response content: {0}'.format(api_response.content))
    if api_response.status_code == 200:
        try:
            return json.loads(api_response.content)
        except ValueError:
            log.error(
                'Could not parse Server Density API Response content: {0}'
                .format(api_response.content)
            )
            raise CommandExecutionError(
                'Failed to create, API Response: {0}'.format(api_response)
            )
    else:
        return None

def install_agent(agent_key):
    '''
    Function downloads Server Density installation agent, and installs sd-agent
    with agent_key.

    CLI Example:

    .. code-block:: bash

        salt '*' serverdensity_device.install_agent c2bbdd6689ff46282bdaa07555641498
    '''
    work_dir = '/tmp/'
    account_url = get_sd_auth('account_url')

    __salt__['cmd.run'](
        cmd='curl https://www.serverdensity.com/downloads/agent-install.sh -o install.sh',
        cwd=work_dir
    )
    __salt__['cmd.run'](cmd='chmod +x install.sh', cwd=work_dir)

    return __salt__['cmd.run'](
        cmd='./install.sh -a {account_url} -k {agent_key}'.format(
            account_url=account_url, agent_key=agent_key),
        cwd=work_dir
    )
<|endoftext|>


<commit_before>
from key import Key

def _object_getattr(obj, field):
  '''Attribute getter for the objects to operate on.

  This function can be overridden in classes or instances of Query, Filter, and
  Order. Thus, a custom function to extract values to attributes can be
  specified, and the system can remain agnostic to the client's data model,
  without loosing query power.

  For example, the default implementation works with attributes and items::

    def _object_getattr(obj, field):
      # check whether this key is an attribute
      if hasattr(obj, field):
        value = getattr(obj, field)

      # if not, perhaps it is an item (raw dicts, etc)
      elif field in obj:
        value = obj[field]

      # return whatever we've got.
      return value

  Or consider a more complex, application-specific structure::

    def _object_getattr(version, field):

      if field in ['key', 'committed', 'created', 'hash']:
        return getattr(version, field)

      else:
        return version.attributes[field]['value']

  '''

  # TODO: consider changing this to raise an exception if no value is found.
  value = None

  # check whether this key is an attribute
  if hasattr(obj, field):
    value = getattr(obj, field)

  # if not, perhaps it is an item (raw dicts, etc)
  elif field in obj:
    value = obj[field]

  # return whatever we've got.
  return value

def limit_gen(limit, iterable):
  '''A generator that applies a count `limit`.'''
  limit = int(limit)
  assert limit >= 0, 'negative limit'

  for item in iterable:
    if limit <= 0:
      break
    yield item
    limit -= 1

def offset_gen(offset, iterable, skip_signal=None):
  '''A generator that applies an `offset`, skipping `offset` elements from
  `iterable`. If skip_signal is a callable, it will be called with every
  skipped element.
  '''
  offset = int(offset)
  assert offset >= 0, 'negative offset'

  for item in iterable:
    if offset > 0:
      offset -= 1
      if callable(skip_signal):
        skip_signal(item)
    else:
      yield item

def chain_gen(iterables):
  '''A generator that chains `iterables`.'''
  for iterable in iterables:
    for item in iterable:
      yield item

class Filter(object):
  '''Represents a Filter for a specific field and its value.

  Filters are used on queries to narrow down the set of matching objects.

  Args:
    field: the attribute name (string) on which to apply the filter.

    op: the conditional operator to apply (one of
        ['<', '<=', '=', '!=', '>=', '>']).

    value: the attribute value to compare against.

  Examples::

    Filter('name', '=', 'John Cleese')
    Filter('age', '>=', 18)

  '''

  conditional_operators = ['<', '<=', '=', '!=', '>=', '>']
  '''Conditional operators that Filters support.'''

  _conditional_cmp = {
    "<"  : lambda a, b: a < b,
    "<=" : lambda a, b: a <= b,
    "="  : lambda a, b: a == b,
    "!=" : lambda a, b: a != b,
    ">=" : lambda a, b: a >= b,
    ">"  : lambda a, b: a > b
  }

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, field, op, value):
    if op not in self.conditional_operators:
      raise ValueError('"%s" is not a valid filter Conditional Operator' % op)

    self.field = field
    self.op = op
    self.value = value

  def __call__(self, obj):
    '''Returns whether this object passes this filter.
    This method aggressively tries to find the appropriate value.
    '''
    value = self.object_getattr(obj, self.field)

    # TODO: which way should the direction go here? it may make more sense to
    #       convert the passed-in value instead. Or try both? Or not at all?
    if not isinstance(value, self.value.__class__) and not self.value is None:
      value = self.value.__class__(value)

    return self.valuePasses(value)

  def valuePasses(self, value):
    '''Returns whether this value passes this filter'''
    return self._conditional_cmp[self.op](value, self.value)

  def __str__(self):
    return '%s %s %s' % (self.field, self.op, self.value)

  def __repr__(self):
    return "Filter('%s', '%s', %s)" % (self.field, self.op, repr(self.value))

  def __eq__(self, o):
    return self.field == o.field and self.op == o.op and self.value == o.value

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def generator(self, iterable):
    '''Generator function that iteratively filters given `items`.'''
    for item in iterable:
      if self(item):
        yield item

  @classmethod
  def filter(cls, filters, iterable):
    '''Returns the elements in `iterable` that pass given `filters`'''
    if isinstance(filters, Filter):
      filters = [filters]

    for filter in filters:
      iterable = filter.generator(iterable)

    return iterable

class Order(object):
  '''Represents an Order upon a specific field, and a direction.
  Orders are used on queries to define how they operate on objects

  Args:
    order: an order in string form. This follows the format: [+-]name
           where + is ascending, - is descending, and name is the name
           of the field to order by.
           Note: if no ordering operator is specified, + is default.

  Examples::

    Order('+name')   #  ascending order by name
    Order('-age')    # descending order by age
    Order('score')   #  ascending order by score

  '''

  order_operators = ['-', '+']
  '''Ordering operators: + is ascending, - is descending.'''

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, order):
    self.op = '+'

    try:
      if order[0] in self.order_operators:
        self.op = order[0]
        order = order[1:]
    except IndexError:
      raise ValueError('Order input be at least two characters long.')

    self.field = order

    if self.op not in self.order_operators:
      raise ValueError('"%s" is not a valid Order Operator.' % op)

  def __str__(self):
    return '%s%s' % (self.op, self.field)

  def __repr__(self):
    return "Order('%s%s')" % (self.op, self.field)

  def __eq__(self, other):
    return self.field == other.field and self.op == other.op

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def isAscending(self):
    return self.op == '+'

  def isDescending(self):
    return not self.isAscending()

  def keyfn(self, obj):
    '''A key function to be used in pythonic sort operations.'''
    return self.object_getattr(obj, self.field)

  @classmethod
  def multipleOrderComparison(cls, orders):
    '''Returns a function that will compare two items according to `orders`'''
    comparers = [ (o.keyfn, 1 if o.isAscending() else -1) for o in orders]

    def cmpfn(a, b):
      for keyfn, ascOrDesc in comparers:
        comparison = cmp(keyfn(a), keyfn(b)) * ascOrDesc
        if comparison is not 0:
          return comparison
      return 0

    return cmpfn

  @classmethod
  def sorted(cls, items, orders):
    '''Returns the elements in `items` sorted according to `orders`'''
    return sorted(items, cmp=cls.multipleOrderComparison(orders))

class Query(object):
  '''A Query describes a set of objects.

  Queries are used to retrieve objects and instances matching a set of criteria
  from Datastores. Query objects themselves are simply descriptions,
  the actual Query implementations are left up to the Datastores.
  '''

  '''Object attribute getter. Can be overridden to match client data model.'''
  object_getattr = staticmethod(_object_getattr)

  def __init__(self, key=Key('/'), limit=None, offset=0, object_getattr=None):
    ''' Initialize a query.

    Parameters
      key: a key representing the level of this query. For example, a Query with
           Key('/MontyPython/Actor:') would query objects in that key path, eg:
             Key('/MontyPython/Actor:JohnCleese')
             Key('/MontyPython/Actor:EricIdle')
             Key('/MontyPython/Actor:GrahamChapman')

            It is up to datastores how to implement this namespacing. E.g.,
            some datastores may store values in different tables or collections.

      limit: an integer representing the maximum number of results to return.

      offset: an integer representing a number of results to skip.

      object_getattr: a function to extract attribute values from an object. It
           is used to satisfy query filters and orders. Defining this function
           allows the client to control the data model of the stored values.
           The default function attempts to access values as attributes
           (__getattr__) or items (__getitem__).
    '''
    if not isinstance(key, Key):
      raise TypeError('key must be of type %s' % Key)

    self.key = key

    self.limit = int(limit) if limit is not None else None
    self.offset = int(offset)

    self.filters = []
    self.orders = []

    if object_getattr:
      self.object_getattr = object_getattr

  def __str__(self):
    '''Returns a string describing this query.'''
    return repr(self)

  def __repr__(self):
    '''Returns the representation of this query. Enables eval(repr(.)).'''
    return 'Query.from_dict(%s)' % self.dict()

  def __call__(self, iterable):
    '''Naively apply this query on an iterable of objects.
    Applying a query applies filters, sorts by appropriate orders, and returns
    a limited set.

    WARNING: When orders are applied, this function operates on the entire set
             of entities directly, not just iterators/generators. That means
             the entire result set will be in memory. Datastores with large
             objects and large query results should translate the Query and
             perform their own optimizations.
    '''

    cursor = Cursor(self, iterable)
    cursor.apply_filter()
    cursor.apply_order()
    cursor.apply_offset()
    cursor.apply_limit()
    return cursor

  def order(self, order):
    '''Adds an Order to this query.

    Args:
      see :py:class:`Order <datastore.query.Order>` constructor

    Returns self for JS-like method chaining::

      query.order('+age').order('-home')

    '''
    order = order if isinstance(order, Order) else Order(order)

    # ensure order gets attr values the same way the rest of the query does.
    order.object_getattr = self.object_getattr
    self.orders.append(order)
    return self # for chaining

  def filter(self, *args):
    '''Adds a Filter to this query.

    Args:
      see :py:class:`Filter <datastore.query.Filter>` constructor

    Returns self for JS-like method chaining::

      query.filter('age', '>', 18).filter('sex', '=', 'Female')

    '''
    if len(args) == 1 and isinstance(args[0], Filter):
      filter = args[0]
    else:
      filter = Filter(*args)

    # ensure filter gets attr values the same way the rest of the query does.
    filter.object_getattr = self.object_getattr
    self.filters.append(filter)
    return self # for chaining

  def __cmp__(self, other):
    return cmp(self.dict(), other.dict())

  def __hash__(self):
    return hash(repr(self))

  def copy(self):
    '''Returns a copy of this query.'''
    if self.object_getattr is Query.object_getattr:
      other = Query(self.key)
    else:
      other = Query(self.key, object_getattr=self.object_getattr)
    other.limit = self.limit
    other.offset = self.offset
    other.filters = self.filters
    other.orders = self.orders
    return other

  def dict(self):
    '''Returns a dictionary representing this query.'''
    d = dict()
    d['key'] = str(self.key)

    if self.limit is not None:
      d['limit'] = self.limit
    if self.offset > 0:
      d['offset'] = self.offset
    if len(self.filters) > 0:
      d['filter'] = [[f.field, f.op, f.value] for f in self.filters]
    if len(self.orders) > 0:
      d['order'] = [str(o) for o in self.orders]

    return d

  @classmethod
  def from_dict(cls, dictionary):
    '''Constructs a query from a dictionary.'''
    query = cls(Key(dictionary['key']))

    for key, value in dictionary.items():

      if key == 'order':
        for order in value:
          query.order(order)

      elif key == 'filter':
        for filter in value:
          if not isinstance(filter, Filter):
            filter = Filter(*filter)
          query.filter(filter)

      elif key in ['limit', 'offset']:
        setattr(query, key, value)
    return query

def is_iterable(obj):
  return hasattr(obj, '__iter__') or hasattr(obj, '__getitem__')

class Cursor(object):
  '''Represents a query result generator.'''

  __slots__ = ('query', '_iterable', '_iterator', 'skipped', 'returned', )

  def __init__(self, query, iterable):
    if not isinstance(query, Query):
      raise ValueError('Cursor received invalid query: %s' % query)

    if not is_iterable(iterable):
      raise ValueError('Cursor received invalid iterable: %s' % iterable)

    self.query = query
    self._iterable = iterable
    self._iterator = None
    self.returned = 0
    self.skipped = 0

  def __iter__(self):
    '''The cursor itself is the iterator. Note that it cannot be used twice,
    and once iteration starts, the cursor cannot be modified.
    '''
    if self._iterator:
      raise RuntimeError('Attempt to iterate over Cursor twice.')

    self._iterator = iter(self._iterable)
    return self

  def next(self):
    '''Iterator next. Build up count of returned elements during iteration.'''

    # if iteration has not begun, begin it.
    if not self._iterator:
      self.__iter__()

    next = self._iterator.next()
    if next is not StopIteration:
      self._returned_inc(next)
    return next

  def _skipped_inc(self, item):
    '''A function to increment the skipped count.'''
    self.skipped += 1

  def _returned_inc(self, item):
    '''A function to increment the returned count.'''
    self.returned += 1

  def _ensure_modification_is_safe(self):
    '''Assertions to ensure modification of this Cursor is safe.'''
    assert self.query, 'Cursor must have a Query.'
    assert is_iterable(self._iterable), 'Cursor must have a resultset iterable.'
    assert not self._iterator, 'Cursor must not be modified after iteration.'

  def apply_filter(self):
    '''Naively apply query filters.'''
    self._ensure_modification_is_safe()

    if len(self.query.filters) > 0:
      self._iterable = Filter.filter(self.query.filters, self._iterable)

  def apply_order(self):
    '''Naively apply query orders.'''
    self._ensure_modification_is_safe()

    if len(self.query.orders) > 0:
      self._iterable = Order.sorted(self._iterable, self.query.orders)
      # not a generator :(

  def apply_offset(self):
    '''Naively apply query offset.'''
    self._ensure_modification_is_safe()

    if self.query.offset != 0:
      self._iterable = \
        offset_gen(self.query.offset, self._iterable, self._skipped_inc)
        # _skipped_inc helps keep count of skipped elements

  def apply_limit(self):
    '''Naively apply query limit.'''
    self._ensure_modification_is_safe()

    if self.query.limit is not None:
      self._iterable = limit_gen(self.query.limit, self._iterable)

<commit_msg>Added offset_key on query to allow datastore implementations that offset on key rather than index<commit_after>
from key import Key

def _object_getattr(obj, field):
  '''Attribute getter for the objects to operate on.

  This function can be overridden in classes or instances of Query, Filter, and
  Order. Thus, a custom function to extract values to attributes can be
  specified, and the system can remain agnostic to the client's data model,
  without loosing query power.

  For example, the default implementation works with attributes and items::

    def _object_getattr(obj, field):
      # check whether this key is an attribute
      if hasattr(obj, field):
        value = getattr(obj, field)

      # if not, perhaps it is an item (raw dicts, etc)
      elif field in obj:
        value = obj[field]

      # return whatever we've got.
      return value

  Or consider a more complex, application-specific structure::

    def _object_getattr(version, field):

      if field in ['key', 'committed', 'created', 'hash']:
        return getattr(version, field)

      else:
        return version.attributes[field]['value']

  '''

  # TODO: consider changing this to raise an exception if no value is found.
  value = None

  # check whether this key is an attribute
  if hasattr(obj, field):
    value = getattr(obj, field)

  # if not, perhaps it is an item (raw dicts, etc)
  elif field in obj:
    value = obj[field]

  # return whatever we've got.
  return value

def limit_gen(limit, iterable):
  '''A generator that applies a count `limit`.'''
  limit = int(limit)
  assert limit >= 0, 'negative limit'

  for item in iterable:
    if limit <= 0:
      break
    yield item
    limit -= 1

def offset_gen(offset, iterable, skip_signal=None):
  '''A generator that applies an `offset`, skipping `offset` elements from
  `iterable`. If skip_signal is a callable, it will be called with every
  skipped element.
  '''
  offset = int(offset)
  assert offset >= 0, 'negative offset'

  for item in iterable:
    if offset > 0:
      offset -= 1
      if callable(skip_signal):
        skip_signal(item)
    else:
      yield item

def chain_gen(iterables):
  '''A generator that chains `iterables`.'''
  for iterable in iterables:
    for item in iterable:
      yield item

class Filter(object):
  '''Represents a Filter for a specific field and its value.

  Filters are used on queries to narrow down the set of matching objects.

  Args:
    field: the attribute name (string) on which to apply the filter.

    op: the conditional operator to apply (one of
        ['<', '<=', '=', '!=', '>=', '>']).

    value: the attribute value to compare against.

  Examples::

    Filter('name', '=', 'John Cleese')
    Filter('age', '>=', 18)

  '''

  conditional_operators = ['<', '<=', '=', '!=', '>=', '>']
  '''Conditional operators that Filters support.'''

  _conditional_cmp = {
    "<"  : lambda a, b: a < b,
    "<=" : lambda a, b: a <= b,
    "="  : lambda a, b: a == b,
    "!=" : lambda a, b: a != b,
    ">=" : lambda a, b: a >= b,
    ">"  : lambda a, b: a > b
  }

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, field, op, value):
    if op not in self.conditional_operators:
      raise ValueError('"%s" is not a valid filter Conditional Operator' % op)

    self.field = field
    self.op = op
    self.value = value

  def __call__(self, obj):
    '''Returns whether this object passes this filter.
    This method aggressively tries to find the appropriate value.
    '''
    value = self.object_getattr(obj, self.field)

    # TODO: which way should the direction go here? it may make more sense to
    #       convert the passed-in value instead. Or try both? Or not at all?
    if not isinstance(value, self.value.__class__) and not self.value is None:
      value = self.value.__class__(value)

    return self.valuePasses(value)

  def valuePasses(self, value):
    '''Returns whether this value passes this filter'''
    return self._conditional_cmp[self.op](value, self.value)

  def __str__(self):
    return '%s %s %s' % (self.field, self.op, self.value)

  def __repr__(self):
    return "Filter('%s', '%s', %s)" % (self.field, self.op, repr(self.value))

  def __eq__(self, o):
    return self.field == o.field and self.op == o.op and self.value == o.value

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def generator(self, iterable):
    '''Generator function that iteratively filters given `items`.'''
    for item in iterable:
      if self(item):
        yield item

  @classmethod
  def filter(cls, filters, iterable):
    '''Returns the elements in `iterable` that pass given `filters`'''
    if isinstance(filters, Filter):
      filters = [filters]

    for filter in filters:
      iterable = filter.generator(iterable)

    return iterable

class Order(object):
  '''Represents an Order upon a specific field, and a direction.
  Orders are used on queries to define how they operate on objects

  Args:
    order: an order in string form. This follows the format: [+-]name
           where + is ascending, - is descending, and name is the name
           of the field to order by.
           Note: if no ordering operator is specified, + is default.

  Examples::

    Order('+name')   #  ascending order by name
    Order('-age')    # descending order by age
    Order('score')   #  ascending order by score

  '''

  order_operators = ['-', '+']
  '''Ordering operators: + is ascending, - is descending.'''

  object_getattr = staticmethod(_object_getattr)
  '''Object attribute getter. Can be overridden to match client data model.
  See :py:meth:`datastore.query._object_getattr`.
  '''

  def __init__(self, order):
    self.op = '+'

    try:
      if order[0] in self.order_operators:
        self.op = order[0]
        order = order[1:]
    except IndexError:
      raise ValueError('Order input be at least two characters long.')

    self.field = order

    if self.op not in self.order_operators:
      raise ValueError('"%s" is not a valid Order Operator.' % op)

  def __str__(self):
    return '%s%s' % (self.op, self.field)

  def __repr__(self):
    return "Order('%s%s')" % (self.op, self.field)

  def __eq__(self, other):
    return self.field == other.field and self.op == other.op

  def __ne__(self, other):
    return not self.__eq__(other)

  def __hash__(self):
    return hash(repr(self))

  def isAscending(self):
    return self.op == '+'

  def isDescending(self):
    return not self.isAscending()

  def keyfn(self, obj):
    '''A key function to be used in pythonic sort operations.'''
    return self.object_getattr(obj, self.field)

  @classmethod
  def multipleOrderComparison(cls, orders):
    '''Returns a function that will compare two items according to `orders`'''
    comparers = [ (o.keyfn, 1 if o.isAscending() else -1) for o in orders]

    def cmpfn(a, b):
      for keyfn, ascOrDesc in comparers:
        comparison = cmp(keyfn(a), keyfn(b)) * ascOrDesc
        if comparison is not 0:
          return comparison
      return 0

    return cmpfn

  @classmethod
  def sorted(cls, items, orders):
    '''Returns the elements in `items` sorted according to `orders`'''
    return sorted(items, cmp=cls.multipleOrderComparison(orders))

class Query(object):
  '''A Query describes a set of objects.

  Queries are used to retrieve objects and instances matching a set of criteria
  from Datastores. Query objects themselves are simply descriptions,
  the actual Query implementations are left up to the Datastores.
  '''

  '''Object attribute getter. Can be overridden to match client data model.'''
  object_getattr = staticmethod(_object_getattr)

  def __init__(self, key=Key('/'), limit=None, offset=0, offset_key=None, object_getattr=None):
    ''' Initialize a query.

    Parameters
      key: a key representing the level of this query. For example, a Query with
           Key('/MontyPython/Actor:') would query objects in that key path, eg:
             Key('/MontyPython/Actor:JohnCleese')
             Key('/MontyPython/Actor:EricIdle')
             Key('/MontyPython/Actor:GrahamChapman')

            It is up to datastores how to implement this namespacing. E.g.,
            some datastores may store values in different tables or collections.

      limit: an integer representing the maximum number of results to return.

      offset: an integer representing a number of results to skip.

      object_getattr: a function to extract attribute values from an object. It
           is used to satisfy query filters and orders. Defining this function
           allows the client to control the data model of the stored values.
           The default function attempts to access values as attributes
           (__getattr__) or items (__getitem__).
    '''
    if not isinstance(key, Key):
      raise TypeError('key must be of type %s' % Key)

    self.key = key

    self.limit = int(limit) if limit is not None else None
    self.offset = int(offset)
    self.offset_key = offset_key

    self.filters = []
    self.orders = []

    if object_getattr:
      self.object_getattr = object_getattr

  def __str__(self):
    '''Returns a string describing this query.'''
    return repr(self)

  def __repr__(self):
    '''Returns the representation of this query. Enables eval(repr(.)).'''
    return 'Query.from_dict(%s)' % self.dict()

  def __call__(self, iterable):
    '''Naively apply this query on an iterable of objects.
    Applying a query applies filters, sorts by appropriate orders, and returns
    a limited set.

    WARNING: When orders are applied, this function operates on the entire set
             of entities directly, not just iterators/generators. That means
             the entire result set will be in memory. Datastores with large
             objects and large query results should translate the Query and
             perform their own optimizations.
    '''

    cursor = Cursor(self, iterable)
    cursor.apply_filter()
    cursor.apply_order()
    cursor.apply_offset()
    cursor.apply_limit()
    return cursor

  def order(self, order):
    '''Adds an Order to this query.

    Args:
      see :py:class:`Order <datastore.query.Order>` constructor

    Returns self for JS-like method chaining::

      query.order('+age').order('-home')

    '''
    order = order if isinstance(order, Order) else Order(order)

    # ensure order gets attr values the same way the rest of the query does.
    order.object_getattr = self.object_getattr
    self.orders.append(order)
    return self # for chaining

  def filter(self, *args):
    '''Adds a Filter to this query.

    Args:
      see :py:class:`Filter <datastore.query.Filter>` constructor

    Returns self for JS-like method chaining::

      query.filter('age', '>', 18).filter('sex', '=', 'Female')

    '''
    if len(args) == 1 and isinstance(args[0], Filter):
      filter = args[0]
    else:
      filter = Filter(*args)

    # ensure filter gets attr values the same way the rest of the query does.
    filter.object_getattr = self.object_getattr
    self.filters.append(filter)
    return self # for chaining

  def __cmp__(self, other):
    return cmp(self.dict(), other.dict())

  def __hash__(self):
    return hash(repr(self))

  def copy(self):
    '''Returns a copy of this query.'''
    if self.object_getattr is Query.object_getattr:
      other = Query(self.key)
    else:
      other = Query(self.key, object_getattr=self.object_getattr)
    other.limit = self.limit
    other.offset = self.offset
    other.offset_key = self.offset_key
    other.filters = self.filters
    other.orders = self.orders
    return other

  def dict(self):
    '''Returns a dictionary representing this query.'''
    d = dict()
    d['key'] = str(self.key)

    if self.limit is not None:
      d['limit'] = self.limit
    if self.offset > 0:
      d['offset'] = self.offset
    if self.offset_key:
      d['offset_key'] = str(self.offset_key)
    if len(self.filters) > 0:
      d['filter'] = [[f.field, f.op, f.value] for f in self.filters]
    if len(self.orders) > 0:
      d['order'] = [str(o) for o in self.orders]

    return d

  @classmethod
  def from_dict(cls, dictionary):
    '''Constructs a query from a dictionary.'''
    query = cls(Key(dictionary['key']))

    for key, value in dictionary.items():

      if key == 'order':
        for order in value:
          query.order(order)

      elif key == 'filter':
        for filter in value:
          if not isinstance(filter, Filter):
            filter = Filter(*filter)
          query.filter(filter)

      elif key in ['limit', 'offset', 'offset_key']:
        setattr(query, key, value)
    return query

def is_iterable(obj):
  return hasattr(obj, '__iter__') or hasattr(obj, '__getitem__')

class Cursor(object):
  '''Represents a query result generator.'''

  __slots__ = ('query', '_iterable', '_iterator', 'skipped', 'returned', )

  def __init__(self, query, iterable):
    if not isinstance(query, Query):
      raise ValueError('Cursor received invalid query: %s' % query)

    if not is_iterable(iterable):
      raise ValueError('Cursor received invalid iterable: %s' % iterable)

    self.query = query
    self._iterable = iterable
    self._iterator = None
    self.returned = 0
    self.skipped = 0

  def __iter__(self):
    '''The cursor itself is the iterator. Note that it cannot be used twice,
    and once iteration starts, the cursor cannot be modified.
    '''
    if self._iterator:
      raise RuntimeError('Attempt to iterate over Cursor twice.')

    self._iterator = iter(self._iterable)
    return self

  def next(self):
    '''Iterator next. Build up count of returned elements during iteration.'''

    # if iteration has not begun, begin it.
    if not self._iterator:
      self.__iter__()

    next = self._iterator.next()
    if next is not StopIteration:
      self._returned_inc(next)
    return next

  def _skipped_inc(self, item):
    '''A function to increment the skipped count.'''
    self.skipped += 1

  def _returned_inc(self, item):
    '''A function to increment the returned count.'''
    self.returned += 1

  def _ensure_modification_is_safe(self):
    '''Assertions to ensure modification of this Cursor is safe.'''
    assert self.query, 'Cursor must have a Query.'
    assert is_iterable(self._iterable), 'Cursor must have a resultset iterable.'
    assert not self._iterator, 'Cursor must not be modified after iteration.'

  def apply_filter(self):
    '''Naively apply query filters.'''
    self._ensure_modification_is_safe()

    if len(self.query.filters) > 0:
      self._iterable = Filter.filter(self.query.filters, self._iterable)

  def apply_order(self):
    '''Naively apply query orders.'''
    self._ensure_modification_is_safe()

    if len(self.query.orders) > 0:
      self._iterable = Order.sorted(self._iterable, self.query.orders)
      # not a generator :(

  def apply_offset(self):
    '''Naively apply query offset.'''
    self._ensure_modification_is_safe()

    if self.query.offset != 0:
      self._iterable = \
        offset_gen(self.query.offset, self._iterable, self._skipped_inc)
        # _skipped_inc helps keep count of skipped elements

  def apply_limit(self):
    '''Naively apply query limit.'''
    self._ensure_modification_is_safe()

    if self.query.limit is not None:
      self._iterable = limit_gen(self.query.limit, self._iterable)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Import Python libs
import logging
log = logging.getLogger(__name__)

# Import pyrax (SDK for Rackspace cloud) third party libs
import pyrax
import pyrax.exceptions

# Import salt classes
from salt.utils.openstack.pyrax import authenticate

class RackspaceQueues(object):
    def __init__(self, username, password, region, **kwargs):
        self.auth = authenticate.Authenticate(username, password, region, **kwargs)
        self.conn = self.auth.conn.queues

    def create(self, qname):
        '''
        Create RackSpace Queue.
        '''
        try:
            if self.exists(qname):
                log.error('Queues "{0}" already exists. Nothing done.'.format(qname))
                return True

            self.conn.create(qname)

            return True
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during creation: {0}'.format(err_msg))
        return False

    def delete(self, qname):
        '''
        Delete an existings RackSpace Queue.
        '''
        try:
            q = self.exists(qname)
            if not q:
                return False
            queue = self.show(qname)
            if queue:
                queue.delete()
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during deletion: {0}'.format(err_msg))
            return False

        return True

    def exists(self, qname):
        '''
        Check to see if a Queue exists.
        '''
        try:
            # First if not exists() -> exit
            if self.conn.queue_exists(qname):
                return True
            return False
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during existing queue check: {0}'.format(err_msg))
        return False

    def show(self, qname):
        '''
        Show information about Queue
        '''
        try:
            # First if not exists() -> exit
            if not self.conn.queue_exists(qname):
                return {}
            # If exist, search the queue to return the Queue Object
            for queue in self.conn.list():
                if queue.name == qname:
                    return queue
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during existing queue check: {0}'.format(err_msg))
        return {}
<commit_msg>Making salt/utils/openstack/pyrax/queues python3 compatible<commit_after>from __future__ import absolute_import
# -*- coding: utf-8 -*-
# Import Python libs
import logging
log = logging.getLogger(__name__)

# Import pyrax (SDK for Rackspace cloud) third party libs
import pyrax
import pyrax.exceptions

# Import salt classes
from salt.utils.openstack.pyrax import authenticate

class RackspaceQueues(object):
    def __init__(self, username, password, region, **kwargs):
        self.auth = authenticate.Authenticate(username, password, region, **kwargs)
        self.conn = self.auth.conn.queues

    def create(self, qname):
        '''
        Create RackSpace Queue.
        '''
        try:
            if self.exists(qname):
                log.error('Queues "{0}" already exists. Nothing done.'.format(qname))
                return True

            self.conn.create(qname)

            return True
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during creation: {0}'.format(err_msg))
        return False

    def delete(self, qname):
        '''
        Delete an existings RackSpace Queue.
        '''
        try:
            q = self.exists(qname)
            if not q:
                return False
            queue = self.show(qname)
            if queue:
                queue.delete()
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during deletion: {0}'.format(err_msg))
            return False

        return True

    def exists(self, qname):
        '''
        Check to see if a Queue exists.
        '''
        try:
            # First if not exists() -> exit
            if self.conn.queue_exists(qname):
                return True
            return False
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during existing queue check: {0}'.format(err_msg))
        return False

    def show(self, qname):
        '''
        Show information about Queue
        '''
        try:
            # First if not exists() -> exit
            if not self.conn.queue_exists(qname):
                return {}
            # If exist, search the queue to return the Queue Object
            for queue in self.conn.list():
                if queue.name == qname:
                    return queue
        except pyrax.exceptions as err_msg:
            log.error('RackSpace API got some problems during existing queue check: {0}'.format(err_msg))
        return {}
<|endoftext|>


<commit_before>import urllib2

from sqlalchemy import func, distinct, asc, desc, and_, or_
from flask import Blueprint, request, jsonify, abort, g, render_template, make_response, redirect, url_for, flash

from dataviva import db, __year_range__, view_cache
from dataviva.attrs.models import Bra, Wld, Hs, Cnae, Cbo, Yb, Course_hedu, Course_sc, University, School, bra_pr, Search
from dataviva.secex.models import Ymp, Ymw
from dataviva.rais.models import Yi, Yo
from dataviva.hedu.models import Yu, Yc_hedu
from dataviva.sc.models import Yc_sc, Ys, Ybs
from dataviva.ask.models import Question

from dataviva.utils.gzip_data import gzip_data
from dataviva.utils.cached_query import cached_query
from dataviva.utils.exist_or_404 import exist_or_404
from dataviva.utils.title_case import title_case
from sqlalchemy import desc
from dataviva.utils.gzip_data import gzipped
from dataviva.utils.cached_query import api_cache_key

mod = Blueprint('attrs', __name__, url_prefix='/attrs')

@mod.errorhandler(404)
def page_not_found(error):
    return error, 404

def fix_name(attr, lang):

    for col in ["desc", "name", "gender", "article", "keywords"]:
        # raise Exception("{}_{}".format(col, lang) in attr)
        if "{}_{}".format(col, lang) in attr:
            attr[col] = title_case(attr["{}_{}".format(col, lang)])
        else:
            attr[col] = False
        if "{}_en".format(col) in attr: del attr["{}_en".format(col)]
        if "{}_pt".format(col) in attr: del attr["{}_pt".format(col)]

    school_type_lang = "school_type_" + lang
    if school_type_lang in attr:
        attr["school_type"] = attr[school_type_lang]
    if "school_type_en" in attr: del attr["school_type_en"]
    if "school_type_pt" in attr: del attr["school_type_pt"]

    if "is_vocational" in attr: del attr["is_vocational"]
    return attr

############################################################
# ----------------------------------------------------------
# All attribute views
#
############################################################
def get_planning_region_map():
    prs = db.session.query(bra_pr).all()
    pr_map = {k:v for k,v in prs}
    return pr_map

@mod.route("/school/in/<bra_id>/")
@gzipped
@view_cache.cached(key_prefix=api_cache_key("attrs"))
def school_attrs(bra_id):
    results = db.engine.execute('''
        SELECT id, school_type_id, name_pt, color
        FROM attrs_school
        LEFT JOIN sc_ybs
        ON attrs_school.id=sc_ybs.school_id
        WHERE sc_ybs.bra_id = %s;
    ''', bra_id)

    data = [{'id': row[0], 'school_type_id': row[1], 'name': row[2], 'color': row[3], 'icon': '/static/img/icons/school/school_{}.png'.format(row[1].lower())} for row in results]

    return jsonify(data=data)

@mod.route('/<attr>/')
@mod.route('/<attr>/<Attr_id>/')
def attrs(attr="bra",Attr_id=None):

    Attr = globals()[attr.capitalize()]
    Attr_weight_mergeid = "{0}_id".format(attr)

    if attr == "bra":
        Attr_weight_tbl = Yb
        Attr_weight_col = "population"
    elif attr == "cnae":
        Attr_weight_tbl = Yi
        Attr_weight_col = "num_emp"
    elif attr == "cbo":
        Attr_weight_tbl = Yo
        Attr_weight_col = "num_emp"
    elif attr == "hs":
        Attr_weight_tbl = Ymp
        Attr_weight_col = "export_val"
    elif attr == "wld":
        Attr_weight_tbl = Ymw
        Attr_weight_col = "export_val"
    elif attr == "course_hedu":
        Attr_weight_tbl = Yc_hedu
        Attr_weight_col = "enrolled"
    elif attr == "university":
        Attr_weight_tbl = Yu
        Attr_weight_col = "enrolled"
    elif attr == "school":
        Attr_weight_tbl = Ys
        Attr_weight_col = "enrolled"
    elif attr == "course_sc":
        Attr_weight_tbl = Yc_sc
        Attr_weight_col = "enrolled"

    depths = {}
    depths["bra"] = [1,3,5,7,9]
    depths["cnae"] = [1,3,6]
    depths["cbo"] = [1,4]
    depths["hs"] = [2,6]
    depths["wld"] = [2,5]
    depths["course_hedu"] = [2,6]
    depths["university"] = [5]
    depths["course_sc"] = [2,5]
    depths["school"] = [8]

    depth = request.args.get('depth', None)
    order = request.args.get('order', None)
    offset = request.args.get('offset', None)
    limit = request.args.get('limit', None)
    if offset:
        offset = float(offset)
        limit = limit or 50
    elif limit:
        offset = float(0)

    lang = request.args.get('lang', None) or g.locale
    ret = {}
    dataset = "rais"
    if Attr == Wld or Attr == Hs:
        dataset = "secex"
    elif Attr == Course_hedu or Attr == University:
        dataset = "hedu"
    elif Attr == Course_sc or Attr == School:
        dataset = "sc"
    elif Attr == Bra:
        dataset = "population"

    cache_id = request.path + lang
    if depth:
        cache_id = cache_id + "/" + depth
    # first lets test if this query is cached
    cached_q = cached_query(cache_id)
    if cached_q and limit is None:
        ret = make_response(cached_q)
        ret.headers['Content-Encoding'] = 'gzip'
        ret.headers['Content-Length'] = str(len(ret.data))
        return ret

    # if an ID is supplied only return that
    if Attr_id:

        # the '.show.' indicates that we are looking for a specific nesting
        if ".show." in Attr_id:
            this_attr, ret["nesting_level"] = Attr_id.split(".show.")
            # filter table by requested nesting level
            attrs = Attr.query \
                    .filter(Attr.id.startswith(this_attr)) \
                    .filter(func.char_length(Attr.id) == ret["nesting_level"]).all()

        # the 'show.' indicates that we are looking for a specific nesting
        elif "show." in Attr_id:
            ret["nesting_level"] = Attr_id.split(".")[1]
            # filter table by requested nesting level
            attrs = Attr.query.filter(func.char_length(Attr.id) == ret["nesting_level"]).all()

        # the '.' here means we want to see all attrs within a certain distance
        elif "." in Attr_id:
            this_attr, distance = Attr_id.split(".")
            this_attr = Attr.query.get_or_404(this_attr)
            attrs = this_attr.get_neighbors(distance)

        else:
            attrs = [Attr.query.get_or_404(Attr_id)]

        ret["data"] = [fix_name(a.serialize(), lang) for a in attrs]
    # an ID/filter was not provided
    else:
        latest_year = __year_range__[dataset][-1]
        latest_month = False
        if "-" in latest_year:
            latest_year, latest_month = latest_year.split("-")
            latest_month = int(latest_month)
        latest_year = int(latest_year)

        conds = [getattr(Attr_weight_tbl,"{0}_id".format(attr)) == Attr.id, Attr_weight_tbl.year == latest_year]
        if latest_month:
            conds.append(Attr_weight_tbl.month == latest_month)

        query = db.session.query(Attr,Attr_weight_tbl).outerjoin(Attr_weight_tbl, and_(*conds))
        if Attr == School:
            query = query.filter(Attr.is_vocational == 1)

        if depth:
            query = query.filter(func.char_length(Attr.id) == depth)
        else:
            query = query.filter(func.char_length(Attr.id).in_(depths[attr]))

        if order:
            direction = "asc"

            if "." in order:
                o, direction = order.split(".")
            else:
                o = order

            if o == "name":
                o = "name_{0}".format(lang)

            if o == Attr_weight_col:
                order_table  = Attr_weight_tbl
            else:
                order_table = Attr

            if direction == "asc":
                query = query.order_by(asc(getattr(order_table,o)))
            elif direction == "desc":
                query = query.order_by(desc(getattr(order_table,o)))

        if limit:
            query = query.limit(limit).offset(offset)

        attrs_all = query.all()

        # just get items available in DB
        attrs_w_data = None
        if depth is None and limit is None:
            attrs_w_data = db.session.query(Attr, Attr_weight_tbl) \
                .filter(getattr(Attr_weight_tbl, Attr_weight_mergeid) == Attr.id) \
                .group_by(Attr.id)
            attrs_w_data = [a[0].id for a in attrs_w_data]

        attrs = []

        # all_planning_regions = {}

        for i, a in enumerate(attrs_all):
            b = a[0].serialize()
            if a[1]:
                c = a[1].serialize()
                if Attr_weight_col in c:
                    b[Attr_weight_col] = c[Attr_weight_col]
                else:
                    b[Attr_weight_col] = 0
            else:
                b[Attr_weight_col] = 0
            a = b
            if attrs_w_data:
                a["available"] = False
                if a["id"] in attrs_w_data:
                    a["available"] = True
            # if Attr_weight_col == "population" and len(a["id"]) == 9 and a["id"][:3] == "4mg":
            #     if not all_planning_regions:
            #         all_planning_regions = get_planning_region_map()
            #     if a["id"] in all_planning_regions:
            #         plr = all_planning_regions[a["id"]]
            #         a["plr"] = plr
            if order:
                a["rank"] = int(i+offset+1)
            if attr == "bra" and "id_ibge" not in a:
                a["id_ibge"] = False
            attrs.append(fix_name(a, lang))

        ret["data"] = attrs

    ret = jsonify(ret)
    ret.data = gzip_data(ret.data)

    if limit is None and cached_q is None:
        cached_query(cache_id, ret.data)

    ret.headers['Content-Encoding'] = 'gzip'
    ret.headers['Content-Length'] = str(len(ret.data))

    return ret

@mod.route('/table/<attr>/<depth>/')
def attrs_table(attr="bra",depth="2"):
    g.page_type = "attrs"
    data_url = "/attrs/{0}/?depth={1}".format(attr,depth)
    return render_template("general/table.html", data_url=data_url)

@mod.route('/search/<term>/')
@view_cache.cached(timeout=86400, key_prefix=api_cache_key("search"))
def attrs_search(term=None):
    result = []
    lang = request.args.get('lang', 'en') or g.locale
    name_col = Search.name_en if lang == 'en' else Search.name_pt
    profiles = Search.query.filter(name_col.like('%{}%'.format(term))).order_by(Search.weight.desc()).limit(20)
    result = [p.serialize(lang == "pt") for p in profiles]
    ret = jsonify({"activities":result})
    return ret
<commit_msg>Fix for accented characters<commit_after>import urllib2

from sqlalchemy import func, distinct, asc, desc, and_, or_
from flask import Blueprint, request, jsonify, abort, g, render_template, make_response, redirect, url_for, flash

from dataviva import db, __year_range__, view_cache
from dataviva.attrs.models import Bra, Wld, Hs, Cnae, Cbo, Yb, Course_hedu, Course_sc, University, School, bra_pr, Search
from dataviva.secex.models import Ymp, Ymw
from dataviva.rais.models import Yi, Yo
from dataviva.hedu.models import Yu, Yc_hedu
from dataviva.sc.models import Yc_sc, Ys, Ybs
from dataviva.ask.models import Question

from dataviva.utils.gzip_data import gzip_data
from dataviva.utils.cached_query import cached_query
from dataviva.utils.exist_or_404 import exist_or_404
from dataviva.utils.title_case import title_case
from sqlalchemy import desc
from dataviva.utils.gzip_data import gzipped
from dataviva.utils.cached_query import api_cache_key

mod = Blueprint('attrs', __name__, url_prefix='/attrs')

@mod.errorhandler(404)
def page_not_found(error):
    return error, 404

def fix_name(attr, lang):

    for col in ["desc", "name", "gender", "article", "keywords"]:
        # raise Exception("{}_{}".format(col, lang) in attr)
        if "{}_{}".format(col, lang) in attr:
            attr[col] = title_case(attr["{}_{}".format(col, lang)])
        else:
            attr[col] = False
        if "{}_en".format(col) in attr: del attr["{}_en".format(col)]
        if "{}_pt".format(col) in attr: del attr["{}_pt".format(col)]

    school_type_lang = "school_type_" + lang
    if school_type_lang in attr:
        attr["school_type"] = attr[school_type_lang]
    if "school_type_en" in attr: del attr["school_type_en"]
    if "school_type_pt" in attr: del attr["school_type_pt"]

    if "is_vocational" in attr: del attr["is_vocational"]
    return attr

############################################################
# ----------------------------------------------------------
# All attribute views
#
############################################################
def get_planning_region_map():
    prs = db.session.query(bra_pr).all()
    pr_map = {k:v for k,v in prs}
    return pr_map

@mod.route("/school/in/<bra_id>/")
@gzipped
@view_cache.cached(key_prefix=api_cache_key("attrs"))
def school_attrs(bra_id):
    results = db.engine.execute('''
        SELECT id, school_type_id, name_pt, color
        FROM attrs_school
        LEFT JOIN sc_ybs
        ON attrs_school.id=sc_ybs.school_id
        WHERE sc_ybs.bra_id = %s;
    ''', bra_id)

    data = [{'id': row[0], 'school_type_id': row[1], 'name': row[2], 'color': row[3], 'icon': '/static/img/icons/school/school_{}.png'.format(row[1].lower())} for row in results]

    return jsonify(data=data)

@mod.route('/<attr>/')
@mod.route('/<attr>/<Attr_id>/')
def attrs(attr="bra",Attr_id=None):

    Attr = globals()[attr.capitalize()]
    Attr_weight_mergeid = "{0}_id".format(attr)

    if attr == "bra":
        Attr_weight_tbl = Yb
        Attr_weight_col = "population"
    elif attr == "cnae":
        Attr_weight_tbl = Yi
        Attr_weight_col = "num_emp"
    elif attr == "cbo":
        Attr_weight_tbl = Yo
        Attr_weight_col = "num_emp"
    elif attr == "hs":
        Attr_weight_tbl = Ymp
        Attr_weight_col = "export_val"
    elif attr == "wld":
        Attr_weight_tbl = Ymw
        Attr_weight_col = "export_val"
    elif attr == "course_hedu":
        Attr_weight_tbl = Yc_hedu
        Attr_weight_col = "enrolled"
    elif attr == "university":
        Attr_weight_tbl = Yu
        Attr_weight_col = "enrolled"
    elif attr == "school":
        Attr_weight_tbl = Ys
        Attr_weight_col = "enrolled"
    elif attr == "course_sc":
        Attr_weight_tbl = Yc_sc
        Attr_weight_col = "enrolled"

    depths = {}
    depths["bra"] = [1,3,5,7,9]
    depths["cnae"] = [1,3,6]
    depths["cbo"] = [1,4]
    depths["hs"] = [2,6]
    depths["wld"] = [2,5]
    depths["course_hedu"] = [2,6]
    depths["university"] = [5]
    depths["course_sc"] = [2,5]
    depths["school"] = [8]

    depth = request.args.get('depth', None)
    order = request.args.get('order', None)
    offset = request.args.get('offset', None)
    limit = request.args.get('limit', None)
    if offset:
        offset = float(offset)
        limit = limit or 50
    elif limit:
        offset = float(0)

    lang = request.args.get('lang', None) or g.locale
    ret = {}
    dataset = "rais"
    if Attr == Wld or Attr == Hs:
        dataset = "secex"
    elif Attr == Course_hedu or Attr == University:
        dataset = "hedu"
    elif Attr == Course_sc or Attr == School:
        dataset = "sc"
    elif Attr == Bra:
        dataset = "population"

    cache_id = request.path + lang
    if depth:
        cache_id = cache_id + "/" + depth
    # first lets test if this query is cached
    cached_q = cached_query(cache_id)
    if cached_q and limit is None:
        ret = make_response(cached_q)
        ret.headers['Content-Encoding'] = 'gzip'
        ret.headers['Content-Length'] = str(len(ret.data))
        return ret

    # if an ID is supplied only return that
    if Attr_id:

        # the '.show.' indicates that we are looking for a specific nesting
        if ".show." in Attr_id:
            this_attr, ret["nesting_level"] = Attr_id.split(".show.")
            # filter table by requested nesting level
            attrs = Attr.query \
                    .filter(Attr.id.startswith(this_attr)) \
                    .filter(func.char_length(Attr.id) == ret["nesting_level"]).all()

        # the 'show.' indicates that we are looking for a specific nesting
        elif "show." in Attr_id:
            ret["nesting_level"] = Attr_id.split(".")[1]
            # filter table by requested nesting level
            attrs = Attr.query.filter(func.char_length(Attr.id) == ret["nesting_level"]).all()

        # the '.' here means we want to see all attrs within a certain distance
        elif "." in Attr_id:
            this_attr, distance = Attr_id.split(".")
            this_attr = Attr.query.get_or_404(this_attr)
            attrs = this_attr.get_neighbors(distance)

        else:
            attrs = [Attr.query.get_or_404(Attr_id)]

        ret["data"] = [fix_name(a.serialize(), lang) for a in attrs]
    # an ID/filter was not provided
    else:
        latest_year = __year_range__[dataset][-1]
        latest_month = False
        if "-" in latest_year:
            latest_year, latest_month = latest_year.split("-")
            latest_month = int(latest_month)
        latest_year = int(latest_year)

        conds = [getattr(Attr_weight_tbl,"{0}_id".format(attr)) == Attr.id, Attr_weight_tbl.year == latest_year]
        if latest_month:
            conds.append(Attr_weight_tbl.month == latest_month)

        query = db.session.query(Attr,Attr_weight_tbl).outerjoin(Attr_weight_tbl, and_(*conds))
        if Attr == School:
            query = query.filter(Attr.is_vocational == 1)

        if depth:
            query = query.filter(func.char_length(Attr.id) == depth)
        else:
            query = query.filter(func.char_length(Attr.id).in_(depths[attr]))

        if order:
            direction = "asc"

            if "." in order:
                o, direction = order.split(".")
            else:
                o = order

            if o == "name":
                o = "name_{0}".format(lang)

            if o == Attr_weight_col:
                order_table  = Attr_weight_tbl
            else:
                order_table = Attr

            if direction == "asc":
                query = query.order_by(asc(getattr(order_table,o)))
            elif direction == "desc":
                query = query.order_by(desc(getattr(order_table,o)))

        if limit:
            query = query.limit(limit).offset(offset)

        attrs_all = query.all()

        # just get items available in DB
        attrs_w_data = None
        if depth is None and limit is None:
            attrs_w_data = db.session.query(Attr, Attr_weight_tbl) \
                .filter(getattr(Attr_weight_tbl, Attr_weight_mergeid) == Attr.id) \
                .group_by(Attr.id)
            attrs_w_data = [a[0].id for a in attrs_w_data]

        attrs = []

        # all_planning_regions = {}

        for i, a in enumerate(attrs_all):
            b = a[0].serialize()
            if a[1]:
                c = a[1].serialize()
                if Attr_weight_col in c:
                    b[Attr_weight_col] = c[Attr_weight_col]
                else:
                    b[Attr_weight_col] = 0
            else:
                b[Attr_weight_col] = 0
            a = b
            if attrs_w_data:
                a["available"] = False
                if a["id"] in attrs_w_data:
                    a["available"] = True
            # if Attr_weight_col == "population" and len(a["id"]) == 9 and a["id"][:3] == "4mg":
            #     if not all_planning_regions:
            #         all_planning_regions = get_planning_region_map()
            #     if a["id"] in all_planning_regions:
            #         plr = all_planning_regions[a["id"]]
            #         a["plr"] = plr
            if order:
                a["rank"] = int(i+offset+1)
            if attr == "bra" and "id_ibge" not in a:
                a["id_ibge"] = False
            attrs.append(fix_name(a, lang))

        ret["data"] = attrs

    ret = jsonify(ret)
    ret.data = gzip_data(ret.data)

    if limit is None and cached_q is None:
        cached_query(cache_id, ret.data)

    ret.headers['Content-Encoding'] = 'gzip'
    ret.headers['Content-Length'] = str(len(ret.data))

    return ret

@mod.route('/table/<attr>/<depth>/')
def attrs_table(attr="bra",depth="2"):
    g.page_type = "attrs"
    data_url = "/attrs/{0}/?depth={1}".format(attr,depth)
    return render_template("general/table.html", data_url=data_url)

@mod.route('/search/<term>/')
@view_cache.cached(timeout=86400, key_prefix=api_cache_key("search"))
def attrs_search(term=None):
    result = []
    lang = request.args.get('lang', 'en') or g.locale
    name_col = Search.name_en if lang == 'en' else Search.name_pt
    profiles = Search.query.filter(name_col.like(u'%{}%'.format(term))).order_by(Search.weight.desc()).limit(20)
    result = [p.serialize(lang == "pt") for p in profiles]
    ret = jsonify({"activities":result})
    return ret
<|endoftext|>


<commit_before>"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

Written (W) 2013 Heiko Strathmann
"""
from abc import abstractmethod
from numpy.lib.function_base import delete
from numpy.ma.core import mean, zeros, log, arange, std
from numpy.random import permutation, rand
import logging

class RussianRoulette(object):
    def __init__(self, threshold, block_size=1):
        self.threshold = threshold
        self.group_size = block_size

    @abstractmethod
    def get_estimate(self, estimates, index):
        start_idx = index * self.block_size
        stop_idx = index * self.block_size + self.block_size

        # if there are enough samples, use them, sub-sample if not
        if stop_idx < len(estimates):
            logging.info("Averaging over %d samples from index %d to %d" %
                         (self.block_size, start_idx, stop_idx))
            indices = arange(start_idx, stop_idx)
        else:
            logging.info("Averaging over a random subset of %d samples" %
                         self.block_size)

            indices = permutation(len(estimates))[:self.block_size]

        return mean(estimates[indices])

    def exponential(self, estimates):
        logging.debug("Entering")

        # find a strict lower bound on the estimates and remove it from list
        bound = estimates.min()
        bound_idx = estimates.argmin()
        estimates = delete(estimates, bound_idx)
        estimates = estimates - bound

        # find an integer close to the mean of the transformed estimates and divide
        E = max(int(round(mean(estimates))), 1)
        estimates = estimates / E

        logging.info("Using %f as lower bound on estimates" % bound)
        logging.info("Computing product of E=%d RR estimates" % E)
        logging.info("Std-deviation after scaling is" % std(estimates))

        # index for iterating through the used estimates
        # (might be averaged, so might be lower than the number of available estimates
        # if the block size is greater than one
        estimate_idx = 0

        samples = zeros(E)
        for iteration in range(E):
            weight = 1

            # start with x^0 which is 1
            samples[iteration] = 1
            term = 1

            # index for computed samples
            series_term_idx = 1

            while weight > 0:
                # update current term of infinite series
                # average over block
                x_inner = self.get_estimate(estimates, estimate_idx)
                term *= (x_inner / series_term_idx)

                # if summation has reached threshold, update weights
                if abs(term) < self.threshold:
                    q = term / self.threshold
                    if rand() < q:
                        # continue and update weight
                        weight = weight / q
                    else:
                        # stop summation
                        weight = 0

                samples[iteration] += weight * term;
                estimate_idx += 1
                series_term_idx += 1

            logging.info("RR estimate %d/%d took %d series terms" %
                         (iteration, E, series_term_idx))
        # now put things together. Note that samples contains an unbiased estimate
        # which might be quite small. However, due to the removal of the bound,
        # this will not cause an underflow and we can just take the log.
        logging.debug("Leaving")
        return bound + sum(log(samples));
<commit_msg>typo bugfix<commit_after>"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or
(at your option) any later version.

Written (W) 2013 Heiko Strathmann
"""
from abc import abstractmethod
from numpy.lib.function_base import delete
from numpy.ma.core import mean, zeros, log, arange, std
from numpy.random import permutation, rand
import logging

class RussianRoulette(object):
    def __init__(self, threshold, block_size=1):
        self.threshold = threshold
        self.block_size = block_size

    @abstractmethod
    def get_estimate(self, estimates, index):
        start_idx = index * self.block_size
        stop_idx = index * self.block_size + self.block_size

        # if there are enough samples, use them, sub-sample if not
        if stop_idx < len(estimates):
            logging.info("Averaging over %d samples from index %d to %d" %
                         (self.block_size, start_idx, stop_idx))
            indices = arange(start_idx, stop_idx)
        else:
            logging.info("Averaging over a random subset of %d samples" %
                         self.block_size)

            indices = permutation(len(estimates))[:self.block_size]

        return mean(estimates[indices])

    def exponential(self, estimates):
        logging.debug("Entering")

        # find a strict lower bound on the estimates and remove it from list
        bound = estimates.min()
        bound_idx = estimates.argmin()
        estimates = delete(estimates, bound_idx)
        estimates = estimates - bound

        # find an integer close to the mean of the transformed estimates and divide
        E = max(int(round(mean(estimates))), 1)
        estimates = estimates / E

        logging.info("Using %f as lower bound on estimates" % bound)
        logging.info("Computing product of E=%d RR estimates" % E)
        logging.info("Std-deviation after scaling is" % std(estimates))

        # index for iterating through the used estimates
        # (might be averaged, so might be lower than the number of available estimates
        # if the block size is greater than one
        estimate_idx = 0

        samples = zeros(E)
        for iteration in range(E):
            weight = 1

            # start with x^0 which is 1
            samples[iteration] = 1
            term = 1

            # index for computed samples
            series_term_idx = 1

            while weight > 0:
                # update current term of infinite series
                # average over block
                x_inner = self.get_estimate(estimates, estimate_idx)
                term *= (x_inner / series_term_idx)

                # if summation has reached threshold, update weights
                if abs(term) < self.threshold:
                    q = term / self.threshold
                    if rand() < q:
                        # continue and update weight
                        weight = weight / q
                    else:
                        # stop summation
                        weight = 0

                samples[iteration] += weight * term;
                estimate_idx += 1
                series_term_idx += 1

            logging.info("RR estimate %d/%d took %d series terms" %
                         (iteration, E, series_term_idx))
        # now put things together. Note that samples contains an unbiased estimate
        # which might be quite small. However, due to the removal of the bound,
        # this will not cause an underflow and we can just take the log.
        logging.debug("Leaving")
        return bound + sum(log(samples));
<|endoftext|>