<commit_before># -*- coding: utf-8 -*-
'''
Jobber Behaviors
'''
# pylint: disable=W0232

# Import python libs
from __future__ import absolute_import
import os
import sys
import types
import logging
import threading
import traceback
import multiprocessing
import subprocess
import json
from collections import deque

# Import salt libs
import salt.daemons.masterapi
import salt.utils.args
import salt.transport
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard

from salt.utils import kinds
from salt.utils.event import tagify

from salt.exceptions import (
        CommandExecutionError, CommandNotFoundError, SaltInvocationError)

# Import ioflo libs
import ioflo.base.deeding

from ioflo.base.consoling import getConsole
console = getConsole()
log = logging.getLogger(__name__)

@ioflo.base.deeding.deedify(
        'SaltRaetShellJobberCheck',
        ioinits={'opts': '.salt.opts',
                 'grains': '.salt.grains',
                 'fun': '.salt.var.fun',
                 'matcher': '.salt.master',
                 'shells': '.salt.var.shells',
                 'stack': '.salt.road.manor.stack'})
def jobber_check(self):
    '''
    Iterate over the shell jobbers and return the ones that have finished
    '''
    for jid in self.shells.value:
        if isinstance(self.shells.value[jid]['proc'].poll(), int):
            data = self.shells.value.pop(jid)
            stdout, stderr = data['proc'].communicate()
            ret = json.loads(stdout)
            route = {'src': (self.stack.value.local.name, 'manor', 'jid_ret'),
                     'dst': (data['msg']['route']['src'][0], None, 'remote_cmd')}
            ret['cmd'] = '_return'
            ret['id'] = self.opts.value['id']
            msg = {'route': route, 'load': ret}
            self.stack.value.transmit(msg, self.stack.value.fetchUidByName('manor'))

@ioflo.base.deeding.deedify(
        'SaltRaetShellJobber',
        ioinits={'opts': '.salt.opts',
                 'grains': '.salt.grains',
                 'fun': '.salt.var.fun',
                 'matcher': '.salt.master',
                 'shells': {'ipath': '.salt.var.shells', 'ival': {}}})
def shell_jobber(self):
    '''
    Shell jobber start!
    '''
    while self.fun.value:
        msg = self.fun.value.popleft()
        data = msg.get('pub')
        match = getattr(
                self.matcher.value,
                '{0}_match'.format(
                    data.get('tgt_type', 'glob')
                    )
                )(data['tgt'])
        if not match:
            continue
        fun = data['fun']
        if fun in self.modules.value:
            func = self.modules.value[fun]
        else:
            continue
        args, kwargs = salt.minion.load_args_and_kwargs(
            func,
            salt.utils.args.parse_input(data['arg']),
            data)
        cmd = ['salt-call',
               '--out', 'json',
               '--metadata',
               '-c', salt.syspaths.CONFIG_DIR]
        if 'return' in data:
            cmd.append('--return')
            cmd.append(data['return'])
        cmd.append(fun)
        for arg in args:
            cmd.append(arg)
        for key in kwargs:
            cmd.append('{0}={1}'.format(key, kwargs[key]))
        que = {'pub': data}
        que['proc'] = subprocess.Popen(
                cmd,
                shell=False,
                stderr=subprocess.PIPE,
                stdout=subprocess.PIPE)
        self.shells.value[data['jid']] = que

class SaltRaetNixJobber(ioflo.base.deeding.Deed):
    '''
    Execute a function call job on a minion on a *nix based system
    FloScript:

    do salt raet nix jobber

    '''
    Ioinits = {'opts_store': '.salt.opts',
               'grains': '.salt.grains',
               'modules': '.salt.loader.modules',
               'returners': '.salt.loader.returners',
               'fun': '.salt.var.fun',
               'matcher': '.salt.matcher',
               'executors': '.salt.track.executors',
               'road_stack': '.salt.road.manor.stack', }

    def _prepare(self):
        '''
        Map opts for convenience
        '''
        self.opts = self.opts_store.value
        self.proc_dir = salt.minion.get_proc_dir(self.opts['cachedir'])
        self.serial = salt.payload.Serial(self.opts)
        self.executors.value = {}

    def _setup_jobber_stack(self):
        '''
        Setup and return the LaneStack and Yard used by the jobber yard
        to communicate with the minion manor yard

        '''
        role = self.opts.get('id', '')
        if not role:
            emsg = ("Missing role required to setup Jobber Lane.")
            log.error(emsg + "\n")
            raise ValueError(emsg)

        kind = self.opts['__role']
        if kind not in kinds.APPL_KINDS:
            emsg = ("Invalid application kind = '{0}' for Jobber lane.".format(kind))
            log.error(emsg + "\n")
            raise ValueError(emsg)

        if kind == 'minion':
            lanename = "{0}_{1}".format(role, kind)
        else:
            emsg = ("Unsupported application kind = '{0}' for Jobber Lane.".format(kind))
            log.error(emsg + '\n')
            raise ValueError(emsg)

        sockdirpath = self.opts['sock_dir']
        name = 'jobber' + nacling.uuid(size=18)
        stack = LaneStack(
                name=name,
                lanename=lanename,
                sockdirpath=sockdirpath)

        stack.Pk = raeting.PackKind.pack.value
        # add remote for the manor yard
        stack.addRemote(RemoteYard(stack=stack,
                                   name='manor',
                                   lanename=lanename,
                                   dirpath=sockdirpath))
        console.concise("Created Jobber Stack {0}\n".format(stack.name))
        return stack

    def _return_pub(self, msg, ret, stack):
        '''
        Send the return data back via the uxd socket
        '''
        route = {'src': (self.road_stack.value.local.name, stack.local.name, 'jid_ret'),
                 'dst': (msg['route']['src'][0], None, 'remote_cmd')}
        mid = self.opts['id']
        ret['cmd'] = '_return'
        ret['id'] = mid
        try:
            oput = self.modules.value[ret['fun']].__outputter__
        except (KeyError, AttributeError, TypeError):
            pass
        else:
            if isinstance(oput, str):
                ret['out'] = oput
        msg = {'route': route, 'load': ret}
        stack.transmit(msg, stack.fetchUidByName('manor'))
        stack.serviceAll()

    def action(self):
        '''
        Pull the queue for functions to execute
        '''
        while self.fun.value:
            msg = self.fun.value.popleft()
            data = msg.get('pub')
            match = getattr(
                    self.matcher.value,
                    '{0}_match'.format(
                        data.get('tgt_type', 'glob')
                        )
                    )(data['tgt'])
            if not match:
                continue
            if 'user' in data:
                log.info(
                        'User {0[user]} Executing command {0[fun]} with jid '
                        '{0[jid]}'.format(data))
            else:
                log.info(
                        'Executing command {0[fun]} with jid {0[jid]}'.format(data)
                        )
            log.debug('Command details {0}'.format(data))

            process = multiprocessing.Process(
                    target=self.proc_run,
                    kwargs={'msg': msg}
                    )
            process.start()
            process.join()

    def proc_run(self, msg):
        '''
        Execute the run in a dedicated process
        '''
        data = msg['pub']
        fn_ = os.path.join(self.proc_dir, data['jid'])
        self.opts['__ex_id'] = data['jid']
        salt.utils.daemonize_if(self.opts)

        salt.transport.jobber_stack = stack = self._setup_jobber_stack()
        # set up return destination from source
        src_estate, src_yard, src_share = msg['route']['src']
        salt.transport.jobber_estate_name = src_estate
        salt.transport.jobber_yard_name = src_yard

        sdata = {'pid': os.getpid()}
        sdata.update(data)
        with salt.utils.fopen(fn_, 'w+') as fp_:
            fp_.write(self.serial.dumps(sdata))
        ret = {'success': False}
        function_name = data['fun']
        if function_name in self.modules.value:
            try:
                func = self.modules.value[data['fun']]
                args, kwargs = salt.minion.load_args_and_kwargs(
                    func,
                    salt.utils.args.parse_input(data['arg']),
                    data)
                sys.modules[func.__module__].__context__['retcode'] = 0
                if self.opts.get('sudo_user', ''):
                    sudo_runas = self.opts.get('sudo_user')
                    if 'sudo.salt_call' in self.modules.value:
                        return_data = self.modules.value['sudo.salt_call'](
                                sudo_runas,
                                data['fun'],
                                *args,
                                **kwargs)
                else:
                    return_data = func(*args, **kwargs)
                if isinstance(return_data, types.GeneratorType):
                    ind = 0
                    iret = {}
                    for single in return_data:
                        if isinstance(single, dict) and isinstance(iret, list):
                            iret.update(single)
                        else:
                            if not iret:
                                iret = []
                            iret.append(single)
                        tag = tagify(
                                [data['jid'], 'prog', self.opts['id'], str(ind)],
                                'job')
                        event_data = {'return': single}
                        self._fire_master(event_data, tag)  # Need to look into this
                        ind += 1
                    ret['return'] = iret
                else:
                    ret['return'] = return_data
                ret['retcode'] = sys.modules[func.__module__].__context__.get(
                    'retcode',
                    0
                )
                ret['success'] = True
            except CommandNotFoundError as exc:
                msg = 'Command required for {0!r} not found'.format(
                    function_name
                )
                log.debug(msg, exc_info=True)
                ret['return'] = '{0}: {1}'.format(msg, exc)
            except CommandExecutionError as exc:
                log.error(
                    'A command in {0!r} had a problem: {1}'.format(
                        function_name,
                        exc
                    ),
                    exc_info_on_loglevel=logging.DEBUG
                )
                ret['return'] = 'ERROR: {0}'.format(exc)
            except SaltInvocationError as exc:
                log.error(
                    'Problem executing {0!r}: {1}'.format(
                        function_name,
                        exc
                    ),
                    exc_info_on_loglevel=logging.DEBUG
                )
                ret['return'] = 'ERROR executing {0!r}: {1}'.format(
                    function_name, exc
                )
            except TypeError as exc:
                msg = ('TypeError encountered executing {0}: {1}. See '
                       'debug log for more info.').format(function_name, exc)
                log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
                ret['return'] = msg
            except Exception:
                msg = 'The minion function caused an exception'
                log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
                ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc())
        else:
            ret['return'] = '{0!r} is not available.'.format(function_name)

        ret['jid'] = data['jid']
        ret['fun'] = data['fun']
        ret['fun_args'] = data['arg']
        self._return_pub(msg, ret, stack)
        if data['ret']:
            ret['id'] = self.opts['id']
            for returner in set(data['ret'].split(',')):
                try:
                    self.returners.value['{0}.returner'.format(
                        returner
                    )](ret)
                except Exception as exc:
                    log.error(
                        'The return failed for job {0} {1}'.format(
                        data['jid'],
                        exc
                        )
                    )
        console.concise("Closing Jobber Stack {0}\n".format(stack.name))
        stack.server.close()
        salt.transport.jobber_stack = None
<commit_msg>pass the msg into the tracking value<commit_after># -*- coding: utf-8 -*-
'''
Jobber Behaviors
'''
# pylint: disable=W0232

# Import python libs
from __future__ import absolute_import
import os
import sys
import types
import logging
import traceback
import multiprocessing
import subprocess
import json

# Import salt libs
import salt.daemons.masterapi
import salt.utils.args
import salt.transport
from raet import raeting, nacling
from raet.lane.stacking import LaneStack
from raet.lane.yarding import RemoteYard

from salt.utils import kinds
from salt.utils.event import tagify

from salt.exceptions import (
        CommandExecutionError, CommandNotFoundError, SaltInvocationError)

# Import ioflo libs
import ioflo.base.deeding

from ioflo.base.consoling import getConsole
console = getConsole()
log = logging.getLogger(__name__)

@ioflo.base.deeding.deedify(
        'SaltRaetShellJobberCheck',
        ioinits={'opts': '.salt.opts',
                 'grains': '.salt.grains',
                 'fun': '.salt.var.fun',
                 'matcher': '.salt.master',
                 'shells': '.salt.var.shells',
                 'stack': '.salt.road.manor.stack'})
def jobber_check(self):
    '''
    Iterate over the shell jobbers and return the ones that have finished
    '''
    for jid in self.shells.value:
        if isinstance(self.shells.value[jid]['proc'].poll(), int):
            data = self.shells.value.pop(jid)
            stdout, stderr = data['proc'].communicate()
            ret = json.loads(stdout)
            route = {'src': (self.stack.value.local.name, 'manor', 'jid_ret'),
                     'dst': (data['msg']['route']['src'][0], None, 'remote_cmd')}
            ret['cmd'] = '_return'
            ret['id'] = self.opts.value['id']
            msg = {'route': route, 'load': ret}
            self.stack.value.transmit(
                    msg,
                    self.stack.value.nameRemotes.get(data['msg']['route']['src'][0]))

@ioflo.base.deeding.deedify(
        'SaltRaetShellJobber',
        ioinits={'opts': '.salt.opts',
                 'grains': '.salt.grains',
                 'fun': '.salt.var.fun',
                 'matcher': '.salt.master',
                 'shells': {'ipath': '.salt.var.shells', 'ival': {}}})
def shell_jobber(self):
    '''
    Shell jobber start!
    '''
    while self.fun.value:
        msg = self.fun.value.popleft()
        data = msg.get('pub')
        match = getattr(
                self.matcher.value,
                '{0}_match'.format(
                    data.get('tgt_type', 'glob')
                    )
                )(data['tgt'])
        if not match:
            continue
        fun = data['fun']
        if fun in self.modules.value:
            func = self.modules.value[fun]
        else:
            continue
        args, kwargs = salt.minion.load_args_and_kwargs(
            func,
            salt.utils.args.parse_input(data['arg']),
            data)
        cmd = ['salt-call',
               '--out', 'json',
               '--metadata',
               '-c', salt.syspaths.CONFIG_DIR]
        if 'return' in data:
            cmd.append('--return')
            cmd.append(data['return'])
        cmd.append(fun)
        for arg in args:
            cmd.append(arg)
        for key in kwargs:
            cmd.append('{0}={1}'.format(key, kwargs[key]))
        que = {'pub': data,
               'msg': msg}
        que['proc'] = subprocess.Popen(
                cmd,
                shell=False,
                stderr=subprocess.PIPE,
                stdout=subprocess.PIPE)
        self.shells.value[data['jid']] = que

class SaltRaetNixJobber(ioflo.base.deeding.Deed):
    '''
    Execute a function call job on a minion on a *nix based system
    FloScript:

    do salt raet nix jobber

    '''
    Ioinits = {'opts_store': '.salt.opts',
               'grains': '.salt.grains',
               'modules': '.salt.loader.modules',
               'returners': '.salt.loader.returners',
               'fun': '.salt.var.fun',
               'matcher': '.salt.matcher',
               'executors': '.salt.track.executors',
               'road_stack': '.salt.road.manor.stack', }

    def _prepare(self):
        '''
        Map opts for convenience
        '''
        self.opts = self.opts_store.value
        self.proc_dir = salt.minion.get_proc_dir(self.opts['cachedir'])
        self.serial = salt.payload.Serial(self.opts)
        self.executors.value = {}

    def _setup_jobber_stack(self):
        '''
        Setup and return the LaneStack and Yard used by the jobber yard
        to communicate with the minion manor yard

        '''
        role = self.opts.get('id', '')
        if not role:
            emsg = ("Missing role required to setup Jobber Lane.")
            log.error(emsg + "\n")
            raise ValueError(emsg)

        kind = self.opts['__role']
        if kind not in kinds.APPL_KINDS:
            emsg = ("Invalid application kind = '{0}' for Jobber lane.".format(kind))
            log.error(emsg + "\n")
            raise ValueError(emsg)

        if kind == 'minion':
            lanename = "{0}_{1}".format(role, kind)
        else:
            emsg = ("Unsupported application kind = '{0}' for Jobber Lane.".format(kind))
            log.error(emsg + '\n')
            raise ValueError(emsg)

        sockdirpath = self.opts['sock_dir']
        name = 'jobber' + nacling.uuid(size=18)
        stack = LaneStack(
                name=name,
                lanename=lanename,
                sockdirpath=sockdirpath)

        stack.Pk = raeting.PackKind.pack.value
        # add remote for the manor yard
        stack.addRemote(RemoteYard(stack=stack,
                                   name='manor',
                                   lanename=lanename,
                                   dirpath=sockdirpath))
        console.concise("Created Jobber Stack {0}\n".format(stack.name))
        return stack

    def _return_pub(self, msg, ret, stack):
        '''
        Send the return data back via the uxd socket
        '''
        route = {'src': (self.road_stack.value.local.name, stack.local.name, 'jid_ret'),
                 'dst': (msg['route']['src'][0], None, 'remote_cmd')}
        mid = self.opts['id']
        ret['cmd'] = '_return'
        ret['id'] = mid
        try:
            oput = self.modules.value[ret['fun']].__outputter__
        except (KeyError, AttributeError, TypeError):
            pass
        else:
            if isinstance(oput, str):
                ret['out'] = oput
        msg = {'route': route, 'load': ret}
        stack.transmit(msg, stack.fetchUidByName('manor'))
        stack.serviceAll()

    def action(self):
        '''
        Pull the queue for functions to execute
        '''
        while self.fun.value:
            msg = self.fun.value.popleft()
            data = msg.get('pub')
            match = getattr(
                    self.matcher.value,
                    '{0}_match'.format(
                        data.get('tgt_type', 'glob')
                        )
                    )(data['tgt'])
            if not match:
                continue
            if 'user' in data:
                log.info(
                        'User {0[user]} Executing command {0[fun]} with jid '
                        '{0[jid]}'.format(data))
            else:
                log.info(
                        'Executing command {0[fun]} with jid {0[jid]}'.format(data)
                        )
            log.debug('Command details {0}'.format(data))

            process = multiprocessing.Process(
                    target=self.proc_run,
                    kwargs={'msg': msg}
                    )
            process.start()
            process.join()

    def proc_run(self, msg):
        '''
        Execute the run in a dedicated process
        '''
        data = msg['pub']
        fn_ = os.path.join(self.proc_dir, data['jid'])
        self.opts['__ex_id'] = data['jid']
        salt.utils.daemonize_if(self.opts)

        salt.transport.jobber_stack = stack = self._setup_jobber_stack()
        # set up return destination from source
        src_estate, src_yard, src_share = msg['route']['src']
        salt.transport.jobber_estate_name = src_estate
        salt.transport.jobber_yard_name = src_yard

        sdata = {'pid': os.getpid()}
        sdata.update(data)
        with salt.utils.fopen(fn_, 'w+') as fp_:
            fp_.write(self.serial.dumps(sdata))
        ret = {'success': False}
        function_name = data['fun']
        if function_name in self.modules.value:
            try:
                func = self.modules.value[data['fun']]
                args, kwargs = salt.minion.load_args_and_kwargs(
                    func,
                    salt.utils.args.parse_input(data['arg']),
                    data)
                sys.modules[func.__module__].__context__['retcode'] = 0
                if self.opts.get('sudo_user', ''):
                    sudo_runas = self.opts.get('sudo_user')
                    if 'sudo.salt_call' in self.modules.value:
                        return_data = self.modules.value['sudo.salt_call'](
                                sudo_runas,
                                data['fun'],
                                *args,
                                **kwargs)
                else:
                    return_data = func(*args, **kwargs)
                if isinstance(return_data, types.GeneratorType):
                    ind = 0
                    iret = {}
                    for single in return_data:
                        if isinstance(single, dict) and isinstance(iret, list):
                            iret.update(single)
                        else:
                            if not iret:
                                iret = []
                            iret.append(single)
                        tag = tagify(
                                [data['jid'], 'prog', self.opts['id'], str(ind)],
                                'job')
                        event_data = {'return': single}
                        self._fire_master(event_data, tag)  # Need to look into this
                        ind += 1
                    ret['return'] = iret
                else:
                    ret['return'] = return_data
                ret['retcode'] = sys.modules[func.__module__].__context__.get(
                    'retcode',
                    0
                )
                ret['success'] = True
            except CommandNotFoundError as exc:
                msg = 'Command required for {0!r} not found'.format(
                    function_name
                )
                log.debug(msg, exc_info=True)
                ret['return'] = '{0}: {1}'.format(msg, exc)
            except CommandExecutionError as exc:
                log.error(
                    'A command in {0!r} had a problem: {1}'.format(
                        function_name,
                        exc
                    ),
                    exc_info_on_loglevel=logging.DEBUG
                )
                ret['return'] = 'ERROR: {0}'.format(exc)
            except SaltInvocationError as exc:
                log.error(
                    'Problem executing {0!r}: {1}'.format(
                        function_name,
                        exc
                    ),
                    exc_info_on_loglevel=logging.DEBUG
                )
                ret['return'] = 'ERROR executing {0!r}: {1}'.format(
                    function_name, exc
                )
            except TypeError as exc:
                msg = ('TypeError encountered executing {0}: {1}. See '
                       'debug log for more info.').format(function_name, exc)
                log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
                ret['return'] = msg
            except Exception:
                msg = 'The minion function caused an exception'
                log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
                ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc())
        else:
            ret['return'] = '{0!r} is not available.'.format(function_name)

        ret['jid'] = data['jid']
        ret['fun'] = data['fun']
        ret['fun_args'] = data['arg']
        self._return_pub(msg, ret, stack)
        if data['ret']:
            ret['id'] = self.opts['id']
            for returner in set(data['ret'].split(',')):
                try:
                    self.returners.value['{0}.returner'.format(
                        returner
                    )](ret)
                except Exception as exc:
                    log.error(
                        'The return failed for job {0} {1}'.format(
                        data['jid'],
                        exc
                        )
                    )
        console.concise("Closing Jobber Stack {0}\n".format(stack.name))
        stack.server.close()
        salt.transport.jobber_stack = None
<|endoftext|>


<commit_before>'''
Package support for FreeBSD
'''

# Import python libs
import logging
import os

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

def __virtual__():
    '''
    Set the virtual pkg module if the os is FreeBSD
    '''
    return 'pkg' if __grains__['os'] == 'FreeBSD' else False

def _check_pkgng():
    '''
    Looks to see if pkgng is being used by checking if database exists
    '''
    return os.path.isfile('/var/db/pkg/local.sqlite')

@salt.utils.memoize
def _cmd(cmd):
    return salt.utils.which(cmd)

def search(pkg_name):
    '''
    Use `pkg search` if pkg is being used.

    CLI Example::

        salt '*' pkg.search 'mysql-server'
    '''
    if _check_pkgng():
        res = __salt__['cmd.run']('{0} search {1}'.format(_cmd('pkg'),
                                                          pkg_name
                                                          ))
        res = [x for x in res.splitlines()]
        return {"Results": res}

def latest_version(*names, **kwargs):
    '''
    Return the latest version of the named package available for upgrade or
    installation. If more than one package name is specified, a dict of
    name/version pairs is returned.

    If the latest version of a given package is already installed, an empty
    string will be returned for that package.

    CLI Example::

        salt '*' pkg.latest_version <package name>
        salt '*' pkg.latest_version <package1> <package2> <package3> ...
    '''

    ret = {}

    if _check_pkgng():
        for line in __salt__['cmd.run_stdout']('{0} upgrade -nq'.format(
            _cmd('pkg'))
        ).splitlines():
            if not line.startswith('\t'):
                continue
            line = line.strip()
            if line.startswith('Installing'):
                _, pkg, ver = line.split()
                pkg = pkg.rstrip(':')
            elif line.startswith('Upgrading'):
                _, pkg, _, _, ver = line.split()
                pkg = pkg.rstrip(':')
            elif line.startswith('Reinstalling'):
                _, pkgver = line.split()
                comps = pkgver.split('-')
                pkg = ''.join(comps[:-1])
                ver = comps[-1]
            else:
                # unexpected string
                continue
            if pkg in names:
                ret[pkg] = ver

        # keep pkg.latest culm
        for pkg in set(names) - set(ret) - set(list_pkgs()):
            for line in __salt__['cmd.run']('{0} search -fe {1}'.format(
                _cmd('pkg'), pkg)
            ).splitlines():
                if line.startswith('Version'):
                    _, _, ver = line.split()[:3]
                    ret[pkg] = ver
                    break

    ret.update(dict.fromkeys(set(names) - set(ret), ''))

    if len(names) == 1:
        return ret.values()[0]

    return ret

# available_version is being deprecated
available_version = latest_version

def version(*names, **kwargs):
    '''
    Returns a string representing the package version or an empty string if not
    installed. If more than one package name is specified, a dict of
    name/version pairs is returned.

    CLI Example::

        salt '*' pkg.version <package name>
        salt '*' pkg.version <package1> <package2> <package3> ...
    '''
    return __salt__['pkg_resource.version'](*names, **kwargs)

def refresh_db():
    '''
    Use pkg update to get latest repo.txz when using pkgng, else update the
    ports tree with portsnap otherwise. If the ports tree does not exist it
    will be downloaded and set up.

    CLI Example::

        salt '*' pkg.refresh_db
    '''
    if _check_pkgng():
        __salt__['cmd.run']('{0} update'.format(_cmd('pkg')))
    return {}

def list_pkgs(versions_as_list=False):
    '''
    List the packages currently installed as a dict::

        {'<package_name>': '<version>'}

    CLI Example::

        salt '*' pkg.list_pkgs
    '''
    versions_as_list = salt.utils.is_true(versions_as_list)
    if _check_pkgng():
        pkg_command = '{0} info'.format(_cmd('pkg'))
    else:
        pkg_command = '{0}'.format(_cmd('pkg_info'))
    ret = {}
    for line in __salt__['cmd.run'](pkg_command).splitlines():
        if not line:
            continue
        pkg, ver = line.split(' ')[0].rsplit('-', 1)
        __salt__['pkg_resource.add_pkg'](ret, pkg, ver)

    __salt__['pkg_resource.sort_pkglist'](ret)
    if not versions_as_list:
        __salt__['pkg_resource.stringify'](ret)
    return ret

def install(name=None, refresh=False, fromrepo=None,
            pkgs=None, sources=None, **kwargs):
    '''
    Install the passed package

    name
        The name of the package to be installed.

    refresh
        Whether or not to refresh the package database before installing.

    fromrepo
        Specify a package repository to install from.

    Multiple Package Installation Options:

    pkgs
        A list of packages to install from a software repository. Must be
        passed as a python list.

        CLI Example::

            salt '*' pkg.install pkgs='["foo","bar"]'

    sources
        A list of packages to install. Must be passed as a list of dicts,
        with the keys being package names, and the values being the source URI
        or local path to the package.

        CLI Example::

            salt '*' pkg.install sources='[{"foo": "salt://foo.deb"},{"bar": "salt://bar.deb"}]'

    Return a dict containing the new package names and versions::

        {'<package>': {'old': '<old-version>',
                       'new': '<new-version>'}}

    CLI Example::

        salt '*' pkg.install <package name>
    '''
    pkg_params, pkg_type = __salt__['pkg_resource.parse_targets'](name,
                                                                  pkgs,
                                                                  sources)

    # Support old "repo" argument
    repo = kwargs.get('repo', '')
    if not fromrepo and repo:
        fromrepo = repo

    if not pkg_params:
        return {}

    env = []
    args = []
    if _check_pkgng():
        cmd = _cmd('pkg')
        if fromrepo:
            log.info('Setting PACKAGESITE={0}'.format(fromrepo))
            env.append(('PACKAGESITE', fromrepo))
    else:
        cmd = _cmd('pkg_add')
        if fromrepo:
            log.info('Setting PACKAGEROOT={0}'.format(fromrepo))
            env.append(('PACKAGEROOT', fromrepo))

    if pkg_type == 'file':
        if _check_pkgng():
            env.append(('ASSUME_ALWAYS_YES', 'yes'))  # might be fixed later
            args.append('add')
    elif pkg_type == 'repository':
        if _check_pkgng():
            args.extend(('install', '-y'))  # Assume yes when asked
            if not refresh:
                args.append('-L')  # do not update repo db
        else:
            args.append('-r')  # use remote repo

    args.extend(pkg_params)

    old = list_pkgs()
    __salt__['cmd.run_all']('{0} {1}'.format(cmd, ' '.join(args)), env=env)
    new = list_pkgs()

    rehash()
    return __salt__['pkg_resource.find_changes'](old, new)

def upgrade():
    '''
    Run pkg upgrade, if pkgng used. Otherwise do nothing

    Return a dict containing the new package names and versions::

        {'<package>': {'old': '<old-version>',
                       'new': '<new-version>'}}

    CLI Example::

        salt '*' pkg.upgrade
    '''

    if not _check_pkgng():
        # There is not easy way to upgrade packages with old package system
        return {}

    old = list_pkgs()
    __salt__['cmd.retcode']('{0} upgrade -y'.format(_cmd('pkg')))
    new = list_pkgs()
    return __salt__['pkg_resource.find_changes'](old, new)

def remove(name=None, pkgs=None, **kwargs):
    '''
    Remove a single package.

    name : None
        The name of the package to be deleted.

    pkgs : None
        A list of packages to delete. Must be passed as a python list.

        CLI Example::

            salt '*' pkg.remove pkgs='["foo","bar"]'

    Returns a list containing the removed packages.

    CLI Example::

        salt '*' pkg.remove <package name>
    '''
    pkg_params, pkg_type = __salt__['pkg_resource.parse_targets'](name,
                                                                  pkgs)
    if not pkg_params:
        return {}

    old = list_pkgs()
    args = []

    for param in pkg_params:
        ver = old.get(param, [])
        if not ver:
            continue
        if isinstance(ver, list):
            args.extend(['{0}-{1}'.format(param, v) for v in ver])
        else:
            args.append('{0}-{1}'.format(param, ver))

    if not args:
        return {}

    for_remove = ' '.join(args)

    if _check_pkgng():
        cmd = '{0} remove -y {1}'.format(_cmd('pkg'), for_remove)
    else:
        cmd = '{0} {1}'.format(_cmd('pkg_remove'), for_remove)

    __salt__['cmd.run_all'](cmd)

    new = list_pkgs()

    return __salt__['pkg_resource.find_changes'](old, new)

def purge(name, **kwargs):
    '''
    Remove a single package with pkg_delete

    Returns a list containing the removed packages.

    CLI Example::

        salt '*' pkg.purge <package name>
    '''
    return remove(name)

def rehash():
    '''
    Recomputes internal hash table for the PATH variable.
    Use whenever a new command is created during the current
    session.

    CLI Example::

        salt '*' pkg.rehash
    '''
    shell = __salt__['cmd.run']('echo $SHELL').split('/')
    if shell[len(shell) - 1] in ['csh', 'tcsh']:
        __salt__['cmd.run']('rehash')

def perform_cmp(pkg1='', pkg2=''):
    '''
    Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if
    pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem
    making the comparison.

    CLI Example::

        salt '*' pkg.perform_cmp '0.2.4-0' '0.2.4.1-0'
        salt '*' pkg.perform_cmp pkg1='0.2.4-0' pkg2='0.2.4.1-0'
    '''
    return __salt__['pkg_resource.perform_cmp'](pkg1=pkg1, pkg2=pkg2)

def compare(pkg1='', oper='==', pkg2=''):
    '''
    Compare two version strings.

    CLI Example::

        salt '*' pkg.compare '0.2.4-0' '<' '0.2.4.1-0'
        salt '*' pkg.compare pkg1='0.2.4-0' oper='<' pkg2='0.2.4.1-0'
    '''
    return __salt__['pkg_resource.compare'](pkg1=pkg1, oper=oper, pkg2=pkg2)

def file_list(*packages):
    '''
    List the files that belong to a package. Not specifying any packages will
    return a list of _every_ file on the system's package database (not
    generally recommended).

    CLI Examples::

        salt '*' pkg.file_list httpd
        salt '*' pkg.file_list httpd postfix
        salt '*' pkg.file_list
    '''
    ret = file_dict(*packages)
    files = []
    for pkg_files in ret['files'].values():
        files.extend(pkg_files)
    ret['files'] = files
    return ret

def file_dict(*packages):
    '''
    List the files that belong to a package, grouped by package. Not
    specifying any packages will return a list of _every_ file on the
    system's package database (not generally recommended).

    CLI Examples::

        salt '*' pkg.file_list httpd
        salt '*' pkg.file_list httpd postfix
        salt '*' pkg.file_list
    '''
    errors = []
    files = {}

    if _check_pkgng():
        if packages:
            match_pattern = '%n ~ {0}'
            matches = [match_pattern.format(p) for p in packages]

            cmd = '{0} query -e \'{1}\' \'%n %Fp\''.format(
                _cmd('pkg'), ' || '.join(matches))
        else:
            cmd = '{0} query -a \'%n %Fp\''.format(_cmd('pkg'))

        for line in __salt__['cmd.run_stdout'](cmd).splitlines():
            pkg, fn = line.split(' ', 1)
            if pkg not in files:
                files[pkg] = []
            files[pkg].append(fn)
    else:
        if packages:
            match_pattern = '\'{0}-[0-9]*\''
            matches = [match_pattern.format(p) for p in packages]

            cmd = '{0} -QL {1}'.format(_cmd('pkg_info'), ' '.join(matches))
        else:
            cmd = '{0} -QLa'.format(_cmd('pkg_info'))

        ret = __salt__['cmd.run_all'](cmd)

        for line in ret['stderr'].splitlines():
            errors.append(line)

        pkg = None
        for line in ret['stdout'].splitlines():
            if pkg is not None and line.startswith('/'):
                files[pkg].append(line)
            elif ':/' in line:
                pkg, fn = line.split(':', 1)
                pkg, ver = pkg.rsplit('-', 1)
                files[pkg] = [fn]
            else:
                continue  # unexpected string
    return {'errors': errors, 'files': files}
<commit_msg>freebsdpkg: multi-pkg for remove/purge, optimizations<commit_after>'''
Package support for FreeBSD
'''

# Import python libs
import copy
import logging
import os

# Import salt libs
import salt.utils

log = logging.getLogger(__name__)

def __virtual__():
    '''
    Set the virtual pkg module if the os is FreeBSD
    '''
    return 'pkg' if __grains__['os'] == 'FreeBSD' else False

def _check_pkgng():
    '''
    Looks to see if pkgng is being used by checking if database exists
    '''
    return os.path.isfile('/var/db/pkg/local.sqlite')

@salt.utils.memoize
def _cmd(cmd):
    return salt.utils.which(cmd)

def search(pkg_name):
    '''
    Use `pkg search` if pkg is being used.

    CLI Example::

        salt '*' pkg.search 'mysql-server'
    '''
    if _check_pkgng():
        res = __salt__['cmd.run_all']('{0} search {1}'.format(_cmd('pkg'),
                                                              pkg_name))
        res = [x for x in res.splitlines()]
        return {'Results': res}

def latest_version(*names, **kwargs):
    '''
    Return the latest version of the named package available for upgrade or
    installation. If more than one package name is specified, a dict of
    name/version pairs is returned.

    If the latest version of a given package is already installed, an empty
    string will be returned for that package.

    CLI Example::

        salt '*' pkg.latest_version <package name>
        salt '*' pkg.latest_version <package1> <package2> <package3> ...
    '''

    ret = {}

    if _check_pkgng():
        for line in __salt__['cmd.run_stdout']('{0} upgrade -nq'.format(
            _cmd('pkg'))
        ).splitlines():
            if not line.startswith('\t'):
                continue
            line = line.strip()
            if line.startswith('Installing'):
                _, pkg, ver = line.split()
                pkg = pkg.rstrip(':')
            elif line.startswith('Upgrading'):
                _, pkg, _, _, ver = line.split()
                pkg = pkg.rstrip(':')
            elif line.startswith('Reinstalling'):
                _, pkgver = line.split()
                comps = pkgver.split('-')
                pkg = ''.join(comps[:-1])
                ver = comps[-1]
            else:
                # unexpected string
                continue
            if pkg in names:
                ret[pkg] = ver

        # keep pkg.latest culm
        for pkg in set(names) - set(ret) - set(list_pkgs()):
            for line in __salt__['cmd.run']('{0} search -fe {1}'.format(
                _cmd('pkg'), pkg)
            ).splitlines():
                if line.startswith('Version'):
                    _, _, ver = line.split()[:3]
                    ret[pkg] = ver
                    break

    ret.update(dict.fromkeys(set(names) - set(ret), ''))

    if len(names) == 1:
        return ret.values()[0]

    return ret

# available_version is being deprecated
available_version = latest_version

def version(*names, **kwargs):
    '''
    Returns a string representing the package version or an empty string if not
    installed. If more than one package name is specified, a dict of
    name/version pairs is returned.

    CLI Example::

        salt '*' pkg.version <package name>
        salt '*' pkg.version <package1> <package2> <package3> ...
    '''
    return __salt__['pkg_resource.version'](*names, **kwargs)

def refresh_db():
    '''
    Use pkg update to get latest repo.txz when using pkgng, else update the
    ports tree with portsnap otherwise. If the ports tree does not exist it
    will be downloaded and set up.

    CLI Example::

        salt '*' pkg.refresh_db
    '''
    if _check_pkgng():
        __salt__['cmd.run_all']('{0} update'.format(_cmd('pkg')))
    return {}

def list_pkgs(versions_as_list=False):
    '''
    List the packages currently installed as a dict::

        {'<package_name>': '<version>'}

    CLI Example::

        salt '*' pkg.list_pkgs
    '''
    versions_as_list = salt.utils.is_true(versions_as_list)

    if 'pkg.list_pkgs' in __context__:
        if versions_as_list:
            return __context__['pkg.list_pkgs']
        else:
            ret = copy.deepcopy(__context__['pkg.list_pkgs'])
            __salt__['pkg_resource.stringify'](ret)
            return ret

    if _check_pkgng():
        pkg_command = '{0} info'.format(_cmd('pkg'))
    else:
        pkg_command = '{0}'.format(_cmd('pkg_info'))
    ret = {}
    for line in __salt__['cmd.run'](pkg_command).splitlines():
        if not line:
            continue
        pkg, ver = line.split(' ')[0].rsplit('-', 1)
        __salt__['pkg_resource.add_pkg'](ret, pkg, ver)

    __salt__['pkg_resource.sort_pkglist'](ret)
    __context__['pkg.list_pkgs'] = ret
    if not versions_as_list:
        __salt__['pkg_resource.stringify'](ret)
    return ret

def install(name=None,
            refresh=False,
            fromrepo=None,
            pkgs=None,
            sources=None,
            **kwargs):
    '''
    Install the passed package

    name
        The name of the package to be installed.

    refresh
        Whether or not to refresh the package database before installing.

    fromrepo
        Specify a package repository to install from.

    Multiple Package Installation Options:

    pkgs
        A list of packages to install from a software repository. Must be
        passed as a python list.

        CLI Example::

            salt '*' pkg.install pkgs='["foo","bar"]'

    sources
        A list of packages to install. Must be passed as a list of dicts,
        with the keys being package names, and the values being the source URI
        or local path to the package.

        CLI Example::

            salt '*' pkg.install sources='[{"foo": "salt://foo.deb"},{"bar": "salt://bar.deb"}]'

    Return a dict containing the new package names and versions::

        {'<package>': {'old': '<old-version>',
                       'new': '<new-version>'}}

    CLI Example::

        salt '*' pkg.install <package name>
    '''
    pkg_params, pkg_type = __salt__['pkg_resource.parse_targets'](name,
                                                                  pkgs,
                                                                  sources)

    # Support old "repo" argument
    repo = kwargs.get('repo', '')
    if not fromrepo and repo:
        fromrepo = repo

    if not pkg_params:
        return {}

    env = []
    args = []
    if _check_pkgng():
        cmd = _cmd('pkg')
        if fromrepo:
            log.info('Setting PACKAGESITE={0}'.format(fromrepo))
            env.append(('PACKAGESITE', fromrepo))
    else:
        cmd = _cmd('pkg_add')
        if fromrepo:
            log.info('Setting PACKAGEROOT={0}'.format(fromrepo))
            env.append(('PACKAGEROOT', fromrepo))

    if pkg_type == 'file':
        if _check_pkgng():
            env.append(('ASSUME_ALWAYS_YES', 'yes'))  # might be fixed later
            args.append('add')
    elif pkg_type == 'repository':
        if _check_pkgng():
            args.extend(('install', '-y'))  # Assume yes when asked
            if not refresh:
                args.append('-L')  # do not update repo db
        else:
            args.append('-r')  # use remote repo

    args.extend(pkg_params)

    old = list_pkgs()
    __salt__['cmd.run_all']('{0} {1}'.format(cmd, ' '.join(args)), env=env)
    __context__.pop('pkg.list_pkgs', None)
    new = list_pkgs()
    rehash()
    return __salt__['pkg_resource.find_changes'](old, new)

def upgrade():
    '''
    Run pkg upgrade, if pkgng used. Otherwise do nothing

    Return a dict containing the new package names and versions::

        {'<package>': {'old': '<old-version>',
                       'new': '<new-version>'}}

    CLI Example::

        salt '*' pkg.upgrade
    '''

    if not _check_pkgng():
        # There is not easy way to upgrade packages with old package system
        return {}

    old = list_pkgs()
    __salt__['cmd.run_all']('{0} upgrade -y'.format(_cmd('pkg')))
    __context__.pop('pkg.list_pkgs', None)
    new = list_pkgs()
    return __salt__['pkg_resource.find_changes'](old, new)

def remove(name=None, pkgs=None, **kwargs):
    '''
    Remove packages.

    name
        The name of the package to be deleted.

    Multiple Package Options:

    pkgs
        A list of packages to delete. Must be passed as a python list. The
        ``name`` parameter will be ignored if this option is passed.

    Returns a dict containing the changes.

    CLI Example::

        salt '*' pkg.remove <package name>
        salt '*' pkg.remove <package1>,<package2>,<package3>
        salt '*' pkg.remove pkgs='["foo", "bar"]'
    '''
    pkg_params = __salt__['pkg_resource.parse_targets'](name, pkgs)[0]

    old = list_pkgs()
    targets = [x for x in pkg_params if x in old]
    if not targets:
        return {}

    for_remove = ' '.join(targets)
    if _check_pkgng():
        cmd = '{0} remove -y {1}'.format(_cmd('pkg'), for_remove)
    else:
        cmd = '{0} {1}'.format(_cmd('pkg_remove'), for_remove)

    __salt__['cmd.run_all'](cmd)
    __context__.pop('pkg.list_pkgs', None)
    new = list_pkgs()
    return __salt__['pkg_resource.find_changes'](old, new)

def purge(name=None, pkgs=None, **kwargs):
    '''
    Package purges are not supported, this function is identical to
    ``remove()``.

    name
        The name of the package to be deleted.

    Multiple Package Options:

    pkgs
        A list of packages to delete. Must be passed as a python list. The
        ``name`` parameter will be ignored if this option is passed.

    Returns a dict containing the changes.

    CLI Example::

        salt '*' pkg.purge <package name>
        salt '*' pkg.purge <package1>,<package2>,<package3>
        salt '*' pkg.purge pkgs='["foo", "bar"]'
    '''
    return remove(name=name, pkgs=pkgs)

def rehash():
    '''
    Recomputes internal hash table for the PATH variable.
    Use whenever a new command is created during the current
    session.

    CLI Example::

        salt '*' pkg.rehash
    '''
    shell = __salt__['cmd.run']('echo $SHELL').split('/')
    if shell[len(shell) - 1] in ['csh', 'tcsh']:
        __salt__['cmd.run_all']('rehash')

def perform_cmp(pkg1='', pkg2=''):
    '''
    Do a cmp-style comparison on two packages. Return -1 if pkg1 < pkg2, 0 if
    pkg1 == pkg2, and 1 if pkg1 > pkg2. Return None if there was a problem
    making the comparison.

    CLI Example::

        salt '*' pkg.perform_cmp '0.2.4-0' '0.2.4.1-0'
        salt '*' pkg.perform_cmp pkg1='0.2.4-0' pkg2='0.2.4.1-0'
    '''
    return __salt__['pkg_resource.perform_cmp'](pkg1=pkg1, pkg2=pkg2)

def compare(pkg1='', oper='==', pkg2=''):
    '''
    Compare two version strings.

    CLI Example::

        salt '*' pkg.compare '0.2.4-0' '<' '0.2.4.1-0'
        salt '*' pkg.compare pkg1='0.2.4-0' oper='<' pkg2='0.2.4.1-0'
    '''
    return __salt__['pkg_resource.compare'](pkg1=pkg1, oper=oper, pkg2=pkg2)

def file_list(*packages):
    '''
    List the files that belong to a package. Not specifying any packages will
    return a list of _every_ file on the system's package database (not
    generally recommended).

    CLI Examples::

        salt '*' pkg.file_list httpd
        salt '*' pkg.file_list httpd postfix
        salt '*' pkg.file_list
    '''
    ret = file_dict(*packages)
    files = []
    for pkg_files in ret['files'].values():
        files.extend(pkg_files)
    ret['files'] = files
    return ret

def file_dict(*packages):
    '''
    List the files that belong to a package, grouped by package. Not
    specifying any packages will return a list of _every_ file on the
    system's package database (not generally recommended).

    CLI Examples::

        salt '*' pkg.file_list httpd
        salt '*' pkg.file_list httpd postfix
        salt '*' pkg.file_list
    '''
    errors = []
    files = {}

    if _check_pkgng():
        if packages:
            match_pattern = '%n ~ {0}'
            matches = [match_pattern.format(p) for p in packages]

            cmd = '{0} query -e \'{1}\' \'%n %Fp\''.format(
                _cmd('pkg'), ' || '.join(matches))
        else:
            cmd = '{0} query -a \'%n %Fp\''.format(_cmd('pkg'))

        for line in __salt__['cmd.run_stdout'](cmd).splitlines():
            pkg, fn = line.split(' ', 1)
            if pkg not in files:
                files[pkg] = []
            files[pkg].append(fn)
    else:
        if packages:
            match_pattern = '\'{0}-[0-9]*\''
            matches = [match_pattern.format(p) for p in packages]

            cmd = '{0} -QL {1}'.format(_cmd('pkg_info'), ' '.join(matches))
        else:
            cmd = '{0} -QLa'.format(_cmd('pkg_info'))

        ret = __salt__['cmd.run_all'](cmd)

        for line in ret['stderr'].splitlines():
            errors.append(line)

        pkg = None
        for line in ret['stdout'].splitlines():
            if pkg is not None and line.startswith('/'):
                files[pkg].append(line)
            elif ':/' in line:
                pkg, fn = line.split(':', 1)
                pkg, ver = pkg.rsplit('-', 1)
                files[pkg] = [fn]
            else:
                continue  # unexpected string
    return {'errors': errors, 'files': files}
<|endoftext|>


<commit_before># Copyright (c) 2007-2012 gocept gmbh & co. kg
# See also LICENSE.txt

from zope.testing import doctest
import BaseHTTPServer
import __future__
import contextlib
import copy
import gocept.jslint
import gocept.selenium.base
import gocept.selenium.ztk
import gocept.testing.assertion
import gocept.zcapatch
import inspect
import json
import logging
import pkg_resources
import random
import re
import string
import sys
import threading
import time
import transaction
import unittest2
import urllib2
import xml.sax.saxutils
import zeit.connector.interfaces
import zope.app.appsetup.product
import zope.app.testing.functional
import zope.app.wsgi
import zope.component
import zope.publisher.browser
import zope.security.management
import zope.security.testing
import zope.site.hooks
import zope.testbrowser.testing
import zope.testing.renormalizing

def ZCMLLayer(
        config_file, module=None, name=None, allow_teardown=True,
        product_config=None):
    if module is None:
        module = inspect.stack()[1][0].f_globals['__name__']
    if name is None:
        name = 'ZCMLLayer(%s)' % config_file
    if not config_file.startswith('/'):
        config_file = pkg_resources.resource_filename(module, config_file)
    if product_config is True:
        product_config = cms_product_config

    def setUp(cls):
        cls.setup = zope.app.testing.functional.FunctionalTestSetup(
            config_file, product_config=product_config)

    def tearDown(cls):
        cls.setup.tearDownCompletely()
        if not allow_teardown:
            raise NotImplementedError

    def testSetUp(cls):
        cls.setup.setUp()
        # ``local_product_config`` contains mutable elements which tests may
        # modify. Create a copy to let setup restore the *original* config.
        cls.setup.local_product_config = copy.deepcopy(
            cls.setup.local_product_config)
        cls.setup.zca = gocept.zcapatch.Patches()

    def testTearDown(cls):
        try:
            connector = zope.component.getUtility(
                zeit.connector.interfaces.IConnector)
        except zope.component.interfaces.ComponentLookupError:
            pass
        else:
            connector._reset()
        cls.setup.zca.reset()
        zope.site.hooks.setSite(None)
        zope.security.management.endInteraction()
        cls.setup.tearDown()

    layer = type(name, (object,), dict(
        __module__=module,
        setUp=classmethod(setUp),
        tearDown=classmethod(tearDown),
        testSetUp=classmethod(testSetUp),
        testTearDown=classmethod(testTearDown),
    ))
    return layer

class HTTPServer(BaseHTTPServer.HTTPServer):
    # shutdown mechanism borrowed from gocept.selenium.static.HTTPServer

    _continue = True

    def __init__(self, *args):
        BaseHTTPServer.HTTPServer.__init__(self, *args)
        self.errors = []

    def handle_error(self, request, client_address):
        self.errors.append((request, client_address))

    def serve_until_shutdown(self):
        while self._continue:
            self.handle_request()

    def shutdown(self):
        self._continue = False
        # We fire a last request at the server in order to take it out of the
        # while loop in `self.serve_until_shutdown`.
        try:
            urllib2.urlopen(
                'http://%s:%s/die' % (self.server_name, self.server_port),
                timeout=1)
        except urllib2.URLError:
            # If the server is already shut down, we receive a socket error,
            # which we ignore.
            pass
        self.server_close()

def HTTPServerLayer(request_handler):
    """Factory for a layer which opens a HTTP port."""
    module = inspect.stack()[1][0].f_globals['__name__']
    port = random.randint(30000, 40000)

    def setUp(cls):
        server_address = ('localhost', port)
        cls.httpd = HTTPServer(server_address, request_handler)
        cls.thread = threading.Thread(target=cls.httpd.serve_until_shutdown)
        cls.thread.daemon = True
        cls.thread.start()
        # Wait as it sometimes takes a while to get the server started.
        # XXX this is a little kludgy
        time.sleep(0.001)

    def tearDown(cls):
        cls.httpd.shutdown()
        cls.thread.join()

    def testTearDown(cls):
        cls.httpd.errors[:] = []
        request_handler.tearDown()

    layer = type('HTTPLayer(%s)' % port, (object,), dict(
        __module__=module,
        setUp=classmethod(setUp),
        tearDown=classmethod(tearDown),
        testTearDown=classmethod(testTearDown),
    ))
    return layer, port

class BaseHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """Handler for testing which does not log to STDOUT."""

    def log_message(self, format, *args):
        pass

    @classmethod
    def tearDown(cls):
        pass

cms_product_config = string.Template("""\
<product-config zeit.cms>
  source-serie file://${base}/content/serie.xml
  source-navigation file://${base}/content/navigation.xml
  source-print-ressort file://${base}/content/print-ressort.xml
  source-keyword file://${base}/content/zeit-ontologie-prism.xml
  source-products file://${base}/content/products.xml
  source-badges file://${base}/asset/badges.xml
  source-banners file://${base}/content/banners.xml

  preview-prefix http://localhost/preview-prefix
  live-prefix http://localhost/live-prefix
  development-preview-prefix http://localhost/development-preview-prefix

  suggest-keyword-email-address none@testing
  suggest-keyword-real-name Dr. No
  whitelist-url file://${base}/tagging/tests/whitelist.xml
  breadcrumbs-use-common-metadata true
</product-config>
""").substitute(
    base=pkg_resources.resource_filename(__name__, ''))

cms_layer = ZCMLLayer('ftesting.zcml', product_config=True)
selenium_layer = gocept.selenium.ztk.Layer(cms_layer)

checker = zope.testing.renormalizing.RENormalizing([
    (re.compile(r'\d{4} \d{1,2} \d{1,2}  \d\d:\d\d:\d\d'), '<FORMATTED DATE>'),
    (re.compile('0x[0-9a-f]+'), "0x..."),
    (re.compile(r'/\+\+noop\+\+[0-9a-f]+'), ''),
    (re.compile('[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'),
     "<GUID>"),
])

def setup_product_config(product_config={}):
    zope.app.appsetup.product._configs.update(product_config)

optionflags = (doctest.REPORT_NDIFF +
               doctest.INTERPRET_FOOTNOTES +
               doctest.NORMALIZE_WHITESPACE +
               doctest.ELLIPSIS)

def DocFileSuite(*paths, **kw):
    kw['package'] = doctest._normalize_module(kw.get('package'))
    kw.setdefault('checker', checker)
    kw.setdefault('optionflags', optionflags)
    return doctest.DocFileSuite(*paths, **kw)

def FunctionalDocFileSuite(*paths, **kw):

    def setUp(test):
        config = test.globs.get('product_config', {})
        __traceback_info__ = (config,)
        setup_product_config(config)

    layer = kw.pop('layer', cms_layer)
    kw['package'] = doctest._normalize_module(kw.get('package'))
    kw['setUp'] = setUp
    globs = kw.setdefault('globs', {})
    globs['product_config'] = kw.pop('product_config', {})
    globs['with_statement'] = __future__.with_statement
    globs['getRootFolder'] = zope.app.testing.functional.getRootFolder
    kw.setdefault('checker', checker)
    kw.setdefault('optionflags', optionflags)

    test = doctest.DocFileSuite(*paths, **kw)
    test.layer = layer

    return test

class RepositoryHelper(object):

    @property
    def repository(self):
        import zeit.cms.repository.interfaces
        with site(self.getRootFolder()):
            return zope.component.getUtility(
                zeit.cms.repository.interfaces.IRepository)

    @repository.setter
    def repository(self, value):
        self.__dict__['repository'] = value

class FunctionalTestCaseCommon(
    unittest2.TestCase,
    gocept.testing.assertion.Ellipsis,
    gocept.testing.assertion.Exceptions,
    RepositoryHelper):

    layer = cms_layer
    product_config = {}

    def getRootFolder(self):
        """Returns the Zope root folder."""
        return zope.app.testing.functional.FunctionalTestSetup().\
            getRootFolder()

    @property
    def zca(self):
        return zope.app.testing.functional.FunctionalTestSetup().zca

    def setUp(self):
        super(FunctionalTestCaseCommon, self).setUp()
        setup_product_config(self.product_config)

class FunctionalTestCase(FunctionalTestCaseCommon):

    def setUp(self):
        super(FunctionalTestCase, self).setUp()
        zope.site.hooks.setSite(self.getRootFolder())
        self.principal = create_interaction(u'zope.user')

class SeleniumTestCase(gocept.selenium.base.TestCase,
                       FunctionalTestCaseCommon):

    layer = selenium_layer
    skin = 'cms'
    log_errors = False
    log_errors_ignore = ()
    level = 2

    TIMEOUT = 10

    def setUp(self):
        super(SeleniumTestCase, self).setUp()
        # XXX waiting for a version of gocept.selenium that handles timeouts
        # consistently (#10750)
        self.layer.selenium.setTimeout(self.TIMEOUT*1000)
        self.layer.selenium.selenium.set_timeout(self.TIMEOUT*1000)

        # XXX The following 5 lines are copied from
        # gocept.selenium.ztk.TestCase in order to avoid inheriting from
        # zope.app.testing.functional.TestCase.
        db = zope.app.testing.functional.FunctionalTestSetup().db
        application = self.layer.http.application
        assert isinstance(application, zope.app.wsgi.WSGIPublisherApplication)
        factory = type(application.requestFactory)
        application.requestFactory = factory(db)

        if self.log_errors:
            with site(self.getRootFolder()):
                error_log = zope.component.getUtility(
                    zope.error.interfaces.IErrorReportingUtility)
                error_log.copy_to_zlog = True
                error_log._ignored_exceptions = self.log_errors_ignore
            self.log_handler = logging.StreamHandler(sys.stdout)
            logging.root.addHandler(self.log_handler)
            self.old_log_level = logging.root.level
            logging.root.setLevel(logging.ERROR)
            transaction.commit()
        self.selenium.getEval('window.sessionStorage.clear()')

        self.original_windows = set(self.selenium.getAllWindowNames())

    def tearDown(self):
        super(SeleniumTestCase, self).tearDown()
        if self.log_errors:
            logging.root.removeHandler(self.log_handler)
            logging.root.setLevel(self.old_log_level)

        current_windows = set(self.selenium.getAllWindowNames())
        for window in current_windows - self.original_windows:
            self.selenium.selectWindow(window)
            self.selenium.close()
        self.selenium.selectWindow()

    def open(self, path, auth='user:userpw'):
        auth_sent = getattr(self.layer, 'auth_sent', None)
        if auth and auth != auth_sent:
            # Only set auth when it changed. Firefox will be confused
            # otherwise.
            self.layer.auth_sent = auth
            auth = auth + '@'
        else:
            auth = ''
        self.selenium.open(
            'http://%s%s/++skin++%s%s' % (
                auth, self.selenium.server, self.skin, path))

    def click_label(self, label):
        self.selenium.click('//label[contains(string(.), %s)]' %
                            xml.sax.saxutils.quoteattr(label))

    setup_globals = """\
        var window = selenium.browserbot.getCurrentWindow();
        var document = window.document;
        var zeit = window.zeit;
        """

    def eval(self, text):
        return self.selenium.getEval(self.setup_globals + text)

    def wait_for_condition(self, text):
        self.selenium.waitForCondition(self.setup_globals + """\
        Boolean(%s);
        """ % text)

def click_wo_redirect(browser, *args, **kwargs):
    browser.mech_browser.set_handle_redirect(False)
    try:
        try:
            browser.getLink(*args, **kwargs).click()
        except urllib2.HTTPError, e:
            print str(e)
            print e.hdrs.get('location')
    finally:
        browser.mech_browser.set_handle_redirect(True)

def set_site(site=None):
    """Encapsulation of the getSite/setSite-dance.

    Sets the given site, preserves the old site in the globs,
    where it will be reset by our FunctionalDocFileSuite's tearDown.
    """

    globs = sys._getframe(1).f_locals
    globs['old_site'] = zope.site.hooks.getSite()
    if site is None:
        site = globs['getRootFolder']()
    zope.site.hooks.setSite(site)

# XXX use zope.publisher.testing for the following two
def create_interaction(name=u'zope.user'):
    principal = zope.security.testing.Principal(
        name, groups=['zope.Authenticated'], description=u'test@example.com')
    request = zope.publisher.browser.TestRequest()
    request.setPrincipal(principal)
    zope.security.management.newInteraction(request)
    return principal

@contextlib.contextmanager
def interaction(principal_id=u'zope.user'):
    if zope.security.management.queryInteraction():
        # There already is an interaction. Great. Leave it alone.
        yield
    else:
        principal = create_interaction(principal_id)
        yield principal
        zope.security.management.endInteraction()

# XXX use zope.component.testing.site instead
@contextlib.contextmanager
def site(root):
    old_site = zope.site.hooks.getSite()
    zope.site.hooks.setSite(root)
    yield
    zope.site.hooks.setSite(old_site)

class BrowserAssertions(gocept.testing.assertion.Ellipsis):

    # XXX backwards-compat method signature for existing tests, should probably
    # be removed at some point
    def assert_ellipsis(self, want, got=None):
        if got is None:
            got = self.browser.contents
        self.assertEllipsis(want, got)

    def assert_json(self, want, got=None):
        if got is None:
            got = self.browser.contents
        data = json.loads(got)
        self.assertEqual(want, data)
        return data

class BrowserTestCase(FunctionalTestCaseCommon, BrowserAssertions):

    def setUp(self):
        super(BrowserTestCase, self).setUp()
        self.browser = zope.testbrowser.testing.Browser()
        self.browser.addHeader('Authorization', 'Basic user:userpw')

class JSLintTestCase(gocept.jslint.TestCase):

    options = (gocept.jslint.TestCase.options +
               ('evil',
                'eqnull',
                'multistr',
                'sub',
                'undef',
                'browser',
                'jquery',
                'devel'
                ))
    predefined = (
        'zeit', 'gocept',
        'application_url', 'context_url',
        'DOMParser', 'escape', 'unescape', 'getSelection',
        'jsontemplate',
        'MochiKit', '$$', 'forEach', 'filter', 'map', 'extend', 'bind',
        'log', 'repr', 'logger', 'logDebug', 'logError', # XXX
        'DIV', 'A', 'UL', 'LI', 'INPUT', 'IMG', 'SELECT', 'OPTION', 'BUTTON',
        'isNull', 'isUndefined', 'isUndefinedOrNull',
        )

    ignore = (
        "Don't make functions within a loop",
        "Expected an identifier and instead saw 'import'",
        "Use '===' to compare with",
        "Use '!==' to compare with",
        "Missing radix parameter",
        "Bad line breaking",
        )
<commit_msg>re #10657: added a test helper method that waits for a dotted name to exist in the JS namespace<commit_after># Copyright (c) 2007-2012 gocept gmbh & co. kg
# See also LICENSE.txt

from zope.testing import doctest
import BaseHTTPServer
import __future__
import contextlib
import copy
import gocept.jslint
import gocept.selenium.base
import gocept.selenium.ztk
import gocept.testing.assertion
import gocept.zcapatch
import inspect
import json
import logging
import pkg_resources
import random
import re
import string
import sys
import threading
import time
import transaction
import unittest2
import urllib2
import xml.sax.saxutils
import zeit.connector.interfaces
import zope.app.appsetup.product
import zope.app.testing.functional
import zope.app.wsgi
import zope.component
import zope.publisher.browser
import zope.security.management
import zope.security.testing
import zope.site.hooks
import zope.testbrowser.testing
import zope.testing.renormalizing

def ZCMLLayer(
        config_file, module=None, name=None, allow_teardown=True,
        product_config=None):
    if module is None:
        module = inspect.stack()[1][0].f_globals['__name__']
    if name is None:
        name = 'ZCMLLayer(%s)' % config_file
    if not config_file.startswith('/'):
        config_file = pkg_resources.resource_filename(module, config_file)
    if product_config is True:
        product_config = cms_product_config

    def setUp(cls):
        cls.setup = zope.app.testing.functional.FunctionalTestSetup(
            config_file, product_config=product_config)

    def tearDown(cls):
        cls.setup.tearDownCompletely()
        if not allow_teardown:
            raise NotImplementedError

    def testSetUp(cls):
        cls.setup.setUp()
        # ``local_product_config`` contains mutable elements which tests may
        # modify. Create a copy to let setup restore the *original* config.
        cls.setup.local_product_config = copy.deepcopy(
            cls.setup.local_product_config)
        cls.setup.zca = gocept.zcapatch.Patches()

    def testTearDown(cls):
        try:
            connector = zope.component.getUtility(
                zeit.connector.interfaces.IConnector)
        except zope.component.interfaces.ComponentLookupError:
            pass
        else:
            connector._reset()
        cls.setup.zca.reset()
        zope.site.hooks.setSite(None)
        zope.security.management.endInteraction()
        cls.setup.tearDown()

    layer = type(name, (object,), dict(
        __module__=module,
        setUp=classmethod(setUp),
        tearDown=classmethod(tearDown),
        testSetUp=classmethod(testSetUp),
        testTearDown=classmethod(testTearDown),
    ))
    return layer

class HTTPServer(BaseHTTPServer.HTTPServer):
    # shutdown mechanism borrowed from gocept.selenium.static.HTTPServer

    _continue = True

    def __init__(self, *args):
        BaseHTTPServer.HTTPServer.__init__(self, *args)
        self.errors = []

    def handle_error(self, request, client_address):
        self.errors.append((request, client_address))

    def serve_until_shutdown(self):
        while self._continue:
            self.handle_request()

    def shutdown(self):
        self._continue = False
        # We fire a last request at the server in order to take it out of the
        # while loop in `self.serve_until_shutdown`.
        try:
            urllib2.urlopen(
                'http://%s:%s/die' % (self.server_name, self.server_port),
                timeout=1)
        except urllib2.URLError:
            # If the server is already shut down, we receive a socket error,
            # which we ignore.
            pass
        self.server_close()

def HTTPServerLayer(request_handler):
    """Factory for a layer which opens a HTTP port."""
    module = inspect.stack()[1][0].f_globals['__name__']
    port = random.randint(30000, 40000)

    def setUp(cls):
        server_address = ('localhost', port)
        cls.httpd = HTTPServer(server_address, request_handler)
        cls.thread = threading.Thread(target=cls.httpd.serve_until_shutdown)
        cls.thread.daemon = True
        cls.thread.start()
        # Wait as it sometimes takes a while to get the server started.
        # XXX this is a little kludgy
        time.sleep(0.001)

    def tearDown(cls):
        cls.httpd.shutdown()
        cls.thread.join()

    def testTearDown(cls):
        cls.httpd.errors[:] = []
        request_handler.tearDown()

    layer = type('HTTPLayer(%s)' % port, (object,), dict(
        __module__=module,
        setUp=classmethod(setUp),
        tearDown=classmethod(tearDown),
        testTearDown=classmethod(testTearDown),
    ))
    return layer, port

class BaseHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
    """Handler for testing which does not log to STDOUT."""

    def log_message(self, format, *args):
        pass

    @classmethod
    def tearDown(cls):
        pass

cms_product_config = string.Template("""\
<product-config zeit.cms>
  source-serie file://${base}/content/serie.xml
  source-navigation file://${base}/content/navigation.xml
  source-print-ressort file://${base}/content/print-ressort.xml
  source-keyword file://${base}/content/zeit-ontologie-prism.xml
  source-products file://${base}/content/products.xml
  source-badges file://${base}/asset/badges.xml
  source-banners file://${base}/content/banners.xml

  preview-prefix http://localhost/preview-prefix
  live-prefix http://localhost/live-prefix
  development-preview-prefix http://localhost/development-preview-prefix

  suggest-keyword-email-address none@testing
  suggest-keyword-real-name Dr. No
  whitelist-url file://${base}/tagging/tests/whitelist.xml
  breadcrumbs-use-common-metadata true
</product-config>
""").substitute(
    base=pkg_resources.resource_filename(__name__, ''))

cms_layer = ZCMLLayer('ftesting.zcml', product_config=True)
selenium_layer = gocept.selenium.ztk.Layer(cms_layer)

checker = zope.testing.renormalizing.RENormalizing([
    (re.compile(r'\d{4} \d{1,2} \d{1,2}  \d\d:\d\d:\d\d'), '<FORMATTED DATE>'),
    (re.compile('0x[0-9a-f]+'), "0x..."),
    (re.compile(r'/\+\+noop\+\+[0-9a-f]+'), ''),
    (re.compile('[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'),
     "<GUID>"),
])

def setup_product_config(product_config={}):
    zope.app.appsetup.product._configs.update(product_config)

optionflags = (doctest.REPORT_NDIFF +
               doctest.INTERPRET_FOOTNOTES +
               doctest.NORMALIZE_WHITESPACE +
               doctest.ELLIPSIS)

def DocFileSuite(*paths, **kw):
    kw['package'] = doctest._normalize_module(kw.get('package'))
    kw.setdefault('checker', checker)
    kw.setdefault('optionflags', optionflags)
    return doctest.DocFileSuite(*paths, **kw)

def FunctionalDocFileSuite(*paths, **kw):

    def setUp(test):
        config = test.globs.get('product_config', {})
        __traceback_info__ = (config,)
        setup_product_config(config)

    layer = kw.pop('layer', cms_layer)
    kw['package'] = doctest._normalize_module(kw.get('package'))
    kw['setUp'] = setUp
    globs = kw.setdefault('globs', {})
    globs['product_config'] = kw.pop('product_config', {})
    globs['with_statement'] = __future__.with_statement
    globs['getRootFolder'] = zope.app.testing.functional.getRootFolder
    kw.setdefault('checker', checker)
    kw.setdefault('optionflags', optionflags)

    test = doctest.DocFileSuite(*paths, **kw)
    test.layer = layer

    return test

class RepositoryHelper(object):

    @property
    def repository(self):
        import zeit.cms.repository.interfaces
        with site(self.getRootFolder()):
            return zope.component.getUtility(
                zeit.cms.repository.interfaces.IRepository)

    @repository.setter
    def repository(self, value):
        self.__dict__['repository'] = value

class FunctionalTestCaseCommon(
    unittest2.TestCase,
    gocept.testing.assertion.Ellipsis,
    gocept.testing.assertion.Exceptions,
    RepositoryHelper):

    layer = cms_layer
    product_config = {}

    def getRootFolder(self):
        """Returns the Zope root folder."""
        return zope.app.testing.functional.FunctionalTestSetup().\
            getRootFolder()

    @property
    def zca(self):
        return zope.app.testing.functional.FunctionalTestSetup().zca

    def setUp(self):
        super(FunctionalTestCaseCommon, self).setUp()
        setup_product_config(self.product_config)

class FunctionalTestCase(FunctionalTestCaseCommon):

    def setUp(self):
        super(FunctionalTestCase, self).setUp()
        zope.site.hooks.setSite(self.getRootFolder())
        self.principal = create_interaction(u'zope.user')

class SeleniumTestCase(gocept.selenium.base.TestCase,
                       FunctionalTestCaseCommon):

    layer = selenium_layer
    skin = 'cms'
    log_errors = False
    log_errors_ignore = ()
    level = 2

    TIMEOUT = 10

    def setUp(self):
        super(SeleniumTestCase, self).setUp()
        # XXX waiting for a version of gocept.selenium that handles timeouts
        # consistently (#10750)
        self.layer.selenium.setTimeout(self.TIMEOUT*1000)
        self.layer.selenium.selenium.set_timeout(self.TIMEOUT*1000)

        # XXX The following 5 lines are copied from
        # gocept.selenium.ztk.TestCase in order to avoid inheriting from
        # zope.app.testing.functional.TestCase.
        db = zope.app.testing.functional.FunctionalTestSetup().db
        application = self.layer.http.application
        assert isinstance(application, zope.app.wsgi.WSGIPublisherApplication)
        factory = type(application.requestFactory)
        application.requestFactory = factory(db)

        if self.log_errors:
            with site(self.getRootFolder()):
                error_log = zope.component.getUtility(
                    zope.error.interfaces.IErrorReportingUtility)
                error_log.copy_to_zlog = True
                error_log._ignored_exceptions = self.log_errors_ignore
            self.log_handler = logging.StreamHandler(sys.stdout)
            logging.root.addHandler(self.log_handler)
            self.old_log_level = logging.root.level
            logging.root.setLevel(logging.ERROR)
            transaction.commit()
        self.selenium.getEval('window.sessionStorage.clear()')

        self.original_windows = set(self.selenium.getAllWindowNames())

    def tearDown(self):
        super(SeleniumTestCase, self).tearDown()
        if self.log_errors:
            logging.root.removeHandler(self.log_handler)
            logging.root.setLevel(self.old_log_level)

        current_windows = set(self.selenium.getAllWindowNames())
        for window in current_windows - self.original_windows:
            self.selenium.selectWindow(window)
            self.selenium.close()
        self.selenium.selectWindow()

    def open(self, path, auth='user:userpw'):
        auth_sent = getattr(self.layer, 'auth_sent', None)
        if auth and auth != auth_sent:
            # Only set auth when it changed. Firefox will be confused
            # otherwise.
            self.layer.auth_sent = auth
            auth = auth + '@'
        else:
            auth = ''
        self.selenium.open(
            'http://%s%s/++skin++%s%s' % (
                auth, self.selenium.server, self.skin, path))

    def click_label(self, label):
        self.selenium.click('//label[contains(string(.), %s)]' %
                            xml.sax.saxutils.quoteattr(label))

    setup_globals = """\
        var window = selenium.browserbot.getCurrentWindow();
        var document = window.document;
        var zeit = window.zeit;
        """

    def eval(self, text):
        return self.selenium.getEval(self.setup_globals + text)

    def wait_for_condition(self, text):
        self.selenium.waitForCondition(self.setup_globals + """\
        Boolean(%s);
        """ % text)

    def wait_for_dotted_name(self, dotted_name):
        partial = []
        for part in dotted_name.split('.'):
            partial.append(part)
            self.wait_for_condition('.'.join(partial))

def click_wo_redirect(browser, *args, **kwargs):
    browser.mech_browser.set_handle_redirect(False)
    try:
        try:
            browser.getLink(*args, **kwargs).click()
        except urllib2.HTTPError, e:
            print str(e)
            print e.hdrs.get('location')
    finally:
        browser.mech_browser.set_handle_redirect(True)

def set_site(site=None):
    """Encapsulation of the getSite/setSite-dance.

    Sets the given site, preserves the old site in the globs,
    where it will be reset by our FunctionalDocFileSuite's tearDown.
    """

    globs = sys._getframe(1).f_locals
    globs['old_site'] = zope.site.hooks.getSite()
    if site is None:
        site = globs['getRootFolder']()
    zope.site.hooks.setSite(site)

# XXX use zope.publisher.testing for the following two
def create_interaction(name=u'zope.user'):
    principal = zope.security.testing.Principal(
        name, groups=['zope.Authenticated'], description=u'test@example.com')
    request = zope.publisher.browser.TestRequest()
    request.setPrincipal(principal)
    zope.security.management.newInteraction(request)
    return principal

@contextlib.contextmanager
def interaction(principal_id=u'zope.user'):
    if zope.security.management.queryInteraction():
        # There already is an interaction. Great. Leave it alone.
        yield
    else:
        principal = create_interaction(principal_id)
        yield principal
        zope.security.management.endInteraction()

# XXX use zope.component.testing.site instead
@contextlib.contextmanager
def site(root):
    old_site = zope.site.hooks.getSite()
    zope.site.hooks.setSite(root)
    yield
    zope.site.hooks.setSite(old_site)

class BrowserAssertions(gocept.testing.assertion.Ellipsis):

    # XXX backwards-compat method signature for existing tests, should probably
    # be removed at some point
    def assert_ellipsis(self, want, got=None):
        if got is None:
            got = self.browser.contents
        self.assertEllipsis(want, got)

    def assert_json(self, want, got=None):
        if got is None:
            got = self.browser.contents
        data = json.loads(got)
        self.assertEqual(want, data)
        return data

class BrowserTestCase(FunctionalTestCaseCommon, BrowserAssertions):

    def setUp(self):
        super(BrowserTestCase, self).setUp()
        self.browser = zope.testbrowser.testing.Browser()
        self.browser.addHeader('Authorization', 'Basic user:userpw')

class JSLintTestCase(gocept.jslint.TestCase):

    options = (gocept.jslint.TestCase.options +
               ('evil',
                'eqnull',
                'multistr',
                'sub',
                'undef',
                'browser',
                'jquery',
                'devel'
                ))
    predefined = (
        'zeit', 'gocept',
        'application_url', 'context_url',
        'DOMParser', 'escape', 'unescape', 'getSelection',
        'jsontemplate',
        'MochiKit', '$$', 'forEach', 'filter', 'map', 'extend', 'bind',
        'log', 'repr', 'logger', 'logDebug', 'logError', # XXX
        'DIV', 'A', 'UL', 'LI', 'INPUT', 'IMG', 'SELECT', 'OPTION', 'BUTTON',
        'isNull', 'isUndefined', 'isUndefinedOrNull',
        )

    ignore = (
        "Don't make functions within a loop",
        "Expected an identifier and instead saw 'import'",
        "Use '===' to compare with",
        "Use '!==' to compare with",
        "Missing radix parameter",
        "Bad line breaking",
        )
<|endoftext|>


<commit_before>from datetime import datetime, timedelta
from zeit.cms.i18n import MessageFactory as _
from zeit.push.interfaces import CONFIG_CHANNEL_NEWS, CONFIG_CHANNEL_BREAKING
import collections
import grokcore.component as grok
import logging
import pytz
import urllib
import urlparse
import zeit.cms.checkout.interfaces
import zeit.cms.content.interfaces
import zeit.cms.interfaces
import zeit.content.image.interfaces
import zeit.push.interfaces
import zeit.push.message
import zope.app.appsetup.product
import zope.cachedescriptors.property
import zope.component
import zope.i18n
import zope.interface

log = logging.getLogger(__name__)

class ConnectionBase(object):
    """Base class to send push notifications to mobile devices."""

    zope.interface.implements(zeit.push.interfaces.IPushNotifier)

    LANGUAGE = 'de'
    APP_IDENTIFIER = 'zeitapp'

    def __init__(self, expire_interval):
        self.expire_interval = expire_interval

    @zope.cachedescriptors.property.Lazy
    def config(self):
        return zope.app.appsetup.product.getProductConfiguration(
            'zeit.push') or {}

    def get_channel_list(self, channels):
        """Return forward-compatible list of channels.

        We currently use channels as a monovalent value, set to either
        CONFIG_CHANNEL_NEWS or CONFIG_CHANNEL_BREAKING. Make sure to retrieve
        the according title from the config and return it as a list.

        If `channels` already is a list, just return it. This is intended for
        forward-compatibility, if we start using multiple channels.

        """
        if isinstance(channels, list):
            return channels
        return [x for x in self.config.get(channels, '').split(' ') if x]

    def get_headline(self, channels):
        """Return translation for the headline, which depends on the channel.

        Since the channel is used to classify the push notification as ordinary
        news or breaking news, it also influences the headline.

        """
        if self.config.get(CONFIG_CHANNEL_NEWS) in channels:
            headline = _('push-news-title')
        else:
            headline = _('push-breaking-title')

        # There's no i18n in the mobile app, so we translate to a hard-coded
        # language here.
        return zope.i18n.translate(headline, target_language=self.LANGUAGE)

    @property
    def expiration_datetime(self):
        return (datetime.now(pytz.UTC).replace(microsecond=0) +
                timedelta(seconds=self.expire_interval))

    def data(self, title, link, **kw):
        """Return data for push notifications to Android and iOS."""
        channels = self.get_channel_list(kw.get('channels'))
        if not channels:
            log.warn('No channel given for push notification for %s', link)

        is_breaking = self.config.get(CONFIG_CHANNEL_BREAKING) in channels

        # Extra tag helps the app know which kind of notification was send
        if is_breaking:
            extra_tag = self.config.get(CONFIG_CHANNEL_BREAKING)
        else:
            extra_tag = self.config.get(CONFIG_CHANNEL_NEWS)

        path = self.strip_to_path(link)
        push_target = self.config.get('push-target-url', '').rstrip('/')
        full_link = '/'.join((push_target, path))
        deep_link = '://'.join((self.APP_IDENTIFIER, path))

        return {
            'ios': {
                'alert': title,
                'deep_link': self.add_tracking(deep_link, channels, 'ios'),
                'headline': self.get_headline(channels),
                'sound': 'chime.aiff' if is_breaking else '',
                'tag': extra_tag,
                'url': self.add_tracking(full_link, channels, 'ios')
            },
            'android': {
                'alert': title,
                'deep_link': self.add_tracking(deep_link, channels, 'android'),
                'headline': 'ZEIT ONLINE {}'.format(
                    self.get_headline(channels)),
                'priority': 2 if is_breaking else 0,
                'tag': extra_tag,
                'url': self.add_tracking(full_link, channels, 'android')
            }
        }

    def send(self, text, link, **kw):
        raise NotImplementedError

    @staticmethod
    def strip_to_path(url):
        if '://' not in url:
            url = '//' + url.lstrip('/')  # ensure valid url
        return urlparse.urlunparse(
            ['', ''] + list(urlparse.urlparse(url)[2:])).lstrip('/')

    @staticmethod
    def add_tracking(url, channels, device):
        config = zope.app.appsetup.product.getProductConfiguration(
            'zeit.push') or {}
        if config.get(CONFIG_CHANNEL_BREAKING) in channels:
            channel = 'eilmeldung'
        else:
            channel = 'wichtige_news'
        if device == 'android':
            device = 'andpush'
        else:
            device = 'iospush'

        tracking = {
            'wt_zmc':
            'fix.int.zonaudev.push.{channel}.zeitde.{device}.link.x'.format(
                channel=channel, device=device),
            'utm_medium': 'fix',
            'utm_source': 'push_zonaudev_int',
            'utm_campaign': channel,
            'utm_content': 'zeitde_{device}_link_x'.format(device=device),
        }

        parts = list(urlparse.urlparse(url))
        query = collections.OrderedDict(urlparse.parse_qs(parts[4]))
        query.update(tracking)
        parts[4] = urllib.urlencode(query, doseq=True)
        return urlparse.urlunparse(parts)

class Message(zeit.push.message.Message):

    grok.context(zeit.cms.content.interfaces.ICommonMetadata)
    grok.name('mobile')

    def send_push_notification(self, service_name, **kw):
        # BBB This used to send to both parse and urbanairship, so we use
        # a generic name in the message_config.
        super(Message, self).send_push_notification('urbanairship', **kw)

    def log_success(self, name):
        message = (
            self.config.get('override_text') or
            self.additional_parameters.get('teaserTitle') or
            self.text)

        notifier = zope.component.getUtility(
            zeit.push.interfaces.IPushNotifier, name=name)
        channels = notifier.get_channel_list(self.config.get('channels'))

        self.object_log.log(_(
            'Push notification for "${name}" sent. '
            '(Message: "${message}", Channels: ${channels})',
            mapping={
                'name': name.capitalize(),
                'message': message,
                'channels': ' '.join(channels)}))

    @zope.cachedescriptors.property.Lazy
    def text(self):
        return self.config.get('override_text', self.context.title)

    @zope.cachedescriptors.property.Lazy
    def additional_parameters(self):
        result = {}
        for name in ['teaserTitle', 'teaserText', 'teaserSupertitle']:
            value = getattr(self.context, name)
            if value:
                result[name] = value
        if self.image:
            result['image_url'] = self.image.uniqueId.replace(
                zeit.cms.interfaces.ID_NAMESPACE,
                self.product_config['mobile-image-url'])
        return result

    @property
    def image(self):
        images = zeit.content.image.interfaces.IImages(self.context, None)
        return getattr(images, 'image', None)

    @zope.cachedescriptors.property.Lazy
    def product_config(self):
        return zope.app.appsetup.product.getProductConfiguration('zeit.push')

@grok.subscribe(
    zeit.cms.content.interfaces.ICommonMetadata,
    zeit.cms.checkout.interfaces.IBeforeCheckinEvent)
def set_push_news_flag(context, event):
    push = zeit.push.interfaces.IPushMessages(context)
    for service in push.message_config:
        if (service['type'] == 'mobile' and
                service.get('enabled') and
                service.get('channels') == CONFIG_CHANNEL_NEWS):
            context.push_news = True
            break
<commit_msg>Sort query string, as zeit.web does, since a consistent order is more cache-friendly<commit_after>from datetime import datetime, timedelta
from zeit.cms.i18n import MessageFactory as _
from zeit.push.interfaces import CONFIG_CHANNEL_NEWS, CONFIG_CHANNEL_BREAKING
import collections
import grokcore.component as grok
import logging
import pytz
import urllib
import urlparse
import zeit.cms.checkout.interfaces
import zeit.cms.content.interfaces
import zeit.cms.interfaces
import zeit.content.image.interfaces
import zeit.push.interfaces
import zeit.push.message
import zope.app.appsetup.product
import zope.cachedescriptors.property
import zope.component
import zope.i18n
import zope.interface

log = logging.getLogger(__name__)

class ConnectionBase(object):
    """Base class to send push notifications to mobile devices."""

    zope.interface.implements(zeit.push.interfaces.IPushNotifier)

    LANGUAGE = 'de'
    APP_IDENTIFIER = 'zeitapp'

    def __init__(self, expire_interval):
        self.expire_interval = expire_interval

    @zope.cachedescriptors.property.Lazy
    def config(self):
        return zope.app.appsetup.product.getProductConfiguration(
            'zeit.push') or {}

    def get_channel_list(self, channels):
        """Return forward-compatible list of channels.

        We currently use channels as a monovalent value, set to either
        CONFIG_CHANNEL_NEWS or CONFIG_CHANNEL_BREAKING. Make sure to retrieve
        the according title from the config and return it as a list.

        If `channels` already is a list, just return it. This is intended for
        forward-compatibility, if we start using multiple channels.

        """
        if isinstance(channels, list):
            return channels
        return [x for x in self.config.get(channels, '').split(' ') if x]

    def get_headline(self, channels):
        """Return translation for the headline, which depends on the channel.

        Since the channel is used to classify the push notification as ordinary
        news or breaking news, it also influences the headline.

        """
        if self.config.get(CONFIG_CHANNEL_NEWS) in channels:
            headline = _('push-news-title')
        else:
            headline = _('push-breaking-title')

        # There's no i18n in the mobile app, so we translate to a hard-coded
        # language here.
        return zope.i18n.translate(headline, target_language=self.LANGUAGE)

    @property
    def expiration_datetime(self):
        return (datetime.now(pytz.UTC).replace(microsecond=0) +
                timedelta(seconds=self.expire_interval))

    def data(self, title, link, **kw):
        """Return data for push notifications to Android and iOS."""
        channels = self.get_channel_list(kw.get('channels'))
        if not channels:
            log.warn('No channel given for push notification for %s', link)

        is_breaking = self.config.get(CONFIG_CHANNEL_BREAKING) in channels

        # Extra tag helps the app know which kind of notification was send
        if is_breaking:
            extra_tag = self.config.get(CONFIG_CHANNEL_BREAKING)
        else:
            extra_tag = self.config.get(CONFIG_CHANNEL_NEWS)

        path = self.strip_to_path(link)
        push_target = self.config.get('push-target-url', '').rstrip('/')
        full_link = '/'.join((push_target, path))
        deep_link = '://'.join((self.APP_IDENTIFIER, path))

        return {
            'ios': {
                'alert': title,
                'deep_link': self.add_tracking(deep_link, channels, 'ios'),
                'headline': self.get_headline(channels),
                'sound': 'chime.aiff' if is_breaking else '',
                'tag': extra_tag,
                'url': self.add_tracking(full_link, channels, 'ios')
            },
            'android': {
                'alert': title,
                'deep_link': self.add_tracking(deep_link, channels, 'android'),
                'headline': 'ZEIT ONLINE {}'.format(
                    self.get_headline(channels)),
                'priority': 2 if is_breaking else 0,
                'tag': extra_tag,
                'url': self.add_tracking(full_link, channels, 'android')
            }
        }

    def send(self, text, link, **kw):
        raise NotImplementedError

    @staticmethod
    def strip_to_path(url):
        if '://' not in url:
            url = '//' + url.lstrip('/')  # ensure valid url
        return urlparse.urlunparse(
            ['', ''] + list(urlparse.urlparse(url)[2:])).lstrip('/')

    @staticmethod
    def add_tracking(url, channels, device):
        config = zope.app.appsetup.product.getProductConfiguration(
            'zeit.push') or {}
        if config.get(CONFIG_CHANNEL_BREAKING) in channels:
            channel = 'eilmeldung'
        else:
            channel = 'wichtige_news'
        if device == 'android':
            device = 'andpush'
        else:
            device = 'iospush'

        tracking = collections.OrderedDict(sorted({
            'wt_zmc':
            'fix.int.zonaudev.push.{channel}.zeitde.{device}.link.x'.format(
                channel=channel, device=device),
            'utm_medium': 'fix',
            'utm_source': 'push_zonaudev_int',
            'utm_campaign': channel,
            'utm_content': 'zeitde_{device}_link_x'.format(device=device),
        }.items()))

        parts = list(urlparse.urlparse(url))
        query = collections.OrderedDict(urlparse.parse_qs(parts[4]))
        for key, value in tracking.items():
            query[key] = value
        parts[4] = urllib.urlencode(query, doseq=True)
        return urlparse.urlunparse(parts)

class Message(zeit.push.message.Message):

    grok.context(zeit.cms.content.interfaces.ICommonMetadata)
    grok.name('mobile')

    def send_push_notification(self, service_name, **kw):
        # BBB This used to send to both parse and urbanairship, so we use
        # a generic name in the message_config.
        super(Message, self).send_push_notification('urbanairship', **kw)

    def log_success(self, name):
        message = (
            self.config.get('override_text') or
            self.additional_parameters.get('teaserTitle') or
            self.text)

        notifier = zope.component.getUtility(
            zeit.push.interfaces.IPushNotifier, name=name)
        channels = notifier.get_channel_list(self.config.get('channels'))

        self.object_log.log(_(
            'Push notification for "${name}" sent. '
            '(Message: "${message}", Channels: ${channels})',
            mapping={
                'name': name.capitalize(),
                'message': message,
                'channels': ' '.join(channels)}))

    @zope.cachedescriptors.property.Lazy
    def text(self):
        return self.config.get('override_text', self.context.title)

    @zope.cachedescriptors.property.Lazy
    def additional_parameters(self):
        result = {}
        for name in ['teaserTitle', 'teaserText', 'teaserSupertitle']:
            value = getattr(self.context, name)
            if value:
                result[name] = value
        if self.image:
            result['image_url'] = self.image.uniqueId.replace(
                zeit.cms.interfaces.ID_NAMESPACE,
                self.product_config['mobile-image-url'])
        return result

    @property
    def image(self):
        images = zeit.content.image.interfaces.IImages(self.context, None)
        return getattr(images, 'image', None)

    @zope.cachedescriptors.property.Lazy
    def product_config(self):
        return zope.app.appsetup.product.getProductConfiguration('zeit.push')

@grok.subscribe(
    zeit.cms.content.interfaces.ICommonMetadata,
    zeit.cms.checkout.interfaces.IBeforeCheckinEvent)
def set_push_news_flag(context, event):
    push = zeit.push.interfaces.IPushMessages(context)
    for service in push.message_config:
        if (service['type'] == 'mobile' and
                service.get('enabled') and
                service.get('channels') == CONFIG_CHANNEL_NEWS):
            context.push_news = True
            break
<|endoftext|>