<commit_before>"""
One of the really important features of |jedi| is to have an option to
understand code like this::

    def foo(bar):
        bar. # completion here
    foo(1)

There's no doubt wheter bar is an ``int`` or not, but if there's also a call
like ``foo('str')``, what would happen? Well, we'll just show both. Because
that's what a human would expect.

It works as follows:

- |Jedi| sees a param
- search for function calls named ``foo``
- execute these calls and check the input. This work with a ``ParamListener``.
"""
from itertools import chain

from jedi._compatibility import unicode
from jedi.parser import tree as pr
from jedi import settings
from jedi import debug
from jedi.evaluate import helpers
from jedi.evaluate.cache import memoize_default
from jedi.evaluate import imports

# This is something like the sys.path, but only for searching params. It means
# that this is the order in which Jedi searches params.
search_param_modules = ['.']

class ParamListener(object):
    """
    This listener is used to get the params for a function.
    """
    def __init__(self):
        self.param_possibilities = []

    def execute(self, params):
        self.param_possibilities += params

@debug.increase_indent
def search_params(evaluator, param):
    """
    A dynamic search for param values. If you try to complete a type:

    >>> def func(foo):
    ...     foo
    >>> func(1)
    >>> func("")

    It is not known what the type ``foo`` without analysing the whole code. You
    have to look for all calls to ``func`` to find out what ``foo`` possibly
    is.
    """
    if not settings.dynamic_params:
        return []
    debug.dbg('Dynamic param search for %s', param)

    func = param.get_parent_until(pr.Function)
    """
        for params in get_posibilities(evaluator, module, func_name):
            for p in params:
                if str(p) == str(param.get_name()):
                    result += p.parent.eval(evaluator)
                    """
    # Compare the param names.
    names = [n for n in search_function_call(evaluator, func)
             if n.value == param.name.value]
    # Evaluate the ExecutedParams to types.
    result = list(chain.from_iterable(n.parent.eval(evaluator) for n in names))
    debug.dbg('Dynamic param result %s', result)
    return result

@memoize_default([], evaluator_is_first_arg=True)
def search_function_call(evaluator, func):
    """
    Returns a list of param names.
    """
    from jedi.evaluate import representation as er

    def get_params_for_module(module):
        """
        Returns the values of a param, or an empty array.
        """
        @memoize_default([], evaluator_is_first_arg=True)
        def get_posibilities(evaluator, module, func_name):
            try:
                names = module.used_names[func_name]
            except KeyError:
                return []

            for name in names:
                stmt = name.get_definition()
                if not isinstance(stmt, (pr.ExprStmt, pr.CompFor, pr.ReturnStmt)):
                    continue
                parent = name.parent
                if pr.is_node(parent, 'trailer'):
                    parent = parent.parent

                trailer = None
                if pr.is_node(parent, 'power'):
                    for t in parent.children[1:]:
                        if t == '**':
                            break
                        if t.start_pos > name.start_pos and t.children[0] == '(':
                            trailer = t
                            break
                if trailer is not None:
                    types = evaluator.goto_definition(name)

                    # We have to remove decorators, because they are not the
                    # "original" functions, this way we can easily compare.
                    # At the same time we also have to remove InstanceElements.
                    undec = []
                    for escope in types:
                        if escope.isinstance(er.Function, er.Instance) \
                                and escope.decorates is not None:
                            undec.append(escope.decorates)
                        elif isinstance(escope, er.InstanceElement):
                            undec.append(escope.var)
                        else:
                            undec.append(escope)

                    if er.wrap(evaluator, compare) in undec:
                        # Only if we have the correct function we execute
                        # it, otherwise just ignore it.
                        evaluator.eval_trailer(types, trailer)

                # TODO REMOVE
                continue
                calls = helpers.scan_statement_for_calls(stmt, func_name)
                for c in calls:
                    # no execution means that params cannot be set
                    call_path = list(c.generate_call_path())
                    pos = c.start_pos
                    scope = stmt.parent

                    # This whole stuff is just to not execute certain parts
                    # (speed improvement), basically we could just call
                    # ``eval_call_path`` on the call_path and it would also
                    # work.
                    def listRightIndex(lst, value):
                        return len(lst) - lst[-1::-1].index(value) - 1

                    # Need to take right index, because there could be a
                    # func usage before.
                    call_path_simple = [unicode(d) if isinstance(d, pr.Name)
                                        else d for d in call_path]
                    i = listRightIndex(call_path_simple, func_name)
                    before, after = call_path[:i], call_path[i + 1:]
                    if not after and not call_path_simple.index(func_name) != i:
                        continue
                    scopes = [scope]
                    if before:
                        scopes = evaluator.eval_call_path(iter(before), c.parent, pos)
                        pos = None
                    for scope in scopes:
                        # Not resolving decorators is a speed hack:
                        # By ignoring them, we get the function that is
                        # probably called really fast. If it's not called, it
                        # doesn't matter. But this is a way to get potential
                        # candidates for calling that function really quick!
                        s = evaluator.find_types(scope, func_name, position=pos,
                                                 search_global=not before,
                                                 resolve_decorator=False)

                        c = [getattr(escope, 'base_func', None) or escope.base
                             for escope in s
                             if escope.isinstance(er.Function, er.Class)]
                        if compare in c:
                            # only if we have the correct function we execute
                            # it, otherwise just ignore it.
                            evaluator.follow_path(iter(after), s, scope)
            return listener.param_possibilities
        return get_posibilities(evaluator, module, func_name)

    current_module = func.get_parent_until()
    func_name = unicode(func.name)
    compare = func
    if func_name == '__init__':
        cls = func.get_parent_scope()
        if isinstance(cls, pr.Class):
            func_name = unicode(cls.name)
            compare = cls

    # add the listener
    listener = ParamListener()
    func.listeners.add(listener)

    try:
        result = []
        # This is like backtracking: Get the first possible result.
        for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name):
            result = get_params_for_module(mod)
            if result:
                break
    finally:
        # cleanup: remove the listener; important: should not stick.
        func.listeners.remove(listener)

    return result
<commit_msg>Clean up the dynamic module.<commit_after>"""
One of the really important features of |jedi| is to have an option to
understand code like this::

    def foo(bar):
        bar. # completion here
    foo(1)

There's no doubt wheter bar is an ``int`` or not, but if there's also a call
like ``foo('str')``, what would happen? Well, we'll just show both. Because
that's what a human would expect.

It works as follows:

- |Jedi| sees a param
- search for function calls named ``foo``
- execute these calls and check the input. This work with a ``ParamListener``.
"""
from itertools import chain

from jedi._compatibility import unicode
from jedi.parser import tree as pr
from jedi import settings
from jedi import debug
from jedi.evaluate.cache import memoize_default
from jedi.evaluate import imports

class ParamListener(object):
    """
    This listener is used to get the params for a function.
    """
    def __init__(self):
        self.param_possibilities = []

    def execute(self, params):
        self.param_possibilities += params

@debug.increase_indent
def search_params(evaluator, param):
    """
    A dynamic search for param values. If you try to complete a type:

    >>> def func(foo):
    ...     foo
    >>> func(1)
    >>> func("")

    It is not known what the type ``foo`` without analysing the whole code. You
    have to look for all calls to ``func`` to find out what ``foo`` possibly
    is.
    """
    if not settings.dynamic_params:
        return []
    debug.dbg('Dynamic param search for %s', param)

    func = param.get_parent_until(pr.Function)
    """
        for params in get_posibilities(evaluator, module, func_name):
            for p in params:
                if str(p) == str(param.get_name()):
                    result += p.parent.eval(evaluator)
                    """
    # Compare the param names.
    names = [n for n in search_function_call(evaluator, func)
             if n.value == param.name.value]
    # Evaluate the ExecutedParams to types.
    result = list(chain.from_iterable(n.parent.eval(evaluator) for n in names))
    debug.dbg('Dynamic param result %s', result)
    return result

@memoize_default([], evaluator_is_first_arg=True)
def search_function_call(evaluator, func):
    """
    Returns a list of param names.
    """
    from jedi.evaluate import representation as er

    def get_params_for_module(module):
        """
        Returns the values of a param, or an empty array.
        """
        @memoize_default([], evaluator_is_first_arg=True)
        def get_posibilities(evaluator, module, func_name):
            try:
                names = module.used_names[func_name]
            except KeyError:
                return []

            for name in names:
                stmt = name.get_definition()
                if not isinstance(stmt, (pr.ExprStmt, pr.CompFor, pr.ReturnStmt)):
                    continue
                parent = name.parent
                if pr.is_node(parent, 'trailer'):
                    parent = parent.parent

                trailer = None
                if pr.is_node(parent, 'power'):
                    for t in parent.children[1:]:
                        if t == '**':
                            break
                        if t.start_pos > name.start_pos and t.children[0] == '(':
                            trailer = t
                            break
                if trailer is not None:
                    types = evaluator.goto_definition(name)

                    # We have to remove decorators, because they are not the
                    # "original" functions, this way we can easily compare.
                    # At the same time we also have to remove InstanceElements.
                    undec = []
                    for escope in types:
                        if escope.isinstance(er.Function, er.Instance) \
                                and escope.decorates is not None:
                            undec.append(escope.decorates)
                        elif isinstance(escope, er.InstanceElement):
                            undec.append(escope.var)
                        else:
                            undec.append(escope)

                    if er.wrap(evaluator, compare) in undec:
                        # Only if we have the correct function we execute
                        # it, otherwise just ignore it.
                        evaluator.eval_trailer(types, trailer)
            return listener.param_possibilities
        return get_posibilities(evaluator, module, func_name)

    current_module = func.get_parent_until()
    func_name = unicode(func.name)
    compare = func
    if func_name == '__init__':
        cls = func.get_parent_scope()
        if isinstance(cls, pr.Class):
            func_name = unicode(cls.name)
            compare = cls

    # add the listener
    listener = ParamListener()
    func.listeners.add(listener)

    try:
        result = []
        # This is like backtracking: Get the first possible result.
        for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name):
            result = get_params_for_module(mod)
            if result:
                break
    finally:
        # cleanup: remove the listener; important: should not stick.
        func.listeners.remove(listener)

    return result
<|endoftext|>


<commit_before>import bson
import eventlet
import mock
import mongoengine as me
import unittest2
from oslo.config import cfg

from st2tests import DbTestCase
from st2common.models.db import MongoDBAccess
from st2common.persistence import Access
from st2common.util import watch

class ExampleDB(me.Document):
    name = me.StringField()

class Example(Access):
    IMPL = MongoDBAccess(ExampleDB)

    @classmethod
    def _get_impl(kls):
        return kls.IMPL

class WatcherTest(DbTestCase):

    def tearDown(self):
        watch._clear_watcher()
        super(WatcherTest, self).tearDown()

    def test_watcher_started(self):
        watcher = watch.get_watcher()
        self.assertTrue(watcher.status())

    def test_watcher_no_auto_start(self):
        watcher = watch.get_watcher(False)
        self.assertFalse(watcher.status())

    def test_watcher_stop(self):
        watcher = watch.get_watcher()

        watcher.stop()

        self.assertFalse(watcher.status())

    def test_watcher_start_running(self):
        watcher = watch.get_watcher()

        self.assertRaises(Exception, watcher.start)

    def test_watcher_stop_not_running(self):
        watcher = watch.get_watcher()
        watcher.stop()

        self.assertRaises(Exception, watcher.stop)

    @unittest2.skip('Test fails on jenkins. Perhaps a race caused by eventlet.sleep.')
    def test_watcher_watches_default(self):
        watcher = watch.get_watcher()
        func = mock.MagicMock()

        watcher.watch(func, ExampleDB)

        eventlet.sleep(0)

        doc = self._create_and_save_test_document()
        self._delete_test_document(doc)

        eventlet.sleep(1)

        self.assertEqual(2, func.call_count)
        self._assert_call_args(func.call_args_list[0][0], dict(doc.to_mongo()), watch.INSERT)
        self._assert_call_args(func.call_args_list[1][0], {'_id': doc.id}, watch.DELETE)

    @unittest2.skip('Test fails on jenkins. Perhaps a race caused by eventlet.sleep.')
    def test_watcher_watches_operation(self):
        watcher = watch.get_watcher()
        func = mock.MagicMock()

        watcher.watch(func, ExampleDB, watch.INSERT)

        eventlet.sleep(0)

        doc = self._create_and_save_test_document()
        self._delete_test_document(doc)

        eventlet.sleep(1)

        self.assertEqual(1, func.call_count)
        self._assert_call_args(func.call_args_list[0][0], dict(doc.to_mongo()), watch.INSERT)

    @staticmethod
    def _create_and_save_test_document():
        return Example.add_or_update(ExampleDB(name='test'))

    @staticmethod
    def _delete_test_document(doc):
        return Example.delete(doc)

    def _assert_call_args(self, args, doc, operation):
        self.assertEqual(args[0], 'st2-test.example_d_b')
        self.assertIsInstance(args[1], bson.Timestamp)
        self.assertEqual(args[2], operation)
        self.assertEqual(args[3], doc['_id'])
        self.assertEqual(args[4], doc)
<commit_msg>Revert "Disable broken tests."<commit_after>import bson
import eventlet
import mock
import mongoengine as me
from oslo.config import cfg

from st2tests import DbTestCase
from st2common.models.db import MongoDBAccess
from st2common.persistence import Access
from st2common.util import watch

class ExampleDB(me.Document):
    name = me.StringField()

class Example(Access):
    IMPL = MongoDBAccess(ExampleDB)

    @classmethod
    def _get_impl(kls):
        return kls.IMPL

class WatcherTest(DbTestCase):

    def tearDown(self):
        watch._clear_watcher()
        super(WatcherTest, self).tearDown()

    def test_watcher_started(self):
        watcher = watch.get_watcher()
        self.assertTrue(watcher.status())

    def test_watcher_no_auto_start(self):
        watcher = watch.get_watcher(False)
        self.assertFalse(watcher.status())

    def test_watcher_stop(self):
        watcher = watch.get_watcher()

        watcher.stop()

        self.assertFalse(watcher.status())

    def test_watcher_start_running(self):
        watcher = watch.get_watcher()

        self.assertRaises(Exception, watcher.start)

    def test_watcher_stop_not_running(self):
        watcher = watch.get_watcher()
        watcher.stop()

        self.assertRaises(Exception, watcher.stop)

    def test_watcher_watches_default(self):
        watcher = watch.get_watcher()
        func = mock.MagicMock()

        watcher.watch(func, ExampleDB)

        eventlet.sleep(0)

        doc = self._create_and_save_test_document()
        self._delete_test_document(doc)

        eventlet.sleep(1)

        self.assertEqual(2, func.call_count)
        self._assert_call_args(func.call_args_list[0][0], dict(doc.to_mongo()), watch.INSERT)
        self._assert_call_args(func.call_args_list[1][0], {'_id': doc.id}, watch.DELETE)

    def test_watcher_watches_operation(self):
        watcher = watch.get_watcher()
        func = mock.MagicMock()

        watcher.watch(func, ExampleDB, watch.INSERT)

        eventlet.sleep(0)

        doc = self._create_and_save_test_document()
        self._delete_test_document(doc)

        eventlet.sleep(1)

        self.assertEqual(1, func.call_count)
        self._assert_call_args(func.call_args_list[0][0], dict(doc.to_mongo()), watch.INSERT)

    @staticmethod
    def _create_and_save_test_document():
        return Example.add_or_update(ExampleDB(name='test'))

    @staticmethod
    def _delete_test_document(doc):
        return Example.delete(doc)

    def _assert_call_args(self, args, doc, operation):
        self.assertEqual(args[0], 'st2-test.example_d_b')
        self.assertIsInstance(args[1], bson.Timestamp)
        self.assertEqual(args[2], operation)
        self.assertEqual(args[3], doc['_id'])
        self.assertEqual(args[4], doc)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function
from sys import argv, stderr
from time import sleep
import liblo

def print_e(*args, **kwargs):
    "Print on standard error, for things that should not be piped"
    # TODO use a logger, maybe
    print(*args, file=stderr, **kwargs)

def print_log(port=1234):
    "Display every message received on the given port (default 1234)"

    print_e("Logging port:", port)
    print_e()

    def callback(path, msg):
        print(path, *msg)

    server = liblo.ServerThread(int(port))
    server.add_method(None, None, callback)  # wildcard callback
    server.start()

    try:
        while True:
            sleep(0.1)
    except KeyboardInterrupt:
        server.stop()

def send(addr, path, *msg):
    "Send message on a given path: ip:port path msg"
    target = liblo.Address("osc.udp://" + addr + "/")
    liblo.send(target, path, *msg)

def forward(out_addr, in_port=1234):
    "Forward received messages: out_ip:out_port in_port"

    print_e("Forwarding  port:", in_port, "to", out_addr)
    print_e()

    def callback(path, msg):
        print(path, *msg)
        send(out_addr, path, *msg)

    server = liblo.ServerThread(int(in_port))
    server.add_method(None, None, callback)  # wildcard callback
    server.start()

    try:
        while True:
            sleep(0.1)
    except KeyboardInterrupt:
        server.stop()

def main():
    "Main function, used when called as script"
    commands = {}

    def print_help():
        "Display commands description"
        print_e("Usage:")
        # generate help from docstrings
        for cmd in commands:
            print_e("  ", argv[0], cmd, "\t", commands[cmd].__doc__)

    commands["log"] = print_log
    commands["send"] = send
    commands["forward"] = forward
    commands["help"] = print_help

    if not argv[1:]:
        print_help()
        return

    cmd = argv[1]
    args = argv[2:]

    try:
        commands.get(cmd, print_help)(*args)
    except Exception as e:
        print_e(e)
        print_e()
        print_help()

if __name__ == "__main__":
    main()
<commit_msg>Suppert callbacks from bash<commit_after>#!/usr/bin/env python
from __future__ import print_function
from sys import argv, stderr
from os import system
from time import sleep
import liblo

def print_e(*args, **kwargs):
    "Print on standard error, for things that should not be piped"
    # TODO use a logger, maybe
    print(*args, file=stderr, **kwargs)

def handle(port, cmd):
    "Do something on message"

    if hasattr(cmd, "__call__"):
        # when called from python
        callback = cmd
    else:
        # when called from bash
        def callback(path, msg):
            system("{} {} {}".format(cmd, path, " ".join(msg)))

    server = liblo.ServerThread(int(port))
    server.add_method(None, None, callback)  # wildcard callback
    server.start()

    try:
        while True:
            sleep(0.1)
    except KeyboardInterrupt:
        server.stop()

def print_log(port=1234):
    "Display every message received on the given port (default 1234)"

    print_e("Logging port:", port)
    print_e()

    def callback(path, msg):
        print(path, *msg)

    handle(port, callback)

def send(addr, path, *msg):
    "Send message on a given path: ip:port path msg"
    target = liblo.Address("osc.udp://" + addr + "/")
    liblo.send(target, path, *msg)

def forward(out_addr, in_port=1234):
    "Forward received messages: out_ip:out_port in_port"

    print_e("Forwarding  port:", in_port, "to", out_addr)
    print_e()

    def callback(path, msg):
        print(path, *msg)
        send(out_addr, path, *msg)

    handle(in_port, callback)

def main():
    "Main function, used when called as script"
    commands = {}

    def print_help():
        "Display commands description"
        print_e("Usage:")
        # generate help from docstrings
        for cmd in commands:
            print_e("  ", argv[0], cmd, "\t", commands[cmd].__doc__)

    commands["log"] = print_log
    commands["send"] = send
    commands["forward"] = forward
    commands["handle"] = handle
    commands["help"] = print_help

    if not argv[1:]:
        print_help()
        return

    cmd = argv[1]
    args = argv[2:]

    try:
        commands.get(cmd, print_help)(*args)
    except Exception as e:
        print_e(e)
        print_e()
        print_help()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from collections import namedtuple
import requests

from .tasks import Task

TASK_TYPES = [
    'categorization',
    'transcription',
    'phonecall',
    'comparison',
    'annotation',
    'polygonannotation',
    'lineannotation',
    'datacollection',
    'audiotranscription',
    'pointannotation',
    'segmentannotation'
]
SCALE_ENDPOINT = 'https://api.scaleapi.com/v1/'
DEFAULT_LIMIT = 100
DEFAULT_OFFSET = 0

class ScaleException(Exception):
    def __init__(self, message, errcode):
        super(ScaleException, self).__init__(message)
        self.code = errcode

class ScaleInvalidRequest(ScaleException, ValueError):
    pass

class Tasklist(list):
    def __init__(self, docs, total, limit, offset, has_more):
        super(Tasklist, self).__init__(docs)
        self.docs = docs
        self.total = total
        self.limit = limit
        self.offset = offset
        self.has_more = has_more

class ScaleClient(object):
    def __init__(self, api_key):
        self.api_key = api_key

    def _getrequest(self, endpoint, params={}):
        """Makes a get request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        r = requests.get(SCALE_ENDPOINT + endpoint,
                         headers={"Content-Type": "application/json"},
                         auth=(self.api_key, ''), params=params)

        if r.status_code == 200:
            return r.json()
        raise ScaleException(r.json()['error'], r.status_code)

    def _postrequest(self, endpoint, payload=None):
        """Makes a post request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        payload = payload or {}
        r = requests.post(SCALE_ENDPOINT + endpoint, json=payload,
                          headers={"Content-Type": "application/json"},
                          auth=(self.api_key, ''))

        if r.status_code == 200:
            return r.json()
        if r.status_code == 400:
            raise ScaleInvalidRequest(r.json()['error'], r.status_code)
        raise ScaleException(r.json()['error'], r.status_code)

    def fetch_task(self, task_id):
        """Fetches a task.

        Returns the associated task.
        """
        return Task(self._getrequest('task/%s' % task_id), self)

    def cancel_task(self, task_id):
        """Cancels a task.

        Returns the associated task.
        Raises a ScaleException if it has already been canceled.
        """
        return Task(self._postrequest('task/%s/cancel' % task_id), self)

    def tasks(self, **kwargs):
        """Returns a list of your tasks.
        Returns up to 100 at a time, to get more use the offset param.

        start/end_time are ISO8601 dates, the time range of tasks to fetch.
        status can be 'completed', 'pending', or 'canceled'.
        type is the task type.
        limit is the max number of results to display per page,
        offset is the number of results to skip (for showing more pages).
        """
        allowed_kwargs = {'start_time', 'end_time', 'status', 'type', 'limit', 'offset'}
        for key in kwargs:
            if key not in allowed_kwargs:
                raise ScaleInvalidRequest('Illegal parameter %s for ScaleClient.tasks()'
                                          % key, None)
        response = self._getrequest('tasks', params=kwargs)
        docs = [Task(json, self) for json in response['docs']]
        return Tasklist(docs, response['total'], response['limit'],
                        response['offset'], response['has_more'])
    def create_task(self, task_type, **kwargs):
        endpoint = 'task/' + task_type
        taskdata = self._postrequest(endpoint, payload=kwargs)
        return Task(taskdata, self)

def _AddTaskTypeCreator(task_type):
    def create_task_wrapper(self, **kwargs):
        return self.create_task(task_type, **kwargs)
    setattr(ScaleClient, 'create_' + task_type + '_task', create_task_wrapper)

for taskType in TASK_TYPES:
    _AddTaskTypeCreator(taskType)
<commit_msg>better exception message; handle failed requests<commit_after>from collections import namedtuple
import requests

from .tasks import Task

TASK_TYPES = [
    'categorization',
    'transcription',
    'phonecall',
    'comparison',
    'annotation',
    'polygonannotation',
    'lineannotation',
    'datacollection',
    'audiotranscription',
    'pointannotation',
    'segmentannotation'
]
SCALE_ENDPOINT = 'https://api.scaleapi.com/v1/'
DEFAULT_LIMIT = 100
DEFAULT_OFFSET = 0

class ScaleException(Exception):
    def __init__(self, message, errcode):
        super(ScaleException, self).__init__('<Response [{}]> {}'.format(errcode, message))
        self.code = errcode

class ScaleInvalidRequest(ScaleException, ValueError):
    pass

class Tasklist(list):
    def __init__(self, docs, total, limit, offset, has_more):
        super(Tasklist, self).__init__(docs)
        self.docs = docs
        self.total = total
        self.limit = limit
        self.offset = offset
        self.has_more = has_more

class ScaleClient(object):
    def __init__(self, api_key):
        self.api_key = api_key

    def _getrequest(self, endpoint, params={}):
        """Makes a get request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        r = requests.get(SCALE_ENDPOINT + endpoint,
                         headers={"Content-Type": "application/json"},
                         auth=(self.api_key, ''), params=params)

        if r.status_code == 200:
            return r.json()
        else:
            try:
                raise ScaleException(r.json()['error'], r.status_code)
            except ValueError:
                pass
            raise ScaleException(r.text, r.status_code)

    def _postrequest(self, endpoint, payload=None):
        """Makes a post request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        payload = payload or {}
        r = requests.post(SCALE_ENDPOINT + endpoint, json=payload,
                          headers={"Content-Type": "application/json"},
                          auth=(self.api_key, ''))

        if r.status_code == 200:
            return r.json()
        else:
            try:
                raise ScaleException(r.json()['error'], r.status_code)
            except ValueError:
                pass
            raise ScaleException(r.text, r.status_code)

    def fetch_task(self, task_id):
        """Fetches a task.

        Returns the associated task.
        """
        return Task(self._getrequest('task/%s' % task_id), self)

    def cancel_task(self, task_id):
        """Cancels a task.

        Returns the associated task.
        Raises a ScaleException if it has already been canceled.
        """
        return Task(self._postrequest('task/%s/cancel' % task_id), self)

    def tasks(self, **kwargs):
        """Returns a list of your tasks.
        Returns up to 100 at a time, to get more use the offset param.

        start/end_time are ISO8601 dates, the time range of tasks to fetch.
        status can be 'completed', 'pending', or 'canceled'.
        type is the task type.
        limit is the max number of results to display per page,
        offset is the number of results to skip (for showing more pages).
        """
        allowed_kwargs = {'start_time', 'end_time', 'status', 'type', 'limit', 'offset'}
        for key in kwargs:
            if key not in allowed_kwargs:
                raise ScaleInvalidRequest('Illegal parameter %s for ScaleClient.tasks()'
                                          % key, None)
        response = self._getrequest('tasks', params=kwargs)
        docs = [Task(json, self) for json in response['docs']]
        return Tasklist(docs, response['total'], response['limit'],
                        response['offset'], response['has_more'])
    def create_task(self, task_type, **kwargs):
        endpoint = 'task/' + task_type
        taskdata = self._postrequest(endpoint, payload=kwargs)
        return Task(taskdata, self)

def _AddTaskTypeCreator(task_type):
    def create_task_wrapper(self, **kwargs):
        return self.create_task(task_type, **kwargs)
    setattr(ScaleClient, 'create_' + task_type + '_task', create_task_wrapper)

for taskType in TASK_TYPES:
    _AddTaskTypeCreator(taskType)
<|endoftext|>


<commit_before>from collections import namedtuple
import requests

from .tasks import Task

TASK_TYPES = [
  'categorization',
  'transcription',
  'phonecall',
  'comparison',
  'annotation',
  'polygonannotation',
  'lineannotation',
  'datacollection',
  'audiotranscription',
  'pointannotation',
  'segmentannotation'
]
SCALE_ENDPOINT = 'https://api.scaleapi.com/v1/'
DEFAULT_LIMIT = 100
DEFAULT_OFFSET = 0

class ScaleException(Exception):
    def __init__(self, message, errcode):
        super(ScaleException, self).__init__(message)
        self.code = errcode

class ScaleInvalidRequest(ScaleException, ValueError):
    pass

class Tasklist(list):
    def __init__(self, docs, total, limit, offset, has_more):
        super(Tasklist, self).__init__(docs)
        self.docs = docs
        self.total = total
        self.limit = limit
        self.offset = offset
        self.has_more = has_more

class ScaleClient(object):
    def __init__(self, api_key):
        self.api_key = api_key

    def _getrequest(self, endpoint, params={}):
        """Makes a get request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        r = requests.get(SCALE_ENDPOINT + endpoint,
                         headers={"Content-Type": "application/json"},
                         auth=(self.api_key, ''), params=params)

        if r.status_code == 200:
            return r.json()
        raise ScaleException(r.json()['error'], r.status_code)

    def _postrequest(self, endpoint, payload=None):
        """Makes a post request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        payload = payload or {}
        r = requests.post(SCALE_ENDPOINT + endpoint, json=payload,
                          headers={"Content-Type": "application/json"},
                          auth=(self.api_key, ''))

        if r.status_code == 200:
            return r.json()
        if r.status_code == 400:
            raise ScaleInvalidRequest(r.json()['error'], r.status_code)
        raise ScaleException(r.json()['error'], r.status_code)

    def fetch_task(self, task_id):
        """Fetches a task.

        Returns the associated task.
        """
        return Task(self._getrequest('task/%s' % task_id), self)

    def cancel_task(self, task_id):
        """Cancels a task.

        Returns the associated task.
        Raises a ScaleException if it has already been canceled.
        """
        return Task(self._postrequest('task/%s/cancel' % task_id), self)

    def tasks(self, **kwargs):
        """Returns a list of your tasks.
        Returns up to 100 at a time, to get more use the offset param.

        start/end_time are ISO8601 dates, the time range of tasks to fetch.
        status can be 'completed', 'pending', or 'canceled'.
        type is the task type.
        limit is the max number of results to display per page,
        offset is the number of results to skip (for showing more pages).
        """
        allowed_kwargs = {'start_time', 'end_time', 'status', 'type', 'limit', 'offset'}
        for key in kwargs:
            if key not in allowed_kwargs:
                raise ScaleInvalidRequest('Illegal parameter %s for ScaleClient.tasks()'
                                          % key, None)
        response = self._getrequest('tasks', params=kwargs)
        docs = [Task(json, self) for json in response['docs']]
        return Tasklist(docs, response['total'], response['limit'],
                        response['offset'], response['has_more'])

def _AddTaskTypeCreator(name):
    endpoint = 'task/' + ('categorize' if name == 'categorization' else name)
    def createTask(self, **kwargs):
        taskdata = self._postrequest(endpoint, payload=kwargs)
        return Task(taskdata, self)
    setattr(ScaleClient, 'create_' + name + '_task', createTask)

for taskType in TASK_TYPES:
    _AddTaskTypeCreator(taskType)
<commit_msg>/categorize is equivelant to /cagtegorization<commit_after>from collections import namedtuple
import requests

from .tasks import Task

TASK_TYPES = [
  'categorization',
  'transcription',
  'phonecall',
  'comparison',
  'annotation',
  'polygonannotation',
  'lineannotation',
  'datacollection',
  'audiotranscription',
  'pointannotation',
  'segmentannotation'
]
SCALE_ENDPOINT = 'https://api.scaleapi.com/v1/'
DEFAULT_LIMIT = 100
DEFAULT_OFFSET = 0

class ScaleException(Exception):
    def __init__(self, message, errcode):
        super(ScaleException, self).__init__(message)
        self.code = errcode

class ScaleInvalidRequest(ScaleException, ValueError):
    pass

class Tasklist(list):
    def __init__(self, docs, total, limit, offset, has_more):
        super(Tasklist, self).__init__(docs)
        self.docs = docs
        self.total = total
        self.limit = limit
        self.offset = offset
        self.has_more = has_more

class ScaleClient(object):
    def __init__(self, api_key):
        self.api_key = api_key

    def _getrequest(self, endpoint, params={}):
        """Makes a get request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        r = requests.get(SCALE_ENDPOINT + endpoint,
                         headers={"Content-Type": "application/json"},
                         auth=(self.api_key, ''), params=params)

        if r.status_code == 200:
            return r.json()
        raise ScaleException(r.json()['error'], r.status_code)

    def _postrequest(self, endpoint, payload=None):
        """Makes a post request to an endpoint.

        If an error occurs, assumes that endpoint returns JSON as:
            { 'status_code': XXX,
              'error': 'I failed' }
        """
        payload = payload or {}
        r = requests.post(SCALE_ENDPOINT + endpoint, json=payload,
                          headers={"Content-Type": "application/json"},
                          auth=(self.api_key, ''))

        if r.status_code == 200:
            return r.json()
        if r.status_code == 400:
            raise ScaleInvalidRequest(r.json()['error'], r.status_code)
        raise ScaleException(r.json()['error'], r.status_code)

    def fetch_task(self, task_id):
        """Fetches a task.

        Returns the associated task.
        """
        return Task(self._getrequest('task/%s' % task_id), self)

    def cancel_task(self, task_id):
        """Cancels a task.

        Returns the associated task.
        Raises a ScaleException if it has already been canceled.
        """
        return Task(self._postrequest('task/%s/cancel' % task_id), self)

    def tasks(self, **kwargs):
        """Returns a list of your tasks.
        Returns up to 100 at a time, to get more use the offset param.

        start/end_time are ISO8601 dates, the time range of tasks to fetch.
        status can be 'completed', 'pending', or 'canceled'.
        type is the task type.
        limit is the max number of results to display per page,
        offset is the number of results to skip (for showing more pages).
        """
        allowed_kwargs = {'start_time', 'end_time', 'status', 'type', 'limit', 'offset'}
        for key in kwargs:
            if key not in allowed_kwargs:
                raise ScaleInvalidRequest('Illegal parameter %s for ScaleClient.tasks()'
                                          % key, None)
        response = self._getrequest('tasks', params=kwargs)
        docs = [Task(json, self) for json in response['docs']]
        return Tasklist(docs, response['total'], response['limit'],
                        response['offset'], response['has_more'])

def _AddTaskTypeCreator(name):
    endpoint = 'task/' + name
    def createTask(self, **kwargs):
        taskdata = self._postrequest(endpoint, payload=kwargs)
        return Task(taskdata, self)
    setattr(ScaleClient, 'create_' + name + '_task', createTask)

for taskType in TASK_TYPES:
    _AddTaskTypeCreator(taskType)
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-

import re
import yaml

from os.path import basename, abspath
from sys import exit, argv
from copy import deepcopy
from getopt import getopt
from getpass import getpass
from functools import partial
from subprocess import CalledProcessError

from lxml import etree
from jenkins import Jenkins, JenkinsError
from requests.exceptions import RequestException, HTTPError

from jenkins_autojobs import __version__
from jenkins_autojobs.util import *

#-----------------------------------------------------------------------------
# Compatibility imports.
try:
    from itertools import ifilterfalse as filterfalse
except ImportError:
    from itertools import filterfalse

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

try:
    input = raw_input
except NameError:
    pass

#-----------------------------------------------------------------------------
usage = '''\
Usage: %s [-rvdtjnyoupUYOP] <config.yaml>

General Options:
  -n dry run
  -v show version and exit
  -d debug config inheritance
  -t debug http requests

Repository Options:
  -r <arg> repository url
  -y <arg> scm username
  -o <arg> scm password
  -Y scm username (read from stdin)
  -O scm password (read from stdin)

Jenkins Options:
  -j <arg> jenkins url
  -u <arg> jenkins username
  -p <arg> jenkins password
  -U jenkins username (read from stdin)
  -P jenkins password (read from stdin)\
''' % basename(argv[0])

#-----------------------------------------------------------------------------
# The *global* connection to jenkins - assigned in main().
jenkins = None

def main(argv, create_job, list_branches, getoptfmt='vdtnr:j:u:p:y:o:UPYO', config=None):
    '''
    :param argv: command-line arguments to parse (defaults to sys.argv[1:])
    :param create_job: scm specific function that configures and creates jobs
    :param list_branches: scm specific function that lists all branches/refs
    :param getoptfmt: getopt short and long options
    :param config: a config dictionary to use instead of parsing the
                   configuration from yaml (useful for testing)
    '''

    if '-h' in argv or '--help' in argv:
        print(usage)
        exit(1)

    opts, args = parse_args(argv, getoptfmt)
    if not args and not config:
        print(usage)
        exit(1)

    # Load config, set default values and compile regexes.
    if not config :
        yamlfn = args[-1]
        print('loading config from "%s"' % abspath(yamlfn))
        config = yaml.load(open(yamlfn))

    config = c = get_default_config(config, opts)

    if config['debughttp']:
        enable_http_logging()

    # Connect to jenkins.
    try:
        global jenkins
        jenkins = main.jenkins = Jenkins(c['jenkins'], c['username'], c['password'])
    except (RequestException, JenkinsError) as e:
        print(e); exit(1)

    #-------------------------------------------------------------------------
    # Get all the template names that the config references.
    templates = set(i['template'] for i in c['refs'].values())

    # Check if all referenced template jobs exist on the server.
    missing = list(filterfalse(jenkins.job_exists, templates))
    if missing:
        missing.insert(0, '\nconfig references non-existent template jobs:')
        print('\n - '.join(missing)); exit(1)

    # Convert them to etree objects of the templates' config xmls.
    templates = dict((i, get_job_etree(i)) for i in templates)

    #-------------------------------------------------------------------------
    # Check if all referenced views exist.
    view_names = set(view for i in c['refs'].values() for view in i['view'])
    missing = list(filterfalse(jenkins.view_exists, view_names))
    if missing:
        missing.insert(0, '\nconfig references non-existent views:')
        print('\n - '.join(missing)); exit(1)

    #-------------------------------------------------------------------------
    # List all git refs, svn branches etc (implemented by child classes).
    try:
        branches = list(list_branches(config))
    except CalledProcessError as e:
        print('! cannot list branches')
        print('! command %s failed' % ' '.join(e.cmd))
        exit(1)

    # See if any of the branches are ignored.
    ignored, branches = get_ignored(branches, c['ignore'])

    if ignored:
        msg = ['\nexplicitly ignored:'] + ignored
        print('\n - '.join(msg))

    # Get branch config for each branch.
    configs = map(partial(resolveconfig, config), branches)
    configs = zip(branches, configs)
    configs = filter(lambda x: bool(x[1]), configs)

    # The names of all successfully created or updated jobs.
    job_names = {}
    for branch, branch_config in configs:
        tmpl = templates[branch_config['template']]
        job_name = create_job(branch, tmpl, config, branch_config)
        job_names[job_name] = branch_config

        # Add newly create jobs to views, if any.
        views = branch_config['view']
        for view_name in views:
            view = jenkins.view(view_name)
            if job_name in view:
                print('. job already in view: %s' % view_name)
            else:
                if not config['dryrun']:
                    jenkins.view_add_job(view_name, job_name)
                print('. job added to view: %s' % view_name)

    if config['cleanup']:
        job_names[config['template']] = {}
        cleanup(config, job_names, jenkins)

#-----------------------------------------------------------------------------
def cleanup(config, job_names, jenkins, verbose=True):
    print('\ncleaning up old jobs:')

    tag = '</createdByJenkinsAutojobs>'
    tagxpath = 'createdByJenkinsAutojobs/tag/text()'

    managed_jobs = (job for job in jenkins.jobs if tag in job.config)
    removed_jobs = []

    for job in managed_jobs:
        if (job.name not in job_names) and job.exists:
            # If cleanup is a tag name, only cleanup builds with that tag.
            if isinstance(config['cleanup'], str):
                xml = etree.fromstring(job.config.encode('utf8'))
                clean_tag = xml.xpath(tagxpath)
                if not config['cleanup'] in clean_tag:
                    continue

            removed_jobs.append(job)
            if not config['dryrun']:
                job_removed = safe_job_delete(job)
            else:
                job_removed = True

            if job_removed:
                print(' - %s' % job.name)
            else:
                print(' ! permission denied for %s' % job.name)

    if not removed_jobs:
        print('. nothing to do')

def safe_job_delete(job, safe_codes=(403,)):
    try:
        job.delete()
        return True
    except HTTPError as error:
        if error.status_code not in safe_codes:
            raise
        return False

#-----------------------------------------------------------------------------
def parse_args(argv, fmt):
    '''Parse getopt arguments as a dictionary.'''
    opts, args = getopt(argv, fmt)
    opts = dict(opts)

    if '-v' in opts:
        print('jenkins-autojobs version %s' % __version__)
        exit(0)

    return opts, args

#-----------------------------------------------------------------------------
def get_default_config(config, opts):
    '''Set default config values and compile regexes.'''

    c, o = deepcopy(config), opts

    # Default global settings (not inheritable).
    c['dryrun'] = False
    c['debug']  = config.get('debug', False)
    c['debughttp'] = config.get('debughttp', False)
    c['cleanup']   = config.get('cleanup', False)
    c['username']  = config.get('username', None)
    c['password']  = config.get('password', None)
    c['scm-username'] = config.get('scm-username', None)
    c['scm-password'] = config.get('scm-password', None)

    # Default settings for each git ref/branch/ config.
    c['defaults'] = {
        'namesep':         c.get('namesep', '-'),
        'namefmt':         c.get('namefmt', '{shortref}'),
        'overwrite':       c.get('overwrite', True),
        'enable':          c.get('enable', 'sticky'),
        'substitute':      c.get('substitute', {}),
        'template':        c.get('template'),
        'sanitize':        c.get('sanitize', {'@!?#&|\^_$%*': '_'}),
        'tag':             c.get('tag', []),
        'view':            c.get('view', []),
        'build-on-create': c.get('build-on-create', False)
    }

    # Make sure some options are always lists.
    c['defaults']['view'] = pluralize(c['defaults']['view'])

    # Some options can be overwritten on the command line.
    if '-r' in o: c['repo'] = o['-r']
    if '-j' in o: c['jenkins'] = o['-j']
    if '-n' in o: c['dryrun'] = True
    if '-d' in o: c['debug'] = True
    if '-t' in o: c['debughttp'] = True

    # Jenkins authentication options.
    if '-u' in o: c['username'] = o['-u']
    if '-p' in o: c['password'] = o['-p']
    if '-U' in o: c['username'] = input('Jenkins User: ')
    if '-P' in o: c['password'] = getpass('Jenkins Password: ')

    # SCM authentication options.
    if '-y' in o: c['scm-username'] = o['-y']
    if '-o' in o: c['scm-password'] = o['-o']
    if '-Y' in o: c['scm-username'] = input('SCM User: ')
    if '-O' in o: c['scm-password'] = getpass('SCM Password: ')

    # Compile ignore regexes.
    c.setdefault('ignore', {})
    c['ignore'] = [re.compile(i) for i in c['ignore']]

    if not 'refs' in c:
        c['refs'] = ['.*']

    # Get the effective (accounting for inheritance) config for all refs.
    cfg = get_effective_branch_config(c['refs'], c['defaults'])
    c['refs'] = cfg

    return c

#-----------------------------------------------------------------------------
def get_effective_branch_config(branches, defaults):
    '''Compile ref/branch regexes and map to their configuration with
       inheritance factored in (think maven help:effective-pom).'''

    ec = OrderedDict()
    assert isinstance(branches, (list, tuple))

    for entry in branches:
        if isinstance(entry, dict):
            key, overrides = list(entry.items())[0]
            config = defaults.copy()
            config.update(overrides)
            ec[re.compile(key)] = config
        else:
            ec[re.compile(entry)] = defaults

    # The 'All' view doesn't have an API endpoint (i.e. no /view/All/api).
    # Since all jobs are added to it by default, it is safe to ignore it.
    for config in ec.values():
        if 'All' in config['view']:
            config['view'].remove('All')

    return ec

def get_ignored(branches, regexes):
    '''Get refs, excluding ignored.'''

    isignored = partial(anymatch, regexes)
    ignored, branches = filtersplit(isignored, branches)

    return ignored, branches

def resolveconfig(effective_config, branch):
    '''Resolve a ref to its effective config.'''

    for regex, config in effective_config['refs'].items():
        if regex.match(branch):
            config['re'] = regex
            return config.copy()

def get_job_etree(job):
    res = jenkins.job(job).config
    res = etree.fromstring(res.encode('utf8'))
    return res

def debug_refconfig(ref_config):
    print('. config:')
    for k,v in ref_config.items():
        if k == 're':
            print('  . %s: %s' % (k, v.pattern))
            continue
        if v: print('  . %s: %s' % (k, v))

def enable_http_logging():
    import logging

    try:
        from http.client import HTTPConnection
    except ImportError:
        from httplib import HTTPConnection

    HTTPConnection.debuglevel = 1
    logging.basicConfig()
    logging.getLogger().setLevel(logging.DEBUG)
    requests_log = logging.getLogger('requests.packages.urllib3')
    requests_log.setLevel(logging.DEBUG)
    requests_log.propagate = True
<commit_msg>Cleanup performance improvements and fixed `403 Client Error: Forbidden` during cleanup.<commit_after># -*- coding: utf-8; -*-

import re
import yaml

from os.path import basename, abspath
from sys import exit, argv
from copy import deepcopy
from getopt import getopt
from getpass import getpass
from functools import partial
from subprocess import CalledProcessError

from lxml import etree
from jenkins import Jenkins, JenkinsError
from requests.exceptions import RequestException, HTTPError

from jenkins_autojobs import __version__
from jenkins_autojobs.util import *

#-----------------------------------------------------------------------------
# Compatibility imports.
try:
    from itertools import ifilterfalse as filterfalse
except ImportError:
    from itertools import filterfalse

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

try:
    input = raw_input
except NameError:
    pass

#-----------------------------------------------------------------------------
usage = '''\
Usage: %s [-rvdtjnyoupUYOP] <config.yaml>

General Options:
  -n dry run
  -v show version and exit
  -d debug config inheritance
  -t debug http requests

Repository Options:
  -r <arg> repository url
  -y <arg> scm username
  -o <arg> scm password
  -Y scm username (read from stdin)
  -O scm password (read from stdin)

Jenkins Options:
  -j <arg> jenkins url
  -u <arg> jenkins username
  -p <arg> jenkins password
  -U jenkins username (read from stdin)
  -P jenkins password (read from stdin)\
''' % basename(argv[0])

#-----------------------------------------------------------------------------
# The *global* connection to jenkins - assigned in main().
jenkins = None

def main(argv, create_job, list_branches, getoptfmt='vdtnr:j:u:p:y:o:UPYO', config=None):
    '''
    :param argv: command-line arguments to parse (defaults to sys.argv[1:])
    :param create_job: scm specific function that configures and creates jobs
    :param list_branches: scm specific function that lists all branches/refs
    :param getoptfmt: getopt short and long options
    :param config: a config dictionary to use instead of parsing the
                   configuration from yaml (useful for testing)
    '''

    if '-h' in argv or '--help' in argv:
        print(usage)
        exit(1)

    opts, args = parse_args(argv, getoptfmt)
    if not args and not config:
        print(usage)
        exit(1)

    # Load config, set default values and compile regexes.
    if not config :
        yamlfn = args[-1]
        print('loading config from "%s"' % abspath(yamlfn))
        config = yaml.load(open(yamlfn))

    config = c = get_default_config(config, opts)

    if config['debughttp']:
        enable_http_logging()

    # Connect to jenkins.
    try:
        global jenkins
        jenkins = main.jenkins = Jenkins(c['jenkins'], c['username'], c['password'])
    except (RequestException, JenkinsError) as e:
        print(e); exit(1)

    #-------------------------------------------------------------------------
    # Get all the template names that the config references.
    templates = set(i['template'] for i in c['refs'].values())

    # Check if all referenced template jobs exist on the server.
    missing = list(filterfalse(jenkins.job_exists, templates))
    if missing:
        missing.insert(0, '\nconfig references non-existent template jobs:')
        print('\n - '.join(missing)); exit(1)

    # Convert them to etree objects of the templates' config xmls.
    templates = dict((i, get_job_etree(i)) for i in templates)

    #-------------------------------------------------------------------------
    # Check if all referenced views exist.
    view_names = set(view for i in c['refs'].values() for view in i['view'])
    missing = list(filterfalse(jenkins.view_exists, view_names))
    if missing:
        missing.insert(0, '\nconfig references non-existent views:')
        print('\n - '.join(missing)); exit(1)

    #-------------------------------------------------------------------------
    # List all git refs, svn branches etc (implemented by child classes).
    try:
        branches = list(list_branches(config))
    except CalledProcessError as e:
        print('! cannot list branches')
        print('! command %s failed' % ' '.join(e.cmd))
        exit(1)

    # See if any of the branches are ignored.
    ignored, branches = get_ignored(branches, c['ignore'])

    if ignored:
        msg = ['\nexplicitly ignored:'] + ignored
        print('\n - '.join(msg))

    # Get branch config for each branch.
    configs = map(partial(resolveconfig, config), branches)
    configs = zip(branches, configs)
    configs = filter(lambda x: bool(x[1]), configs)

    # The names of all successfully created or updated jobs.
    job_names = {}
    for branch, branch_config in configs:
        tmpl = templates[branch_config['template']]
        job_name = create_job(branch, tmpl, config, branch_config)
        job_names[job_name] = branch_config

        # Add newly create jobs to views, if any.
        views = branch_config['view']
        for view_name in views:
            view = jenkins.view(view_name)
            if job_name in view:
                print('. job already in view: %s' % view_name)
            else:
                if not config['dryrun']:
                    jenkins.view_add_job(view_name, job_name)
                print('. job added to view: %s' % view_name)

    if config['cleanup']:
        job_names[config['template']] = {}
        cleanup(config, job_names, jenkins)

#-----------------------------------------------------------------------------
def cleanup(config, job_names, jenkins, verbose=True):
    print('\ncleaning up old jobs:')

    tagxpath = 'createdByJenkinsAutojobs/tag/text()'
    removed_jobs = []

    for job, job_config in get_managed_jobs(job_names, jenkins):
        # If cleanup is a tag name, only cleanup builds with that tag.
        if isinstance(config['cleanup'], str):
            xml = etree.fromstring(job_config.encode('utf8'))
            clean_tag = xml.xpath(tagxpath)
            if not config['cleanup'] in clean_tag:
                continue

        removed_jobs.append(job)
        if not config['dryrun']:
            job_removed = safe_job_delete(job)
        else:
            job_removed = True

        if job_removed:
            print(' - %s' % job.name)
        else:
            print(' ! permission denied for %s' % job.name)

    if not removed_jobs:
        print('. nothing to do')

def get_managed_jobs(job_names, jenkins, safe_codes=(403,)):
    tag = '</createdByJenkinsAutojobs>'

    for job in jenkins.jobs:
        if job.name not in job_names:
            try:
                job_config = job.config
                if tag in job_config:
                    yield job, job_config
            except HTTPError as error:
                if error.response.status_code  not in safe_codes:
                    raise

def safe_job_delete(job, safe_codes=(403,)):
    try:
        job.delete()
        return True
    except HTTPError as error:
        if error.response.status_code not in safe_codes:
            raise
        return False

#-----------------------------------------------------------------------------
def parse_args(argv, fmt):
    '''Parse getopt arguments as a dictionary.'''
    opts, args = getopt(argv, fmt)
    opts = dict(opts)

    if '-v' in opts:
        print('jenkins-autojobs version %s' % __version__)
        exit(0)

    return opts, args

#-----------------------------------------------------------------------------
def get_default_config(config, opts):
    '''Set default config values and compile regexes.'''

    c, o = deepcopy(config), opts

    # Default global settings (not inheritable).
    c['dryrun'] = False
    c['debug']  = config.get('debug', False)
    c['debughttp'] = config.get('debughttp', False)
    c['cleanup']   = config.get('cleanup', False)
    c['username']  = config.get('username', None)
    c['password']  = config.get('password', None)
    c['scm-username'] = config.get('scm-username', None)
    c['scm-password'] = config.get('scm-password', None)

    # Default settings for each git ref/branch/ config.
    c['defaults'] = {
        'namesep':         c.get('namesep', '-'),
        'namefmt':         c.get('namefmt', '{shortref}'),
        'overwrite':       c.get('overwrite', True),
        'enable':          c.get('enable', 'sticky'),
        'substitute':      c.get('substitute', {}),
        'template':        c.get('template'),
        'sanitize':        c.get('sanitize', {'@!?#&|\^_$%*': '_'}),
        'tag':             c.get('tag', []),
        'view':            c.get('view', []),
        'build-on-create': c.get('build-on-create', False)
    }

    # Make sure some options are always lists.
    c['defaults']['view'] = pluralize(c['defaults']['view'])

    # Some options can be overwritten on the command line.
    if '-r' in o: c['repo'] = o['-r']
    if '-j' in o: c['jenkins'] = o['-j']
    if '-n' in o: c['dryrun'] = True
    if '-d' in o: c['debug'] = True
    if '-t' in o: c['debughttp'] = True

    # Jenkins authentication options.
    if '-u' in o: c['username'] = o['-u']
    if '-p' in o: c['password'] = o['-p']
    if '-U' in o: c['username'] = input('Jenkins User: ')
    if '-P' in o: c['password'] = getpass('Jenkins Password: ')

    # SCM authentication options.
    if '-y' in o: c['scm-username'] = o['-y']
    if '-o' in o: c['scm-password'] = o['-o']
    if '-Y' in o: c['scm-username'] = input('SCM User: ')
    if '-O' in o: c['scm-password'] = getpass('SCM Password: ')

    # Compile ignore regexes.
    c.setdefault('ignore', {})
    c['ignore'] = [re.compile(i) for i in c['ignore']]

    if not 'refs' in c:
        c['refs'] = ['.*']

    # Get the effective (accounting for inheritance) config for all refs.
    cfg = get_effective_branch_config(c['refs'], c['defaults'])
    c['refs'] = cfg

    return c

#-----------------------------------------------------------------------------
def get_effective_branch_config(branches, defaults):
    '''Compile ref/branch regexes and map to their configuration with
       inheritance factored in (think maven help:effective-pom).'''

    ec = OrderedDict()
    assert isinstance(branches, (list, tuple))

    for entry in branches:
        if isinstance(entry, dict):
            key, overrides = list(entry.items())[0]
            config = defaults.copy()
            config.update(overrides)
            ec[re.compile(key)] = config
        else:
            ec[re.compile(entry)] = defaults

    # The 'All' view doesn't have an API endpoint (i.e. no /view/All/api).
    # Since all jobs are added to it by default, it is safe to ignore it.
    for config in ec.values():
        if 'All' in config['view']:
            config['view'].remove('All')

    return ec

def get_ignored(branches, regexes):
    '''Get refs, excluding ignored.'''

    isignored = partial(anymatch, regexes)
    ignored, branches = filtersplit(isignored, branches)

    return ignored, branches

def resolveconfig(effective_config, branch):
    '''Resolve a ref to its effective config.'''

    for regex, config in effective_config['refs'].items():
        if regex.match(branch):
            config['re'] = regex
            return config.copy()

def get_job_etree(job):
    res = jenkins.job(job).config
    res = etree.fromstring(res.encode('utf8'))
    return res

def debug_refconfig(ref_config):
    print('. config:')
    for k,v in ref_config.items():
        if k == 're':
            print('  . %s: %s' % (k, v.pattern))
            continue
        if v: print('  . %s: %s' % (k, v))

def enable_http_logging():
    import logging

    try:
        from http.client import HTTPConnection
    except ImportError:
        from httplib import HTTPConnection

    HTTPConnection.debuglevel = 1
    logging.basicConfig()
    logging.getLogger().setLevel(logging.DEBUG)
    requests_log = logging.getLogger('requests.packages.urllib3')
    requests_log.setLevel(logging.DEBUG)
    requests_log.propagate = True
<|endoftext|>


<commit_before>import unittest

import responses

from seaworthy.checks import docker_client, dockertest
from seaworthy.client import ContainerClient
from seaworthy.containers.base import ContainerBase
from seaworthy.dockerhelper import DockerHelper, fetch_images

# Small (<4MB) image that echoes HTTP requests and runs without configuration
IMG = 'jmalloc/echo-server'

@dockertest()
def setUpModule():  # noqa: N802 (The camelCase is mandated by unittest.)
    with docker_client() as client:
        fetch_images(client, [IMG])

class DummySession:
    def __init__(self):
        self.requests = []
        self.was_closed = False

    def request(self, *args, **kwargs):
        self.requests.append((args, kwargs))

    def close(self):
        self.was_closed = True

    def check_was_closed(self):
        was_closed, self.was_closed = self.was_closed, False
        return was_closed

class TestContainerClient(unittest.TestCase):
    def make_helper(self):
        dh = DockerHelper()
        self.addCleanup(dh.teardown)
        dh.setup()
        return dh

    @responses.activate
    def test_defaults(self):
        """
        When the container client is configured with a host address and port,
        requests are made to that address and port.
        """
        client = ContainerClient('127.0.0.1', '12345')

        responses.add(responses.GET, 'http://127.0.0.1:12345/', status=200)
        response = client.request('GET')

        self.assertEqual(response.status_code, 200)

        self.assertEqual(len(responses.calls), 1)
        [call] = responses.calls
        self.assertEqual(call.request.url, 'http://127.0.0.1:12345/')

    @responses.activate
    def test_url_defaults(self):
        """
        When the container client is configured with a host address and port,
        and some URL defaults are set, requests are made to that address and
        port with the expected URL.
        """
        client = ContainerClient('127.0.0.1', '12345', url_defaults={
            'scheme': 'https',
            'fragment': 'test',
        })

        responses.add(responses.GET, 'https://127.0.0.1:12345/baz', status=200)
        response = client.request('GET', ['baz'], url_kwargs={
            'query': (('foo', 'bar'),),
        })

        self.assertEqual(response.status_code, 200)

        [call] = responses.calls
        self.assertEqual(
            call.request.url, 'https://127.0.0.1:12345/baz?foo=bar#test')

    @responses.activate
    def test_methods(self):
        """
        When the HTTP method-specific methods are called, the correct request
        method is used.
        """
        client = ContainerClient('127.0.0.1', '45678')

        responses.add(responses.GET, 'http://127.0.0.1:45678/', status=200)
        responses.add(
            responses.OPTIONS, 'http://127.0.0.1:45678/foo', status=201)
        responses.add(responses.HEAD, 'http://127.0.0.1:45678/bar', status=403)
        responses.add(responses.POST, 'http://127.0.0.1:45678/baz', status=404)
        responses.add(responses.PUT, 'http://127.0.0.1:45678/test', status=418)
        responses.add(
            responses.PATCH, 'http://127.0.0.1:45678/a/b/c', status=501)
        responses.add(
            responses.DELETE, 'http://127.0.0.1:45678/d/e/f', status=503)

        get_response = client.get()
        options_response = client.options(['foo'])
        head_response = client.head(['bar'])
        post_response = client.post(['baz'])
        put_response = client.put(['test'])
        patch_response = client.patch(['a', 'b', 'c'])
        delete_response = client.delete(['d', 'e', 'f'])

        self.assertEqual(get_response.status_code, 200)
        self.assertEqual(options_response.status_code, 201)
        self.assertEqual(head_response.status_code, 403)
        self.assertEqual(post_response.status_code, 404)
        self.assertEqual(put_response.status_code, 418)
        self.assertEqual(patch_response.status_code, 501)
        self.assertEqual(delete_response.status_code, 503)

        self.assertEqual(len(responses.calls), 7)

    def test_session(self):
        """
        When a custom session object is given, that object is used to make
        requests and is closed when ``close()`` is called.
        """
        session = DummySession()
        client = ContainerClient('127.0.0.1', '12345', session=session)

        client.request('GET', ['foo'])
        client.request('POST', ['bar'])
        self.assertEqual(session.requests, [
            (('GET', 'http://127.0.0.1:12345/foo'), {}),
            (('POST', 'http://127.0.0.1:12345/bar'), {}),
        ])

        client.close()
        self.assertTrue(session.check_was_closed())

    def test_session_context_manager(self):
        """
        When a custom session object is given, that object is used to make
        requests and is closed when the context is exited when the container
        client is used as a context manager.
        """
        session = DummySession()
        client = ContainerClient('127.0.0.1', '12345', session=session)

        with client:
            client.request('GET', ['foo'])
            self.assertEqual(session.requests, [
                (('GET', 'http://127.0.0.1:12345/foo'), {}),
            ])

        self.assertTrue(session.check_was_closed())

    @dockertest()
    def test_for_container_first_port(self):
        """
        The ``for_container()`` class method returns a container client that
        connects to the container's first port when a specific port is not
        specified.
        """
        dh = self.make_helper()
        container = ContainerBase('first_port', IMG, create_kwargs={
            'ports': {'8080/tcp': ('127.0.0.1', None)}
        })
        container.create_and_start(dh)
        self.addCleanup(container.stop_and_remove, dh)

        client = ContainerClient.for_container(container)

        response = client.request('GET', ['foo'])

        self.assertEqual(response.status_code, 200)
        response_lines = response.text.splitlines()
        self.assertIn('HTTP/1.1 GET /foo', response_lines)

        addr, port = container.get_first_host_port()
        self.assertIn('Host: {}:{}'.format(addr, port), response_lines)

    @dockertest()
    def test_for_container_specific_port(self):
        """
        The ``for_container()`` class method returns a container client that
        connects to the container port specified.
        """
        dh = self.make_helper()
        container = ContainerBase('first_port', IMG, create_kwargs={
            'ports': {
                '8080/tcp': ('127.0.0.1', None),
                '5353/udp': ('127.0.0.1', None),
            }
        })
        container.create_and_start(dh)
        self.addCleanup(container.stop_and_remove, dh)

        client = ContainerClient.for_container(
            container, container_port='8080')

        response = client.request('GET', ['foo'])

        self.assertEqual(response.status_code, 200)
        response_lines = response.text.splitlines()
        self.assertIn('HTTP/1.1 GET /foo', response_lines)

        addr, port = container.get_host_port('8080')
        self.assertIn('Host: {}:{}'.format(addr, port), response_lines)
<commit_msg>Remove extra len(responses.calls) == 1 check<commit_after>import unittest

import responses

from seaworthy.checks import docker_client, dockertest
from seaworthy.client import ContainerClient
from seaworthy.containers.base import ContainerBase
from seaworthy.dockerhelper import DockerHelper, fetch_images

# Small (<4MB) image that echoes HTTP requests and runs without configuration
IMG = 'jmalloc/echo-server'

@dockertest()
def setUpModule():  # noqa: N802 (The camelCase is mandated by unittest.)
    with docker_client() as client:
        fetch_images(client, [IMG])

class DummySession:
    def __init__(self):
        self.requests = []
        self.was_closed = False

    def request(self, *args, **kwargs):
        self.requests.append((args, kwargs))

    def close(self):
        self.was_closed = True

    def check_was_closed(self):
        was_closed, self.was_closed = self.was_closed, False
        return was_closed

class TestContainerClient(unittest.TestCase):
    def make_helper(self):
        dh = DockerHelper()
        self.addCleanup(dh.teardown)
        dh.setup()
        return dh

    @responses.activate
    def test_defaults(self):
        """
        When the container client is configured with a host address and port,
        requests are made to that address and port.
        """
        client = ContainerClient('127.0.0.1', '12345')

        responses.add(responses.GET, 'http://127.0.0.1:12345/', status=200)
        response = client.request('GET')

        self.assertEqual(response.status_code, 200)

        [call] = responses.calls
        self.assertEqual(call.request.url, 'http://127.0.0.1:12345/')

    @responses.activate
    def test_url_defaults(self):
        """
        When the container client is configured with a host address and port,
        and some URL defaults are set, requests are made to that address and
        port with the expected URL.
        """
        client = ContainerClient('127.0.0.1', '12345', url_defaults={
            'scheme': 'https',
            'fragment': 'test',
        })

        responses.add(responses.GET, 'https://127.0.0.1:12345/baz', status=200)
        response = client.request('GET', ['baz'], url_kwargs={
            'query': (('foo', 'bar'),),
        })

        self.assertEqual(response.status_code, 200)

        [call] = responses.calls
        self.assertEqual(
            call.request.url, 'https://127.0.0.1:12345/baz?foo=bar#test')

    @responses.activate
    def test_methods(self):
        """
        When the HTTP method-specific methods are called, the correct request
        method is used.
        """
        client = ContainerClient('127.0.0.1', '45678')

        responses.add(responses.GET, 'http://127.0.0.1:45678/', status=200)
        responses.add(
            responses.OPTIONS, 'http://127.0.0.1:45678/foo', status=201)
        responses.add(responses.HEAD, 'http://127.0.0.1:45678/bar', status=403)
        responses.add(responses.POST, 'http://127.0.0.1:45678/baz', status=404)
        responses.add(responses.PUT, 'http://127.0.0.1:45678/test', status=418)
        responses.add(
            responses.PATCH, 'http://127.0.0.1:45678/a/b/c', status=501)
        responses.add(
            responses.DELETE, 'http://127.0.0.1:45678/d/e/f', status=503)

        get_response = client.get()
        options_response = client.options(['foo'])
        head_response = client.head(['bar'])
        post_response = client.post(['baz'])
        put_response = client.put(['test'])
        patch_response = client.patch(['a', 'b', 'c'])
        delete_response = client.delete(['d', 'e', 'f'])

        self.assertEqual(get_response.status_code, 200)
        self.assertEqual(options_response.status_code, 201)
        self.assertEqual(head_response.status_code, 403)
        self.assertEqual(post_response.status_code, 404)
        self.assertEqual(put_response.status_code, 418)
        self.assertEqual(patch_response.status_code, 501)
        self.assertEqual(delete_response.status_code, 503)

        self.assertEqual(len(responses.calls), 7)

    def test_session(self):
        """
        When a custom session object is given, that object is used to make
        requests and is closed when ``close()`` is called.
        """
        session = DummySession()
        client = ContainerClient('127.0.0.1', '12345', session=session)

        client.request('GET', ['foo'])
        client.request('POST', ['bar'])
        self.assertEqual(session.requests, [
            (('GET', 'http://127.0.0.1:12345/foo'), {}),
            (('POST', 'http://127.0.0.1:12345/bar'), {}),
        ])

        client.close()
        self.assertTrue(session.check_was_closed())

    def test_session_context_manager(self):
        """
        When a custom session object is given, that object is used to make
        requests and is closed when the context is exited when the container
        client is used as a context manager.
        """
        session = DummySession()
        client = ContainerClient('127.0.0.1', '12345', session=session)

        with client:
            client.request('GET', ['foo'])
            self.assertEqual(session.requests, [
                (('GET', 'http://127.0.0.1:12345/foo'), {}),
            ])

        self.assertTrue(session.check_was_closed())

    @dockertest()
    def test_for_container_first_port(self):
        """
        The ``for_container()`` class method returns a container client that
        connects to the container's first port when a specific port is not
        specified.
        """
        dh = self.make_helper()
        container = ContainerBase('first_port', IMG, create_kwargs={
            'ports': {'8080/tcp': ('127.0.0.1', None)}
        })
        container.create_and_start(dh)
        self.addCleanup(container.stop_and_remove, dh)

        client = ContainerClient.for_container(container)

        response = client.request('GET', ['foo'])

        self.assertEqual(response.status_code, 200)
        response_lines = response.text.splitlines()
        self.assertIn('HTTP/1.1 GET /foo', response_lines)

        addr, port = container.get_first_host_port()
        self.assertIn('Host: {}:{}'.format(addr, port), response_lines)

    @dockertest()
    def test_for_container_specific_port(self):
        """
        The ``for_container()`` class method returns a container client that
        connects to the container port specified.
        """
        dh = self.make_helper()
        container = ContainerBase('first_port', IMG, create_kwargs={
            'ports': {
                '8080/tcp': ('127.0.0.1', None),
                '5353/udp': ('127.0.0.1', None),
            }
        })
        container.create_and_start(dh)
        self.addCleanup(container.stop_and_remove, dh)

        client = ContainerClient.for_container(
            container, container_port='8080')

        response = client.request('GET', ['foo'])

        self.assertEqual(response.status_code, 200)
        response_lines = response.text.splitlines()
        self.assertIn('HTTP/1.1 GET /foo', response_lines)

        addr, port = container.get_host_port('8080')
        self.assertIn('Host: {}:{}'.format(addr, port), response_lines)
<|endoftext|>