<commit_before># Copyright 2012 Kevin Goodsell
#
# This software is licensed under the Eclipse Public License (EPL) V1.0.

'''
STAF data type marshalling.
'''

import re

from ._errors import STAFError

class STAFUnmarshallError(STAFError):
    pass

class NamedConstant(object):
    '''
    Simple class for singleton constants.
    '''
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return '.'.join([self.__module__, self.name])

UNMARSHALL_RECURSIVE = NamedConstant('UNMARSHALL_RECURSIVE')
UNMARSHALL_NON_RECURSIVE = NamedConstant('UNMARSHALL_NON_RECURSIVE')
UNMARSHALL_NONE = NamedConstant('UNMARSHALL_NONE')

marker = '@SDT/'

def unmarshall(data, mode=UNMARSHALL_RECURSIVE):
    '''
    Try to unmarshall the string in 'data'. 'mode' determines how unmarshalling
    is done.

    'mode' can be one of the following:

        UNMARSHALL_RECURSIVE (default) - Tries to recursively unmarshall any
        string result.

        UNMARSHALL_NON_RECURSIVE - leaves string results as they are, doesn't
        attept further unmarshalling.

        UNMARSHALL_NONE - doesn't do any unmarshalling.

    Returns 'data' if it doesn't appear to be a marshalled object, or if 'mode'
    is UNMARSHALL_NONE.
    '''
    try:
        return unmarshall_force(data, mode)
    except STAFUnmarshallError:
        return data

def unmarshall_force(data, mode=UNMARSHALL_RECURSIVE):
    '''
    Same as unmarshall, but raises STAFUnmarshallError if unmarshalling isn't
    possible.
    '''
    if mode == UNMARSHALL_NONE:
        return data

    (obj, remainder) = unmarshall_internal(data, mode)

    if remainder != '':
        raise STAFUnmarshallError('unexpected trailing data')

    return obj

# Special classes for dealing with STAF Map Class types. Unmarshalled map
# classes will be instances of MapClass, which is derived from dict and can be
# used as a dict. It also provides display_name() and display_short_name() to
# access those attributes if desired.

class MapClassDefinition(object):
    '''
    Represents a map class definition, which provides the set of key names and
    long and short display names for the keys.
    '''

    def __init__(self, name):
        self.name = name
        self.keys = []
        self._names = {} # {'key' : ('display name', 'short display name')}

    def __repr__(self):
        cls = self.__class__
        return '<%s.%s %s>' % (cls.__module__, cls.__name__, self.name)

    def add_item(self, key, display_name, display_short_name=None):
        '''
        Add a key and a display name for it. Optionally include a short display
        name.
        '''
        self.keys.append(key)
        self._names[key] = (display_name, display_short_name)

    def display_name(self, key):
        '''
        Returns the display name for 'key'.
        '''
        return self._names[key][0]

    def display_short_name(self, key):
        '''
        Returns the short form of the display name for 'key', or None if no
        short name is defined.
        '''
        return self._names[key][1]

    def map_class(self):
        '''
        Return a MapClass instance based on this definition.
        '''
        return MapClass(self.name, self.keys, self._names)

def add_docstring(method):
    '''
    Add a docstring to method by pulling it from the corresponding method in
    dict. Only for use with dict-derived classes overriding dict methods.
    '''
    method.__doc__ = getattr(dict, method.__name__).__doc__

class MapClass(dict):
    '''
    Represents a map class instance. This is a dict and can be accessed the same
    way. It also has 'display_name' and 'display_short_name' methods for getting
    the display names for a key.

    Adding and removing keys are not supported.
    '''

    def __init__(self, class_name, keys, disp_names):
        '''
        Create a new MapClass. 'class_name' gives the Map Class name, 'keys'
        is a sequence giving the keys in the Map and their ordering, and
        disp_names is a dict mapping key names to (display_name,
        display_short_name) tuples, where display_short_name may be None.

        This should generally not be used directly. Instead, create a
        MapClassDefinition, populate it with keys, then use its map_class method
        to create MapClass instances.
        '''
        super(MapClass, self).__init__()

        self.class_name = class_name
        self._keys = list(keys)
        self._names = dict(disp_names)

        for key in self._keys:
            self[key] = None

    def display_name(self, key):
        '''
        Returns the display name for 'key'.
        '''
        return self._names[key][0]

    def display_short_name(self, key):
        '''
        Returns the short form of the display name for 'key', or None if no
        short name is defined.
        '''
        return self._names[key][1]

    def definition(self):
        '''
        Return a new MapClassDefinition using the structure of this MapClass.
        '''
        definition = MapClassDefinition(self.class_name)
        for key in self._keys:
            (disp_name, disp_short) = self._names[key]
            definition.add_item(key, disp_name, disp_short)

        return definition

    # Overrides to prevent adding/removing keys.

    def __setitem__(self, key, value):
        if key in self._names:
            super(MapClass, self).__setitem__(key, value)
        else:
            raise KeyError(key)

    def __delitem__(self, key):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('item deletion is not supported in MapClass')

    def clear(self):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('clear() is not supported in MapClass')

    def pop(self, key, default=None):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('pop() is not supported in MapClass')

    def popitem(self):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('popitem() is not supported in MapClass')

    def setdefault(key, default=None):
        if key in self:
            return super(MapClass, self).setdefault(key, default)
        else:
            raise KeyError(key)

    def update(self, *args, **kwargs):
        as_dict = dict(*args, **kwargs)
        for key in as_dict.iterkeys():
            if key not in self:
                raise KeyError(key)

        self.update(as_dict)

    # Overrides to preserve ordering on item access.

    def __iter__(self):
        return iter(self._keys)

    def iterkeys(self):
        return iter(self._keys)

    def keys(self):
        return list(self.iterkeys())

    def itervalues(self):
        for key in self._keys:
            yield self[key]

    def values(self):
        return list(self.itervalues())

    def iteritems(self):
        for key in self._keys:
            yield (key, self[key])

    def items(self):
        return list(self.iteritems())

    # The default dict repr would be fine, but the order of the fields is kind
    # of important for MapClass instances.
    def __repr__(self):
        items = []
        for (k, v) in self.iteritems():
            items.append('%r: %r' % (k, v))

        return '{%s}' % ', '.join(items)

    add_docstring(__setitem__)
    add_docstring(setdefault)
    add_docstring(update)
    add_docstring(__iter__)
    add_docstring(iterkeys)
    add_docstring(keys)
    add_docstring(itervalues)
    add_docstring(values)
    add_docstring(iteritems)
    add_docstring(items)
    add_docstring(__repr__)

def unmarshall_internal(data, mode, context=None):
    if context is None:
        context = {}

    if not data.startswith(marker):
        raise STAFUnmarshallError('missing marshalled data marker')

    sym_index = len(marker)
    try:
        symbol = data[sym_index]
        rest = data[sym_index+1:]
    except IndexError:
        raise STAFUnmarshallError('incomplete marshalled data')

    unmarshaller = get_unmarshaller(symbol)
    if unmarshaller is None:
        raise STAFUnmarshallError('unrecognized data type indicator')

    return unmarshaller.unmarshall(rest, mode, context)

def get_unmarshaller(symbol):
    if symbol == '$':
        return ScalarUnmarshaller
    elif symbol == '{':
        return MapUnmarshaller
    elif symbol == '[':
        return ListUnmarshaller
    elif symbol == '%':
        return MapClassUnmarshaller
    elif symbol == '*':
        return ContextUnmarshaller
    else:
        return None

class Unmarshaller(object):
    '''
    Base class for all unmarshallers.
    '''
    clc_matcher = re.compile(
        r'''^
        :(?P<len>\d+):  # Colon-length-colon
        (?P<rest>.*)    # Data
        $''', re.VERBOSE | re.DOTALL)

    @classmethod
    def unmarshall(cls, data, mode, context):
        raise NotImplementedError()

    @classmethod
    def read_clc_obj(cls, data):
        '''
        Read a colon-length-colon denoted object from data. That is, a colon,
        integer length, colon, and finally a sequence of characters of the given
        length. Returns a tuple of the object and everything that's left in
        data. Raises STAFUnmarshallError if the format is not as expected.
        '''
        m = cls.clc_matcher.match(data)
        if m is None:
            raise STAFUnmarshallError('bad format for colon-length-colon '
                                      'object')

        (length, rest) = m.group('len', 'rest')
        length = int(length)

        if length > len(rest):
            raise STAFUnmarshallError('specified length exceeds available data')

        obj = rest[:length]
        rest = rest[length:]

        return (obj, rest)

class ScalarUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        if not data.startswith('0') and not data.startswith('S'):
            raise STAFUnmarshallError('bad format for scalar object')

        typ = data[0]
        (obj, rest) = cls.read_clc_obj(data[1:])

        if typ == '0':
            if obj != '':
                raise STAFUnmarshallError('bad format for none object')
            return (None, rest)

        else: # typ == 'S'
            # Possibly do recursive unmarshalling.
            if mode == UNMARSHALL_RECURSIVE:
                obj = unmarshall(obj, mode)

            return (obj, rest)

class MapUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        (items, remainder) = cls.read_clc_obj(data)

        result = {}

        while items:
            (key, items) = cls.read_clc_obj(items)
            (val, items) = unmarshall_internal(items, mode, context)

            result[key] = val

        return (result, remainder)

class ListUnmarshaller(Unmarshaller):
    count_matcher = re.compile(r'''^
        (?P<count>\d+)  # Number of elements
        (?P<rest>.*)    # List items
        $''', re.VERBOSE | re.DOTALL)

    @classmethod
    def unmarshall(cls, data, mode, context):
        m = cls.count_matcher.match(data)
        if m is None:
            raise STAFUnmarshallError('bad format for list object')

        (count, rest) = m.group('count', 'rest')
        count = int(count)

        (items, remainder) = cls.read_clc_obj(rest)

        result = []
        for i in range(count):
            (obj, items) = unmarshall_internal(items, mode, context)
            result.append(obj)

        if items != '':
            raise STAFUnmarshallError('unexpected trailing data')

        return (result, remainder)

class MapClassUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        (content, remainder) = cls.read_clc_obj(data)
        (class_name, values) = cls.read_clc_obj(content)

        class_def = context.get(class_name)
        if class_def is None:
            raise STAFUnmarshallError('missing map class definition for %r' %
                                      class_name)

        result = class_def.map_class()
        # The ordering and names of the keys are given by class_def.keys.
        for key in class_def.keys:
            (value, values) = unmarshall_internal(values, mode, context)
            result[key] = value

        if values != '':
            raise STAFUnmarshallError('unexpected trailing data')

        return (result, remainder)

class ContextUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        (content, remainder) = cls.read_clc_obj(data)
        (context_map, root_data) = unmarshall_internal(content, mode, context)

        class_map = context_map.get('map-class-map', {})

        # Build a map of names to MapClassDefinitions.
        new_context = {}
        for (name, info) in class_map.iteritems():
            class_def = MapClassDefinition(name)
            for item in info['keys']:
                class_def.add_item(item['key'], item['display-name'],
                                   item.get('display-short-name'))

            new_context[name] = class_def

        # Note that we may be forsaking an existing class map in 'context' in
        # favor of new_context. Nested contexts probably shouldn't happen, but
        # if they do this means the objects in the inner context won't be able
        # to reference objects in the outer context. This is probably fine.
        (root_obj, trailing) = unmarshall_internal(root_data, mode, new_context)

        if trailing != '':
            raise STAFUnmarshallError('unexpected trailing data')

        return (root_obj, remainder)
<commit_msg>Don't inherit dict docstrings for these methods.<commit_after># Copyright 2012 Kevin Goodsell
#
# This software is licensed under the Eclipse Public License (EPL) V1.0.

'''
STAF data type marshalling.
'''

import re

from ._errors import STAFError

class STAFUnmarshallError(STAFError):
    pass

class NamedConstant(object):
    '''
    Simple class for singleton constants.
    '''
    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return '.'.join([self.__module__, self.name])

UNMARSHALL_RECURSIVE = NamedConstant('UNMARSHALL_RECURSIVE')
UNMARSHALL_NON_RECURSIVE = NamedConstant('UNMARSHALL_NON_RECURSIVE')
UNMARSHALL_NONE = NamedConstant('UNMARSHALL_NONE')

marker = '@SDT/'

def unmarshall(data, mode=UNMARSHALL_RECURSIVE):
    '''
    Try to unmarshall the string in 'data'. 'mode' determines how unmarshalling
    is done.

    'mode' can be one of the following:

        UNMARSHALL_RECURSIVE (default) - Tries to recursively unmarshall any
        string result.

        UNMARSHALL_NON_RECURSIVE - leaves string results as they are, doesn't
        attept further unmarshalling.

        UNMARSHALL_NONE - doesn't do any unmarshalling.

    Returns 'data' if it doesn't appear to be a marshalled object, or if 'mode'
    is UNMARSHALL_NONE.
    '''
    try:
        return unmarshall_force(data, mode)
    except STAFUnmarshallError:
        return data

def unmarshall_force(data, mode=UNMARSHALL_RECURSIVE):
    '''
    Same as unmarshall, but raises STAFUnmarshallError if unmarshalling isn't
    possible.
    '''
    if mode == UNMARSHALL_NONE:
        return data

    (obj, remainder) = unmarshall_internal(data, mode)

    if remainder != '':
        raise STAFUnmarshallError('unexpected trailing data')

    return obj

# Special classes for dealing with STAF Map Class types. Unmarshalled map
# classes will be instances of MapClass, which is derived from dict and can be
# used as a dict. It also provides display_name() and display_short_name() to
# access those attributes if desired.

class MapClassDefinition(object):
    '''
    Represents a map class definition, which provides the set of key names and
    long and short display names for the keys.
    '''

    def __init__(self, name):
        self.name = name
        self.keys = []
        self._names = {} # {'key' : ('display name', 'short display name')}

    def __repr__(self):
        cls = self.__class__
        return '<%s.%s %s>' % (cls.__module__, cls.__name__, self.name)

    def add_item(self, key, display_name, display_short_name=None):
        '''
        Add a key and a display name for it. Optionally include a short display
        name.
        '''
        self.keys.append(key)
        self._names[key] = (display_name, display_short_name)

    def display_name(self, key):
        '''
        Returns the display name for 'key'.
        '''
        return self._names[key][0]

    def display_short_name(self, key):
        '''
        Returns the short form of the display name for 'key', or None if no
        short name is defined.
        '''
        return self._names[key][1]

    def map_class(self):
        '''
        Return a MapClass instance based on this definition.
        '''
        return MapClass(self.name, self.keys, self._names)

def add_docstring(method):
    '''
    Add a docstring to method by pulling it from the corresponding method in
    dict. Only for use with dict-derived classes overriding dict methods.
    '''
    method.__doc__ = getattr(dict, method.__name__).__doc__

class MapClass(dict):
    '''
    Represents a map class instance. This is a dict and can be accessed the same
    way. It also has 'display_name' and 'display_short_name' methods for getting
    the display names for a key.

    Adding and removing keys are not supported.
    '''

    def __init__(self, class_name, keys, disp_names):
        '''
        Create a new MapClass. 'class_name' gives the Map Class name, 'keys'
        is a sequence giving the keys in the Map and their ordering, and
        disp_names is a dict mapping key names to (display_name,
        display_short_name) tuples, where display_short_name may be None.

        This should generally not be used directly. Instead, create a
        MapClassDefinition, populate it with keys, then use its map_class method
        to create MapClass instances.
        '''
        super(MapClass, self).__init__()

        self.class_name = class_name
        self._keys = list(keys)
        self._names = dict(disp_names)

        for key in self._keys:
            self[key] = None

    def display_name(self, key):
        '''
        Returns the display name for 'key'.
        '''
        return self._names[key][0]

    def display_short_name(self, key):
        '''
        Returns the short form of the display name for 'key', or None if no
        short name is defined.
        '''
        return self._names[key][1]

    def definition(self):
        '''
        Return a new MapClassDefinition using the structure of this MapClass.
        '''
        definition = MapClassDefinition(self.class_name)
        for key in self._keys:
            (disp_name, disp_short) = self._names[key]
            definition.add_item(key, disp_name, disp_short)

        return definition

    # Overrides to prevent adding/removing keys.

    def __setitem__(self, key, value):
        '''
        Implements self[key] = value, but raises KeyError if key is not in the
        Map Class Definition.
        '''
        if key in self._names:
            super(MapClass, self).__setitem__(key, value)
        else:
            raise KeyError(key)

    def __delitem__(self, key):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('item deletion is not supported in MapClass')

    def clear(self):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('clear() is not supported in MapClass')

    def pop(self, key, default=None):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('pop() is not supported in MapClass')

    def popitem(self):
        'Not supported, raises NotImplementedError.'
        raise NotImplementedError('popitem() is not supported in MapClass')

    def setdefault(key, default=None):
        '''
        Modified dict.setdefault, raises KeyError if the key is not in the Map
        Class Definition.
        '''
        if key in self:
            return super(MapClass, self).setdefault(key, default)
        else:
            raise KeyError(key)

    def update(self, *args, **kwargs):
        '''
        Modified dict.update, raises KeyError if the any key is not in the Map
        Class Definition.
        '''
        as_dict = dict(*args, **kwargs)
        for key in as_dict.iterkeys():
            if key not in self:
                raise KeyError(key)

        self.update(as_dict)

    # Overrides to preserve ordering on item access.

    def __iter__(self):
        return iter(self._keys)

    def iterkeys(self):
        return iter(self._keys)

    def keys(self):
        return list(self.iterkeys())

    def itervalues(self):
        for key in self._keys:
            yield self[key]

    def values(self):
        return list(self.itervalues())

    def iteritems(self):
        for key in self._keys:
            yield (key, self[key])

    def items(self):
        return list(self.iteritems())

    # The default dict repr would be fine, but the order of the fields is kind
    # of important for MapClass instances.
    def __repr__(self):
        items = []
        for (k, v) in self.iteritems():
            items.append('%r: %r' % (k, v))

        return '{%s}' % ', '.join(items)

    add_docstring(__iter__)
    add_docstring(iterkeys)
    add_docstring(keys)
    add_docstring(itervalues)
    add_docstring(values)
    add_docstring(iteritems)
    add_docstring(items)
    add_docstring(__repr__)

def unmarshall_internal(data, mode, context=None):
    if context is None:
        context = {}

    if not data.startswith(marker):
        raise STAFUnmarshallError('missing marshalled data marker')

    sym_index = len(marker)
    try:
        symbol = data[sym_index]
        rest = data[sym_index+1:]
    except IndexError:
        raise STAFUnmarshallError('incomplete marshalled data')

    unmarshaller = get_unmarshaller(symbol)
    if unmarshaller is None:
        raise STAFUnmarshallError('unrecognized data type indicator')

    return unmarshaller.unmarshall(rest, mode, context)

def get_unmarshaller(symbol):
    if symbol == '$':
        return ScalarUnmarshaller
    elif symbol == '{':
        return MapUnmarshaller
    elif symbol == '[':
        return ListUnmarshaller
    elif symbol == '%':
        return MapClassUnmarshaller
    elif symbol == '*':
        return ContextUnmarshaller
    else:
        return None

class Unmarshaller(object):
    '''
    Base class for all unmarshallers.
    '''
    clc_matcher = re.compile(
        r'''^
        :(?P<len>\d+):  # Colon-length-colon
        (?P<rest>.*)    # Data
        $''', re.VERBOSE | re.DOTALL)

    @classmethod
    def unmarshall(cls, data, mode, context):
        raise NotImplementedError()

    @classmethod
    def read_clc_obj(cls, data):
        '''
        Read a colon-length-colon denoted object from data. That is, a colon,
        integer length, colon, and finally a sequence of characters of the given
        length. Returns a tuple of the object and everything that's left in
        data. Raises STAFUnmarshallError if the format is not as expected.
        '''
        m = cls.clc_matcher.match(data)
        if m is None:
            raise STAFUnmarshallError('bad format for colon-length-colon '
                                      'object')

        (length, rest) = m.group('len', 'rest')
        length = int(length)

        if length > len(rest):
            raise STAFUnmarshallError('specified length exceeds available data')

        obj = rest[:length]
        rest = rest[length:]

        return (obj, rest)

class ScalarUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        if not data.startswith('0') and not data.startswith('S'):
            raise STAFUnmarshallError('bad format for scalar object')

        typ = data[0]
        (obj, rest) = cls.read_clc_obj(data[1:])

        if typ == '0':
            if obj != '':
                raise STAFUnmarshallError('bad format for none object')
            return (None, rest)

        else: # typ == 'S'
            # Possibly do recursive unmarshalling.
            if mode == UNMARSHALL_RECURSIVE:
                obj = unmarshall(obj, mode)

            return (obj, rest)

class MapUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        (items, remainder) = cls.read_clc_obj(data)

        result = {}

        while items:
            (key, items) = cls.read_clc_obj(items)
            (val, items) = unmarshall_internal(items, mode, context)

            result[key] = val

        return (result, remainder)

class ListUnmarshaller(Unmarshaller):
    count_matcher = re.compile(r'''^
        (?P<count>\d+)  # Number of elements
        (?P<rest>.*)    # List items
        $''', re.VERBOSE | re.DOTALL)

    @classmethod
    def unmarshall(cls, data, mode, context):
        m = cls.count_matcher.match(data)
        if m is None:
            raise STAFUnmarshallError('bad format for list object')

        (count, rest) = m.group('count', 'rest')
        count = int(count)

        (items, remainder) = cls.read_clc_obj(rest)

        result = []
        for i in range(count):
            (obj, items) = unmarshall_internal(items, mode, context)
            result.append(obj)

        if items != '':
            raise STAFUnmarshallError('unexpected trailing data')

        return (result, remainder)

class MapClassUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        (content, remainder) = cls.read_clc_obj(data)
        (class_name, values) = cls.read_clc_obj(content)

        class_def = context.get(class_name)
        if class_def is None:
            raise STAFUnmarshallError('missing map class definition for %r' %
                                      class_name)

        result = class_def.map_class()
        # The ordering and names of the keys are given by class_def.keys.
        for key in class_def.keys:
            (value, values) = unmarshall_internal(values, mode, context)
            result[key] = value

        if values != '':
            raise STAFUnmarshallError('unexpected trailing data')

        return (result, remainder)

class ContextUnmarshaller(Unmarshaller):
    @classmethod
    def unmarshall(cls, data, mode, context):
        (content, remainder) = cls.read_clc_obj(data)
        (context_map, root_data) = unmarshall_internal(content, mode, context)

        class_map = context_map.get('map-class-map', {})

        # Build a map of names to MapClassDefinitions.
        new_context = {}
        for (name, info) in class_map.iteritems():
            class_def = MapClassDefinition(name)
            for item in info['keys']:
                class_def.add_item(item['key'], item['display-name'],
                                   item.get('display-short-name'))

            new_context[name] = class_def

        # Note that we may be forsaking an existing class map in 'context' in
        # favor of new_context. Nested contexts probably shouldn't happen, but
        # if they do this means the objects in the inner context won't be able
        # to reference objects in the outer context. This is probably fine.
        (root_obj, trailing) = unmarshall_internal(root_data, mode, new_context)

        if trailing != '':
            raise STAFUnmarshallError('unexpected trailing data')

        return (root_obj, remainder)
<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import sys
import yaml
import psutil
import subprocess
import tempfile
import logging
import getpass

from teuthology import beanstalk
from teuthology import report
from teuthology.config import config
from teuthology import misc

log = logging.getLogger(__name__)

def main(args):
    run_name = args['--run']
    job = args['--job']
    jobspec = args['--jobspec']
    archive_base = args['--archive']
    owner = args['--owner']
    machine_type = args['--machine-type']
    preserve_queue = args['--preserve-queue']

    if jobspec:
        split_spec = jobspec.split('/')
        run_name = split_spec[0]
        job = [split_spec[1]]

    if job:
        for job_id in job:
            kill_job(run_name, job_id, archive_base, owner)
    else:
        kill_run(run_name, archive_base, owner, machine_type,
                 preserve_queue=preserve_queue)

def kill_run(run_name, archive_base=None, owner=None, machine_type=None,
             preserve_queue=False):
    run_info = {}
    serializer = report.ResultsSerializer(archive_base)
    if archive_base:
        run_archive_dir = os.path.join(archive_base, run_name)
        if os.path.isdir(run_archive_dir):
            run_info = find_run_info(serializer, run_name)
            machine_type = run_info['machine_type']
            owner = run_info['owner']
        elif machine_type is None:
            raise RuntimeError("The run is still entirely enqueued; " +
                               "you must also pass --machine-type")

    if not preserve_queue:
        remove_beanstalk_jobs(run_name, machine_type)
        remove_paddles_jobs(run_name)
    kill_processes(run_name, run_info.get('pids'))
    if owner is not None:
        targets = find_targets(run_name, owner)
        nuke_targets(targets, owner)

def kill_job(run_name, job_id, archive_base=None, owner=None):
    serializer = report.ResultsSerializer(archive_base)
    job_info = serializer.job_info(run_name, job_id)
    if not owner:
        if 'owner' not in job_info:
            raise RuntimeError(
                "I could not figure out the owner of the requested job. "
                "Please pass --owner <owner>.")
        owner = job_info['owner']
    kill_processes(run_name, [job_info.get('pid')])
    # Because targets can be missing for some cases, for example, when all
    # the necessary nodes ain't locked yet, we do not use job_info to get them,
    # but use find_targets():
    targets = find_targets(run_name, owner, job_id)
    nuke_targets(targets, owner)

def find_run_info(serializer, run_name):
    log.info("Assembling run information...")
    run_info_fields = [
        'machine_type',
        'owner',
    ]

    pids = []
    run_info = {}
    job_info = {}
    job_num = 0
    jobs = serializer.jobs_for_run(run_name)
    job_total = len(jobs)
    for (job_id, job_dir) in jobs.items():
        if not os.path.isdir(job_dir):
            continue
        job_num += 1
        beanstalk.print_progress(job_num, job_total, 'Reading Job: ')
        job_info = serializer.job_info(run_name, job_id, simple=True)
        for key in job_info.keys():
            if key in run_info_fields and key not in run_info:
                run_info[key] = job_info[key]
        if 'pid' in job_info:
            pids.append(job_info['pid'])
    run_info['pids'] = pids
    return run_info

def remove_paddles_jobs(run_name):
    jobs = report.ResultsReporter().get_jobs(run_name, fields=['status'])
    job_ids = [job['job_id'] for job in jobs if job['status'] == 'queued']
    if job_ids:
        log.info("Deleting jobs from paddles: %s", str(job_ids))
        report.try_delete_jobs(run_name, job_ids)

def remove_beanstalk_jobs(run_name, tube_name):
    qhost = config.queue_host
    qport = config.queue_port
    if qhost is None or qport is None:
        raise RuntimeError(
            'Beanstalk queue information not found in {conf_path}'.format(
                conf_path=config.yaml_path))
    log.info("Checking Beanstalk Queue...")
    beanstalk_conn = beanstalk.connect()
    real_tube_name = beanstalk.watch_tube(beanstalk_conn, tube_name)

    curjobs = beanstalk_conn.stats_tube(real_tube_name)['current-jobs-ready']
    if curjobs != 0:
        x = 1
        while x != curjobs:
            x += 1
            job = beanstalk_conn.reserve(timeout=20)
            if job is None:
                continue
            job_config = yaml.safe_load(job.body)
            if run_name == job_config['name']:
                job_id = job.stats()['id']
                msg = "Deleting job from queue. ID: " + \
                    "{id} Name: {name} Desc: {desc}".format(
                        id=str(job_id),
                        name=job_config['name'],
                        desc=job_config['description'],
                    )
                log.info(msg)
                job.delete()
    else:
        print("No jobs in Beanstalk Queue")
    beanstalk_conn.close()

def kill_processes(run_name, pids=None):
    if pids:
        to_kill = set(pids).intersection(psutil.pids())
    else:
        to_kill = find_pids(run_name)

    # Remove processes that don't match run-name from the set
    to_check = set(to_kill)
    for pid in to_check:
        if not process_matches_run(pid, run_name):
            to_kill.remove(pid)

    if len(to_kill) == 0:
        log.info("No teuthology processes running")
    else:
        log.info("Killing Pids: " + str(to_kill))
        for pid in to_kill:
            args = ['kill', str(pid)]
            # Don't attempt to use sudo if it's not necessary
            proc_user = psutil.Process(int(pid)).username()
            if proc_user != getpass.getuser():
                args.insert(0, 'sudo')
            subprocess.call(args)

def process_matches_run(pid, run_name):
    try:
        p = psutil.Process(pid)
        cmd = p.cmdline()
        if run_name in cmd and sys.argv[0] not in cmd:
            return True
    except psutil.NoSuchProcess:
        pass
    return False

def find_pids(run_name):
    run_pids = []
    for pid in psutil.pids():
        if process_matches_run(pid, run_name):
            run_pids.append(pid)
    return run_pids

def find_targets(run_name, owner, job_id=None):
    lock_args = [
        'teuthology-lock',
        '--list-targets',
        '--desc-pattern',
        '/' + run_name + '/' + str(job_id or ''),
        '--status',
        'up',
        '--owner',
        owner
    ]
    proc = subprocess.Popen(lock_args, stdout=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    out_obj = yaml.safe_load(stdout)
    if not out_obj or 'targets' not in out_obj:
        return {}

    return out_obj

def nuke_targets(targets_dict, owner):
    targets = targets_dict.get('targets')
    if not targets:
        log.info("No locked machines. Not nuking anything")
        return

    to_nuke = []
    for target in targets:
        to_nuke.append(misc.decanonicalize_hostname(target))

    target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t')
    target_file.write(yaml.safe_dump(targets_dict))
    target_file.close()

    log.info("Nuking machines: " + str(to_nuke))
    nuke_args = [
        'teuthology-nuke',
        '-t',
        target_file.name,
        '--unlock',
        '-r',
        '--owner',
        owner
    ]
    proc = subprocess.Popen(
        nuke_args,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    for line in proc.stdout:
        line = line.replace(b'\r', b'').replace(b'\n', b'')
        log.info(line.decode())
        sys.stdout.flush()

    os.unlink(target_file.name)
<commit_msg>kill.py: Allow deleting runs where bootstrap is failing<commit_after>#!/usr/bin/python
import os
import sys
import yaml
import psutil
import subprocess
import tempfile
import logging
import getpass

from teuthology import beanstalk
from teuthology import report
from teuthology.config import config
from teuthology import misc

log = logging.getLogger(__name__)

def main(args):
    run_name = args['--run']
    job = args['--job']
    jobspec = args['--jobspec']
    archive_base = args['--archive']
    owner = args['--owner']
    machine_type = args['--machine-type']
    preserve_queue = args['--preserve-queue']

    if jobspec:
        split_spec = jobspec.split('/')
        run_name = split_spec[0]
        job = [split_spec[1]]

    if job:
        for job_id in job:
            kill_job(run_name, job_id, archive_base, owner)
    else:
        kill_run(run_name, archive_base, owner, machine_type,
                 preserve_queue=preserve_queue)

def kill_run(run_name, archive_base=None, owner=None, machine_type=None,
             preserve_queue=False):
    run_info = {}
    serializer = report.ResultsSerializer(archive_base)
    if archive_base:
        run_archive_dir = os.path.join(archive_base, run_name)
        if os.path.isdir(run_archive_dir):
            run_info = find_run_info(serializer, run_name)
            if 'machine_type' in run_info:
                machine_type = run_info['machine_type']
                owner = run_info['owner']
            else:
                log.warn("The run info does not have machine type: %s" % run_info)
                log.warn("Run archive used: %s" % run_archive_dir)
                log.info("Using machine type '%s' and owner '%s'" % (machine_type, owner))
        elif machine_type is None:
            raise RuntimeError("The run is still entirely enqueued; " +
                               "you must also pass --machine-type")

    if not preserve_queue:
        remove_beanstalk_jobs(run_name, machine_type)
        remove_paddles_jobs(run_name)
    kill_processes(run_name, run_info.get('pids'))
    if owner is not None:
        targets = find_targets(run_name, owner)
        nuke_targets(targets, owner)

def kill_job(run_name, job_id, archive_base=None, owner=None):
    serializer = report.ResultsSerializer(archive_base)
    job_info = serializer.job_info(run_name, job_id)
    if not owner:
        if 'owner' not in job_info:
            raise RuntimeError(
                "I could not figure out the owner of the requested job. "
                "Please pass --owner <owner>.")
        owner = job_info['owner']
    kill_processes(run_name, [job_info.get('pid')])
    # Because targets can be missing for some cases, for example, when all
    # the necessary nodes ain't locked yet, we do not use job_info to get them,
    # but use find_targets():
    targets = find_targets(run_name, owner, job_id)
    nuke_targets(targets, owner)

def find_run_info(serializer, run_name):
    log.info("Assembling run information...")
    run_info_fields = [
        'machine_type',
        'owner',
    ]

    pids = []
    run_info = {}
    job_info = {}
    job_num = 0
    jobs = serializer.jobs_for_run(run_name)
    job_total = len(jobs)
    for (job_id, job_dir) in jobs.items():
        if not os.path.isdir(job_dir):
            continue
        job_num += 1
        beanstalk.print_progress(job_num, job_total, 'Reading Job: ')
        job_info = serializer.job_info(run_name, job_id, simple=True)
        for key in job_info.keys():
            if key in run_info_fields and key not in run_info:
                run_info[key] = job_info[key]
        if 'pid' in job_info:
            pids.append(job_info['pid'])
    run_info['pids'] = pids
    return run_info

def remove_paddles_jobs(run_name):
    jobs = report.ResultsReporter().get_jobs(run_name, fields=['status'])
    job_ids = [job['job_id'] for job in jobs if job['status'] == 'queued']
    if job_ids:
        log.info("Deleting jobs from paddles: %s", str(job_ids))
        report.try_delete_jobs(run_name, job_ids)

def remove_beanstalk_jobs(run_name, tube_name):
    qhost = config.queue_host
    qport = config.queue_port
    if qhost is None or qport is None:
        raise RuntimeError(
            'Beanstalk queue information not found in {conf_path}'.format(
                conf_path=config.yaml_path))
    log.info("Checking Beanstalk Queue...")
    beanstalk_conn = beanstalk.connect()
    real_tube_name = beanstalk.watch_tube(beanstalk_conn, tube_name)

    curjobs = beanstalk_conn.stats_tube(real_tube_name)['current-jobs-ready']
    if curjobs != 0:
        x = 1
        while x != curjobs:
            x += 1
            job = beanstalk_conn.reserve(timeout=20)
            if job is None:
                continue
            job_config = yaml.safe_load(job.body)
            if run_name == job_config['name']:
                job_id = job.stats()['id']
                msg = "Deleting job from queue. ID: " + \
                    "{id} Name: {name} Desc: {desc}".format(
                        id=str(job_id),
                        name=job_config['name'],
                        desc=job_config['description'],
                    )
                log.info(msg)
                job.delete()
    else:
        print("No jobs in Beanstalk Queue")
    beanstalk_conn.close()

def kill_processes(run_name, pids=None):
    if pids:
        to_kill = set(pids).intersection(psutil.pids())
    else:
        to_kill = find_pids(run_name)

    # Remove processes that don't match run-name from the set
    to_check = set(to_kill)
    for pid in to_check:
        if not process_matches_run(pid, run_name):
            to_kill.remove(pid)

    if len(to_kill) == 0:
        log.info("No teuthology processes running")
    else:
        log.info("Killing Pids: " + str(to_kill))
        for pid in to_kill:
            args = ['kill', str(pid)]
            # Don't attempt to use sudo if it's not necessary
            proc_user = psutil.Process(int(pid)).username()
            if proc_user != getpass.getuser():
                args.insert(0, 'sudo')
            subprocess.call(args)

def process_matches_run(pid, run_name):
    try:
        p = psutil.Process(pid)
        cmd = p.cmdline()
        if run_name in cmd and sys.argv[0] not in cmd:
            return True
    except psutil.NoSuchProcess:
        pass
    return False

def find_pids(run_name):
    run_pids = []
    for pid in psutil.pids():
        if process_matches_run(pid, run_name):
            run_pids.append(pid)
    return run_pids

def find_targets(run_name, owner, job_id=None):
    lock_args = [
        'teuthology-lock',
        '--list-targets',
        '--desc-pattern',
        '/' + run_name + '/' + str(job_id or ''),
        '--status',
        'up',
        '--owner',
        owner
    ]
    proc = subprocess.Popen(lock_args, stdout=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    out_obj = yaml.safe_load(stdout)
    if not out_obj or 'targets' not in out_obj:
        return {}

    return out_obj

def nuke_targets(targets_dict, owner):
    targets = targets_dict.get('targets')
    if not targets:
        log.info("No locked machines. Not nuking anything")
        return

    to_nuke = []
    for target in targets:
        to_nuke.append(misc.decanonicalize_hostname(target))

    target_file = tempfile.NamedTemporaryFile(delete=False, mode='w+t')
    target_file.write(yaml.safe_dump(targets_dict))
    target_file.close()

    log.info("Nuking machines: " + str(to_nuke))
    nuke_args = [
        'teuthology-nuke',
        '-t',
        target_file.name,
        '--unlock',
        '-r',
        '--owner',
        owner
    ]
    proc = subprocess.Popen(
        nuke_args,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    for line in proc.stdout:
        line = line.replace(b'\r', b'').replace(b'\n', b'')
        log.info(line.decode())
        sys.stdout.flush()

    os.unlink(target_file.name)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gets a Chromium archived build, and unpacks it
   into a target directory.

  Use -r option to specify the revison number
  Use -t option to specify the directory to unzip the build into.

Usage:
  $ get_chromium_build.py -r <revision> -t <target>
"""

import logging
import optparse
import os
import platform
import shutil
import subprocess
import sys
import time
import urllib
import urllib2
import zipfile

# Example chromium build location:
# gs://chromium-browser-snapshots/Linux_x64/228977/chrome-linux.zip
CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
                    'chromium-browser-snapshots/%s/%s/%s')

class BuildUpdater(object):
  _PLATFORM_PATHS_MAP = {
      'Linux': { 'zipfiles': ['chrome-linux.zip'],
                 'folder': 'chrome_linux',
                 'archive_path': 'Linux_x64'},
      'Darwin': {'zipfiles': ['chrome-mac.zip'],
                 'folder': 'chrome_mac',
                 'archive_path': 'Mac'},
      'Windows': {'zipfiles': ['chrome-win32.zip',
                               'chrome-win32-syms.zip'],
                 'folder': 'chrome_win',
                 'archive_path': 'Win'}}

  def __init__(self, options):
    platform_data = BuildUpdater._PLATFORM_PATHS_MAP[platform.system()]
    self._zipfiles = platform_data['zipfiles']
    self._folder = platform_data['folder']
    self._archive_path = platform_data['archive_path']
    self._revision = int(options.revision)
    self._target_dir = options.target_dir
    self._download_dir = os.path.join(self._target_dir, 'downloads')

  def _GetBuildUrl(self, revision, filename):
    return CHROMIUM_URL_FMT % (self._archive_path, revision, filename)

  def _FindBuildRevision(self, revision, filename):
    MAX_REVISIONS_PER_BUILD = 100
    for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
      if self._DoesBuildExist(revision_guess, filename):
        return revision_guess
      else:
        time.sleep(.1)
    return None

  def _DoesBuildExist(self, revision_guess, filename):
    url = self._GetBuildUrl(revision_guess, filename)

    r = urllib2.Request(url)
    r.get_method = lambda: 'HEAD'
    try:
      urllib2.urlopen(r)
      return True
    except urllib2.HTTPError, err:
      if err.code == 404:
        return False

  def _DownloadBuild(self):
    if not os.path.exists(self._download_dir):
      os.makedirs(self._download_dir)
    for zipfile in self._zipfiles:
      build_revision = self._FindBuildRevision(self._revision, zipfile)
      if not build_revision:
        logging.critical('Failed to find %s build for r%s\n',
                         self._archive_path,
                         self._revision)
        sys.exit(1)
      url = self._GetBuildUrl(build_revision, zipfile)
      logging.info('Downloading %s', url)
      r = urllib2.urlopen(url)
      with file(os.path.join(self._download_dir, zipfile), 'wb') as f:
        f.write(r.read())

  def _UnzipFile(self, dl_file, dest_dir):
    if not zipfile.is_zipfile(dl_file):
      return False
    logging.info('Unzipping %s', dl_file)
    with zipfile.ZipFile(dl_file, 'r') as z:
      for content in z.namelist():
        dest = os.path.join(dest_dir, content[content.find('/')+1:])
        # Create dest parent dir if it does not exist.
        if not os.path.isdir(os.path.dirname(dest)):
          logging.info('Making %s', dest)
          os.makedirs(os.path.dirname(dest))
        # If dest is just a dir listing, do nothing.
        if not os.path.basename(dest):
          continue
        with z.open(content) as unzipped_content:
          logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
          with file(dest, 'wb') as dest_file:
            dest_file.write(unzipped_content.read())
          permissions = z.getinfo(content).external_attr >> 16
          if permissions:
            os.chmod(dest, permissions)
    return True

  def _ClearDir(self, dir):
    """Clears all files in |dir| except for hidden files and folders."""
    for root, dirs, files in os.walk(dir):
      # Skip hidden files and folders (like .svn and .git).
      files = [f for f in files if f[0] != '.']
      dirs[:] = [d for d in dirs if d[0] != '.']

      for f in files:
        os.remove(os.path.join(root, f))

  def _ExtractBuild(self):
    dest_dir = os.path.join(self._target_dir, self._folder)
    self._ClearDir(dest_dir)
    for root, _, dl_files in os.walk(os.path.join(self._download_dir)):
      for dl_file in dl_files:
        dl_file = os.path.join(root, dl_file)
        if not self._UnzipFile(dl_file, dest_dir):
          logging.info('Copying %s to %s', dl_file, dest_dir)
          shutil.copy(dl_file, dest_dir)
    shutil.rmtree(self._download_dir)

  def DownloadAndUpdateBuild(self):
    self._DownloadBuild()
    self._ExtractBuild()

def ParseOptions(argv):
  parser = optparse.OptionParser()
  usage = 'usage: %prog <options>'
  parser.set_usage(usage)
  parser.add_option('-r', dest='revision',
                    help='Revision to download.')
  parser.add_option('-t', dest='target_dir',
                    help='Target directory for unzipped Chromium.')

  (options, _) = parser.parse_args(argv)
  if not options.revision:
    logging.critical('Must specify -r.\n')
    sys.exit(1)
  if not options.target_dir:
    logging.critical('Must specify -t.\n')
    sys.exit(1)
  return options

def main(argv):
  logging.getLogger().setLevel(logging.DEBUG)
  options = ParseOptions(argv)
  b = BuildUpdater(options)
  b.DownloadAndUpdateBuild()
  logging.info('Successfully got archived Chromium build.')

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>Multivm branch: Fetch reference build from archive using git hash, not revision.<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Gets a Chromium archived build, and unpacks it
   into a target directory.

  Use -r option to specify the revison number
  Use -t option to specify the directory to unzip the build into.

Usage:
  $ get_chromium_build.py -r <revision> -t <target>
"""

import json
import logging
import optparse
import os
import platform
import shutil
import subprocess
import sys
import time
import urllib
import urllib2
import zipfile

# Example chromium build location:
# gs://chromium-browser-snapshots/Linux_x64/228977/chrome-linux.zip
CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
                    'chromium-browser-snapshots/%s/%s/%s')

class BuildUpdater(object):
  _PLATFORM_PATHS_MAP = {
      'Linux': { 'zipfiles': ['chrome-linux.zip'],
                 'folder': 'chrome_linux',
                 'archive_path': 'Linux_x64'},
      'Darwin': {'zipfiles': ['chrome-mac.zip'],
                 'folder': 'chrome_mac',
                 'archive_path': 'Mac'},
      'Windows': {'zipfiles': ['chrome-win32.zip',
                               'chrome-win32-syms.zip'],
                 'folder': 'chrome_win',
                 'archive_path': 'Win'}}

  def __init__(self, options):
    platform_data = BuildUpdater._PLATFORM_PATHS_MAP[platform.system()]
    self._zipfiles = platform_data['zipfiles']
    self._folder = platform_data['folder']
    self._archive_path = platform_data['archive_path']
    self._revision = options.revision
    self._target_dir = options.target_dir
    self._download_dir = os.path.join(self._target_dir, 'downloads')

  def _GetBuildUrl(self, revision, filename):
    return CHROMIUM_URL_FMT % (self._archive_path, revision, filename)

  def _FindBuildRevision(self, revision, filename):
    git_hash = json.loads(revision)[platform.system()]
    if self._DoesBuildExist(git_hash, filename):
      return git_hash
    return None

  def _DoesBuildExist(self, revision_guess, filename):
    url = self._GetBuildUrl(revision_guess, filename)

    r = urllib2.Request(url)
    r.get_method = lambda: 'HEAD'
    try:
      urllib2.urlopen(r)
      return True
    except urllib2.HTTPError, err:
      if err.code == 404:
        return False

  def _DownloadBuild(self):
    if not os.path.exists(self._download_dir):
      os.makedirs(self._download_dir)
    for zipfile in self._zipfiles:
      build_revision = self._FindBuildRevision(self._revision, zipfile)
      if not build_revision:
        logging.critical('Failed to find %s build for r%s\n',
                         self._archive_path,
                         self._revision)
        sys.exit(1)
      url = self._GetBuildUrl(build_revision, zipfile)
      logging.info('Downloading %s', url)
      r = urllib2.urlopen(url)
      with file(os.path.join(self._download_dir, zipfile), 'wb') as f:
        f.write(r.read())

  def _UnzipFile(self, dl_file, dest_dir):
    if not zipfile.is_zipfile(dl_file):
      return False
    logging.info('Unzipping %s', dl_file)
    with zipfile.ZipFile(dl_file, 'r') as z:
      for content in z.namelist():
        dest = os.path.join(dest_dir, content[content.find('/')+1:])
        # Create dest parent dir if it does not exist.
        if not os.path.isdir(os.path.dirname(dest)):
          logging.info('Making %s', dest)
          os.makedirs(os.path.dirname(dest))
        # If dest is just a dir listing, do nothing.
        if not os.path.basename(dest):
          continue
        with z.open(content) as unzipped_content:
          logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
          with file(dest, 'wb') as dest_file:
            dest_file.write(unzipped_content.read())
          permissions = z.getinfo(content).external_attr >> 16
          if permissions:
            os.chmod(dest, permissions)
    return True

  def _ClearDir(self, dir):
    """Clears all files in |dir| except for hidden files and folders."""
    for root, dirs, files in os.walk(dir):
      # Skip hidden files and folders (like .svn and .git).
      files = [f for f in files if f[0] != '.']
      dirs[:] = [d for d in dirs if d[0] != '.']

      for f in files:
        os.remove(os.path.join(root, f))

  def _ExtractBuild(self):
    dest_dir = os.path.join(self._target_dir, self._folder)
    self._ClearDir(dest_dir)
    for root, _, dl_files in os.walk(os.path.join(self._download_dir)):
      for dl_file in dl_files:
        dl_file = os.path.join(root, dl_file)
        if not self._UnzipFile(dl_file, dest_dir):
          logging.info('Copying %s to %s', dl_file, dest_dir)
          shutil.copy(dl_file, dest_dir)
    shutil.rmtree(self._download_dir)

  def DownloadAndUpdateBuild(self):
    self._DownloadBuild()
    self._ExtractBuild()

def ParseOptions(argv):
  parser = optparse.OptionParser()
  usage = 'usage: %prog <options>'
  parser.set_usage(usage)
  parser.add_option('-r', dest='revision',
                    help='Revision to download.')
  parser.add_option('-t', dest='target_dir',
                    help='Target directory for unzipped Chromium.')

  (options, _) = parser.parse_args(argv)
  if not options.revision:
    logging.critical('Must specify -r.\n')
    sys.exit(1)
  if not options.target_dir:
    logging.critical('Must specify -t.\n')
    sys.exit(1)
  return options

def main(argv):
  logging.getLogger().setLevel(logging.DEBUG)
  options = ParseOptions(argv)
  b = BuildUpdater(options)
  b.DownloadAndUpdateBuild()
  logging.info('Successfully got archived Chromium build.')

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>
from django import template
from django.conf import settings
from django.template.loader import render_to_string
from django.contrib.auth.models import User
from django.utils.encoding import smart_str
from django.template import Node, NodeList, Variable, Library
from django.template import TemplateSyntaxError, VariableDoesNotExist
from django.core.urlresolvers import reverse

from pagelets.models import Pagelet, Page

register = template.Library()

@register.inclusion_tag('pagelets/_render_pagelet.html', takes_context=True)
def render_pagelet(context, pagelet, *args):
    """
    Renders the named pagelet in the calling template.
    """
    print 'args', args
    if isinstance(pagelet, basestring):
        # add the slug separately because we need it in the template even
        # if this pagelet doesn't exist
        context['pagelet_slug'] = pagelet
        context['pagelet_name'] = unicode(pagelet)
        try:
            pagelet = Pagelet.objects.get(slug=pagelet)
        except Pagelet.DoesNotExist:
            pagelet = None

    if pagelet:
        # add the slug separately because we need it in the template even
        # if this pagelet doesn't exist
        context['pagelet_slug'] = pagelet.slug
        context['pagelet_name'] = unicode(pagelet)
        pagelet.rendered_content = pagelet.render(context)

    context['pagelet'] = pagelet
    if args:
        context['link_text'] = args[0]
    return context

@register.inclusion_tag('pagelets/_pagelink_ifexists.html', takes_context=True)
def pagelink_ifexists(context, page, link_text):
    """
    Renders a link to the given page in the calling template.
    """
    if isinstance(page, basestring):
        # add the slug separately because we need it in the template even
        # if this page doesn't exist
        context['page_slug'] = page
        context['pagelet_name'] = unicode(pagelet)
        try:
            page = Page.objects.get(slug=page)
        except Page.DoesNotExist:
            page = None

    if page:
        # add the slug separately because we need it in the template even
        # if this page doesn't exist
        context['page_slug'] = page.slug
        context['pagelet_name'] = unicode(pagelet)

    context['page'] = page
    context['link_text'] = link_text
    return context

<commit_msg>remove print statement refs #1<commit_after>
from django import template
from django.conf import settings
from django.template.loader import render_to_string
from django.contrib.auth.models import User
from django.utils.encoding import smart_str
from django.template import Node, NodeList, Variable, Library
from django.template import TemplateSyntaxError, VariableDoesNotExist
from django.core.urlresolvers import reverse

from pagelets.models import Pagelet, Page

register = template.Library()

@register.inclusion_tag('pagelets/_render_pagelet.html', takes_context=True)
def render_pagelet(context, pagelet, *args):
    """
    Renders the named pagelet in the calling template.
    """
    if isinstance(pagelet, basestring):
        # add the slug separately because we need it in the template even
        # if this pagelet doesn't exist
        context['pagelet_slug'] = pagelet
        context['pagelet_name'] = unicode(pagelet)
        try:
            pagelet = Pagelet.objects.get(slug=pagelet)
        except Pagelet.DoesNotExist:
            pagelet = None

    if pagelet:
        # add the slug separately because we need it in the template even
        # if this pagelet doesn't exist
        context['pagelet_slug'] = pagelet.slug
        context['pagelet_name'] = unicode(pagelet)
        pagelet.rendered_content = pagelet.render(context)

    context['pagelet'] = pagelet
    if args:
        context['link_text'] = args[0]
    return context

@register.inclusion_tag('pagelets/_pagelink_ifexists.html', takes_context=True)
def pagelink_ifexists(context, page, link_text):
    """
    Renders a link to the given page in the calling template.
    """
    if isinstance(page, basestring):
        # add the slug separately because we need it in the template even
        # if this page doesn't exist
        context['page_slug'] = page
        context['pagelet_name'] = unicode(pagelet)
        try:
            page = Page.objects.get(slug=page)
        except Page.DoesNotExist:
            page = None

    if page:
        # add the slug separately because we need it in the template even
        # if this page doesn't exist
        context['page_slug'] = page.slug
        context['pagelet_name'] = unicode(pagelet)

    context['page'] = page
    context['link_text'] = link_text
    return context

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2015-2015 breakwa11
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from __future__ import absolute_import, division, print_function, \
    with_statement

import os
import sys
import hashlib
import logging
import binascii
import struct
import base64
import time
import random
import hmac
import hashlib
import string

from shadowsocks import common
from shadowsocks.obfsplugin import plain
from shadowsocks.common import to_bytes, to_str, ord
from shadowsocks import lru_cache

def create_tls_ticket_auth_obfs(method):
    return tls_ticket_auth(method)

obfs_map = {
        'tls1.2_ticket_auth': (create_tls_ticket_auth_obfs,),
        'tls1.2_ticket_auth_compatible': (create_tls_ticket_auth_obfs,),
}

def match_begin(str1, str2):
    if len(str1) >= len(str2):
        if str1[:len(str2)] == str2:
            return True
    return False

class obfs_auth_data(object):
    def __init__(self):
        self.client_data = lru_cache.LRUCache(60 * 5)
        self.client_id = os.urandom(32)
        self.startup_time = int(time.time() - 60 * 30) & 0xFFFFFFFF

class tls_ticket_auth(plain.plain):
    def __init__(self, method):
        self.method = method
        self.handshake_status = 0
        self.send_buffer = b''
        self.recv_buffer = b''
        self.client_id = b''
        self.max_time_dif = 60 * 60 * 24 # time dif (second) setting
        self.tls_version = b'\x03\x03'

    def init_data(self):
        return obfs_auth_data()

    def sni(self, url):
        url = common.to_bytes(url)
        data = b"\x00" + struct.pack('>H', len(url)) + url
        data = b"\x00\x00" + struct.pack('>H', len(data) + 2) + struct.pack('>H', len(data)) + data
        return data

    def pack_auth_data(self, client_id):
        utc_time = int(time.time()) & 0xFFFFFFFF
        data = struct.pack('>I', utc_time) + os.urandom(18)
        data += hmac.new(self.server_info.key + client_id, data, hashlib.sha1).digest()[:10]
        return data

    def client_encode(self, buf):
        if self.handshake_status == -1:
            return buf
        if self.handshake_status == 8:
            ret = b''
            while len(buf) > 2048:
                size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf))
                ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size]
                buf = buf[size:]
            if len(buf) > 0:
                ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf
            return ret
        self.send_buffer += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf
        if self.handshake_status == 0:
            self.handshake_status = 1
            data = self.tls_version + self.pack_auth_data(self.server_info.data.client_id) + b"\x20" + self.server_info.data.client_id + binascii.unhexlify(b"001cc02bc02fcca9cca8cc14cc13c00ac014c009c013009c0035002f000a" + b"0100")
            ext = binascii.unhexlify(b"ff01000100")
            host = self.server_info.obfs_param or self.server_info.host
            if host and host[-1] in string.digits:
                host = ''
            hosts = host.split(',')
            host = random.choice(hosts)
            ext += self.sni(host)
            ext += b"\x00\x17\x00\x00"
            ext += b"\x00\x23\x00\xd0" + os.urandom(208) # ticket
            ext += binascii.unhexlify(b"000d001600140601060305010503040104030301030302010203")
            ext += binascii.unhexlify(b"000500050100000000")
            ext += binascii.unhexlify(b"00120000")
            ext += binascii.unhexlify(b"75500000")
            ext += binascii.unhexlify(b"000b00020100")
            ext += binascii.unhexlify(b"000a0006000400170018")
            data += struct.pack('>H', len(ext)) + ext
            data = b"\x01\x00" + struct.pack('>H', len(data)) + data
            data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data
            return data
        elif self.handshake_status == 1 and len(buf) == 0:
            data = b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec
            data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished
            data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10]
            ret = data + self.send_buffer
            self.send_buffer = b''
            self.handshake_status = 8
            return ret
        return b''

    def client_decode(self, buf):
        if self.handshake_status == -1:
            return (buf, False)

        if self.handshake_status == 8:
            ret = b''
            self.recv_buffer += buf
            while len(self.recv_buffer) > 5:
                if ord(self.recv_buffer[0]) != 0x17:
                    logging.info("data = %s" % (binascii.hexlify(self.recv_buffer)))
                    raise Exception('server_decode appdata error')
                size = struct.unpack('>H', self.recv_buffer[3:5])[0]
                if len(self.recv_buffer) < size + 5:
                    break
                buf = self.recv_buffer[5:size+5]
                ret += buf
                self.recv_buffer = self.recv_buffer[size+5:]
            return (ret, False)

        if len(buf) < 11 + 32 + 1 + 32:
            raise Exception('client_decode data error')
        verify = buf[11:33]
        if hmac.new(self.server_info.key + self.server_info.data.client_id, verify, hashlib.sha1).digest()[:10] != buf[33:43]:
            raise Exception('client_decode data error')
        if hmac.new(self.server_info.key + self.server_info.data.client_id, buf[:-10], hashlib.sha1).digest()[:10] != buf[-10:]:
            raise Exception('client_decode data error')
        return (b'', True)

    def server_encode(self, buf):
        if self.handshake_status == -1:
            return buf
        if self.handshake_status == 8:
            ret = b''
            while len(buf) > 2048:
                size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf))
                ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size]
                buf = buf[size:]
            if len(buf) > 0:
                ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf
            return ret
        self.handshake_status = 3
        data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100")
        data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello
        data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data
        if random.randint(0, 255) < 128:
            ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 256) + 64)
            ticket = struct.pack('>H', len(ticket) + 4) b"\x04\x00" + struct.pack('>H', len(ticket))
            data += b"\x16" + self.tls_version + ticket #New session ticket
        data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec
        data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished
        data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10]
        return data

    def decode_error_return(self, buf):
        self.handshake_status = -1
        if self.method == 'tls1.2_ticket_auth':
            return (b'E'*2048, False, False)
        return (buf, True, False)

    def server_decode(self, buf):
        if self.handshake_status == -1:
            return (buf, True, False)

        if self.handshake_status == 8:
            ret = b''
            self.recv_buffer += buf
            while len(self.recv_buffer) > 5:
                if ord(self.recv_buffer[0]) != 0x17 or ord(self.recv_buffer[1]) != 0x3 or ord(self.recv_buffer[2]) != 0x3:
                    logging.info("data = %s" % (binascii.hexlify(self.recv_buffer)))
                    raise Exception('server_decode appdata error')
                size = struct.unpack('>H', self.recv_buffer[3:5])[0]
                if len(self.recv_buffer) < size + 5:
                    break
                ret += self.recv_buffer[5:size+5]
                self.recv_buffer = self.recv_buffer[size+5:]
            return (ret, True, False)

        if self.handshake_status == 3:
            self.recv_buffer += buf
            buf = self.recv_buffer
            verify = buf
            verify_len = 43 - 10
            if len(buf) < 43:
                raise Exception('server_decode data error')
            if not match_begin(buf, b"\x14" + self.tls_version + b"\x00\x01\x01"): #ChangeCipherSpec
                raise Exception('server_decode data error')
            buf = buf[6:]
            if not match_begin(buf, b"\x16" + self.tls_version + b"\x00\x20"): #Finished
                raise Exception('server_decode data error')
            if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]:
                raise Exception('server_decode data error')
            if len(buf) < 37:
                raise Exception('server_decode data error')
            self.recv_buffer = buf[37:]
            self.handshake_status = 8
            return self.server_decode(b'')

        #raise Exception("handshake data = %s" % (binascii.hexlify(buf)))
        self.recv_buffer += buf
        buf = self.recv_buffer
        ogn_buf = buf
        if len(buf) < 3:
            return (b'', False, False)
        if not match_begin(buf, b'\x16\x03\x01'):
            return self.decode_error_return(ogn_buf)
        buf = buf[3:]
        if struct.unpack('>H', buf[:2])[0] > len(buf) - 2:
            return (b'', False, False)

        self.recv_buffer = self.recv_buffer[struct.unpack('>H', buf[:2])[0] + 5:]
        self.handshake_status = 2
        buf = buf[2:]
        if not match_begin(buf, b'\x01\x00'): #client hello
            logging.info("tls_auth not client hello message")
            return self.decode_error_return(ogn_buf)
        buf = buf[2:]
        if struct.unpack('>H', buf[:2])[0] != len(buf) - 2:
            logging.info("tls_auth wrong message size")
            return self.decode_error_return(ogn_buf)
        buf = buf[2:]
        if not match_begin(buf, self.tls_version):
            logging.info("tls_auth wrong tls version")
            return self.decode_error_return(ogn_buf)
        buf = buf[2:]
        verifyid = buf[:32]
        buf = buf[32:]
        sessionid_len = ord(buf[0])
        if sessionid_len < 32:
            logging.info("tls_auth wrong sessionid_len")
            return self.decode_error_return(ogn_buf)
        sessionid = buf[1:sessionid_len + 1]
        buf = buf[sessionid_len+1:]
        self.client_id = sessionid
        sha1 = hmac.new(self.server_info.key + sessionid, verifyid[:22], hashlib.sha1).digest()[:10]
        utc_time = struct.unpack('>I', verifyid[:4])[0]
        time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time)
        if self.server_info.obfs_param:
            try:
                self.max_time_dif = int(self.server_info.obfs_param)
            except:
                pass
        if self.max_time_dif > 0 and (time_dif < -self.max_time_dif or time_dif > self.max_time_dif \
                or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2):
            logging.info("tls_auth wrong time")
            return self.decode_error_return(ogn_buf)
        if sha1 != verifyid[22:]:
            logging.info("tls_auth wrong sha1")
            return self.decode_error_return(ogn_buf)
        if self.server_info.data.client_data.get(verifyid[:22]):
            logging.info("replay attack detect, id = %s" % (binascii.hexlify(verifyid)))
            return self.decode_error_return(ogn_buf)
        self.server_info.data.client_data.sweep()
        self.server_info.data.client_data[verifyid[:22]] = sessionid
        # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back)

        buf = buf[48:]

        host_name = ''
        for index in range(len(buf)):
            if index + 4 < len(buf):
                if buf[index:index + 4] == b"\x00\x17\x00\x00":
                    if buf[:index] != '':
                        host_name = buf[:index]

        return (b'', False, True, host_name)
<commit_msg>fix syntax<commit_after>#!/usr/bin/env python
#
# Copyright 2015-2015 breakwa11
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from __future__ import absolute_import, division, print_function, \
    with_statement

import os
import sys
import hashlib
import logging
import binascii
import struct
import base64
import time
import random
import hmac
import hashlib
import string

from shadowsocks import common
from shadowsocks.obfsplugin import plain
from shadowsocks.common import to_bytes, to_str, ord
from shadowsocks import lru_cache

def create_tls_ticket_auth_obfs(method):
    return tls_ticket_auth(method)

obfs_map = {
        'tls1.2_ticket_auth': (create_tls_ticket_auth_obfs,),
        'tls1.2_ticket_auth_compatible': (create_tls_ticket_auth_obfs,),
}

def match_begin(str1, str2):
    if len(str1) >= len(str2):
        if str1[:len(str2)] == str2:
            return True
    return False

class obfs_auth_data(object):
    def __init__(self):
        self.client_data = lru_cache.LRUCache(60 * 5)
        self.client_id = os.urandom(32)
        self.startup_time = int(time.time() - 60 * 30) & 0xFFFFFFFF

class tls_ticket_auth(plain.plain):
    def __init__(self, method):
        self.method = method
        self.handshake_status = 0
        self.send_buffer = b''
        self.recv_buffer = b''
        self.client_id = b''
        self.max_time_dif = 60 * 60 * 24 # time dif (second) setting
        self.tls_version = b'\x03\x03'

    def init_data(self):
        return obfs_auth_data()

    def sni(self, url):
        url = common.to_bytes(url)
        data = b"\x00" + struct.pack('>H', len(url)) + url
        data = b"\x00\x00" + struct.pack('>H', len(data) + 2) + struct.pack('>H', len(data)) + data
        return data

    def pack_auth_data(self, client_id):
        utc_time = int(time.time()) & 0xFFFFFFFF
        data = struct.pack('>I', utc_time) + os.urandom(18)
        data += hmac.new(self.server_info.key + client_id, data, hashlib.sha1).digest()[:10]
        return data

    def client_encode(self, buf):
        if self.handshake_status == -1:
            return buf
        if self.handshake_status == 8:
            ret = b''
            while len(buf) > 2048:
                size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf))
                ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size]
                buf = buf[size:]
            if len(buf) > 0:
                ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf
            return ret
        self.send_buffer += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf
        if self.handshake_status == 0:
            self.handshake_status = 1
            data = self.tls_version + self.pack_auth_data(self.server_info.data.client_id) + b"\x20" + self.server_info.data.client_id + binascii.unhexlify(b"001cc02bc02fcca9cca8cc14cc13c00ac014c009c013009c0035002f000a" + b"0100")
            ext = binascii.unhexlify(b"ff01000100")
            host = self.server_info.obfs_param or self.server_info.host
            if host and host[-1] in string.digits:
                host = ''
            hosts = host.split(',')
            host = random.choice(hosts)
            ext += self.sni(host)
            ext += b"\x00\x17\x00\x00"
            ext += b"\x00\x23\x00\xd0" + os.urandom(208) # ticket
            ext += binascii.unhexlify(b"000d001600140601060305010503040104030301030302010203")
            ext += binascii.unhexlify(b"000500050100000000")
            ext += binascii.unhexlify(b"00120000")
            ext += binascii.unhexlify(b"75500000")
            ext += binascii.unhexlify(b"000b00020100")
            ext += binascii.unhexlify(b"000a0006000400170018")
            data += struct.pack('>H', len(ext)) + ext
            data = b"\x01\x00" + struct.pack('>H', len(data)) + data
            data = b"\x16\x03\x01" + struct.pack('>H', len(data)) + data
            return data
        elif self.handshake_status == 1 and len(buf) == 0:
            data = b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec
            data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished
            data += hmac.new(self.server_info.key + self.server_info.data.client_id, data, hashlib.sha1).digest()[:10]
            ret = data + self.send_buffer
            self.send_buffer = b''
            self.handshake_status = 8
            return ret
        return b''

    def client_decode(self, buf):
        if self.handshake_status == -1:
            return (buf, False)

        if self.handshake_status == 8:
            ret = b''
            self.recv_buffer += buf
            while len(self.recv_buffer) > 5:
                if ord(self.recv_buffer[0]) != 0x17:
                    logging.info("data = %s" % (binascii.hexlify(self.recv_buffer)))
                    raise Exception('server_decode appdata error')
                size = struct.unpack('>H', self.recv_buffer[3:5])[0]
                if len(self.recv_buffer) < size + 5:
                    break
                buf = self.recv_buffer[5:size+5]
                ret += buf
                self.recv_buffer = self.recv_buffer[size+5:]
            return (ret, False)

        if len(buf) < 11 + 32 + 1 + 32:
            raise Exception('client_decode data error')
        verify = buf[11:33]
        if hmac.new(self.server_info.key + self.server_info.data.client_id, verify, hashlib.sha1).digest()[:10] != buf[33:43]:
            raise Exception('client_decode data error')
        if hmac.new(self.server_info.key + self.server_info.data.client_id, buf[:-10], hashlib.sha1).digest()[:10] != buf[-10:]:
            raise Exception('client_decode data error')
        return (b'', True)

    def server_encode(self, buf):
        if self.handshake_status == -1:
            return buf
        if self.handshake_status == 8:
            ret = b''
            while len(buf) > 2048:
                size = min(struct.unpack('>H', os.urandom(2))[0] % 4096 + 100, len(buf))
                ret += b"\x17" + self.tls_version + struct.pack('>H', size) + buf[:size]
                buf = buf[size:]
            if len(buf) > 0:
                ret += b"\x17" + self.tls_version + struct.pack('>H', len(buf)) + buf
            return ret
        self.handshake_status = 3
        data = self.tls_version + self.pack_auth_data(self.client_id) + b"\x20" + self.client_id + binascii.unhexlify(b"c02f000005ff01000100")
        data = b"\x02\x00" + struct.pack('>H', len(data)) + data #server hello
        data = b"\x16\x03\x03" + struct.pack('>H', len(data)) + data
        if random.randint(0, 255) < 128:
            ticket = os.urandom((struct.unpack('>H', os.urandom(2))[0] % 256) + 64)
            ticket = struct.pack('>H', len(ticket) + 4) + b"\x04\x00" + struct.pack('>H', len(ticket))
            data += b"\x16" + self.tls_version + ticket #New session ticket
        data += b"\x14" + self.tls_version + b"\x00\x01\x01" #ChangeCipherSpec
        data += b"\x16" + self.tls_version + b"\x00\x20" + os.urandom(22) #Finished
        data += hmac.new(self.server_info.key + self.client_id, data, hashlib.sha1).digest()[:10]
        return data

    def decode_error_return(self, buf):
        self.handshake_status = -1
        if self.method == 'tls1.2_ticket_auth':
            return (b'E'*2048, False, False)
        return (buf, True, False)

    def server_decode(self, buf):
        if self.handshake_status == -1:
            return (buf, True, False)

        if self.handshake_status == 8:
            ret = b''
            self.recv_buffer += buf
            while len(self.recv_buffer) > 5:
                if ord(self.recv_buffer[0]) != 0x17 or ord(self.recv_buffer[1]) != 0x3 or ord(self.recv_buffer[2]) != 0x3:
                    logging.info("data = %s" % (binascii.hexlify(self.recv_buffer)))
                    raise Exception('server_decode appdata error')
                size = struct.unpack('>H', self.recv_buffer[3:5])[0]
                if len(self.recv_buffer) < size + 5:
                    break
                ret += self.recv_buffer[5:size+5]
                self.recv_buffer = self.recv_buffer[size+5:]
            return (ret, True, False)

        if self.handshake_status == 3:
            self.recv_buffer += buf
            buf = self.recv_buffer
            verify = buf
            verify_len = 43 - 10
            if len(buf) < 43:
                raise Exception('server_decode data error')
            if not match_begin(buf, b"\x14" + self.tls_version + b"\x00\x01\x01"): #ChangeCipherSpec
                raise Exception('server_decode data error')
            buf = buf[6:]
            if not match_begin(buf, b"\x16" + self.tls_version + b"\x00\x20"): #Finished
                raise Exception('server_decode data error')
            if hmac.new(self.server_info.key + self.client_id, verify[:verify_len], hashlib.sha1).digest()[:10] != verify[verify_len:verify_len+10]:
                raise Exception('server_decode data error')
            if len(buf) < 37:
                raise Exception('server_decode data error')
            self.recv_buffer = buf[37:]
            self.handshake_status = 8
            return self.server_decode(b'')

        #raise Exception("handshake data = %s" % (binascii.hexlify(buf)))
        self.recv_buffer += buf
        buf = self.recv_buffer
        ogn_buf = buf
        if len(buf) < 3:
            return (b'', False, False)
        if not match_begin(buf, b'\x16\x03\x01'):
            return self.decode_error_return(ogn_buf)
        buf = buf[3:]
        if struct.unpack('>H', buf[:2])[0] > len(buf) - 2:
            return (b'', False, False)

        self.recv_buffer = self.recv_buffer[struct.unpack('>H', buf[:2])[0] + 5:]
        self.handshake_status = 2
        buf = buf[2:]
        if not match_begin(buf, b'\x01\x00'): #client hello
            logging.info("tls_auth not client hello message")
            return self.decode_error_return(ogn_buf)
        buf = buf[2:]
        if struct.unpack('>H', buf[:2])[0] != len(buf) - 2:
            logging.info("tls_auth wrong message size")
            return self.decode_error_return(ogn_buf)
        buf = buf[2:]
        if not match_begin(buf, self.tls_version):
            logging.info("tls_auth wrong tls version")
            return self.decode_error_return(ogn_buf)
        buf = buf[2:]
        verifyid = buf[:32]
        buf = buf[32:]
        sessionid_len = ord(buf[0])
        if sessionid_len < 32:
            logging.info("tls_auth wrong sessionid_len")
            return self.decode_error_return(ogn_buf)
        sessionid = buf[1:sessionid_len + 1]
        buf = buf[sessionid_len+1:]
        self.client_id = sessionid
        sha1 = hmac.new(self.server_info.key + sessionid, verifyid[:22], hashlib.sha1).digest()[:10]
        utc_time = struct.unpack('>I', verifyid[:4])[0]
        time_dif = common.int32((int(time.time()) & 0xffffffff) - utc_time)
        if self.server_info.obfs_param:
            try:
                self.max_time_dif = int(self.server_info.obfs_param)
            except:
                pass
        if self.max_time_dif > 0 and (time_dif < -self.max_time_dif or time_dif > self.max_time_dif \
                or common.int32(utc_time - self.server_info.data.startup_time) < -self.max_time_dif / 2):
            logging.info("tls_auth wrong time")
            return self.decode_error_return(ogn_buf)
        if sha1 != verifyid[22:]:
            logging.info("tls_auth wrong sha1")
            return self.decode_error_return(ogn_buf)
        if self.server_info.data.client_data.get(verifyid[:22]):
            logging.info("replay attack detect, id = %s" % (binascii.hexlify(verifyid)))
            return self.decode_error_return(ogn_buf)
        self.server_info.data.client_data.sweep()
        self.server_info.data.client_data[verifyid[:22]] = sessionid
        # (buffer_to_recv, is_need_decrypt, is_need_to_encode_and_send_back)

        buf = buf[48:]

        host_name = ''
        for index in range(len(buf)):
            if index + 4 < len(buf):
                if buf[index:index + 4] == b"\x00\x17\x00\x00":
                    if buf[:index] != '':
                        host_name = buf[:index]

        return (b'', False, True, host_name)
<|endoftext|>


<commit_before>from flask import request
import requests
from ..utils.status import get_status
from ..utils.reminders import create_reminder
import twilio.twiml
import json
import dateutil.parser

def call():
    resp = twilio.twiml.Response()
    resp.record(timeout=10, transcribe=True,
            transcribeCallback='http://queri.me/internal/rec', )
    return str(resp)

def do_wit(body, phone):
    wit = requests.get('https://api.wit.ai/message?v=20140905&q=%s' % body, headers={'Authorization':'Bearer L3VB34V6YTDFO4BRXNDQNAYMVOOF4BHB'}).text
    jso = json.loads(wit)
    print str(jso)
    intent = jso['outcomes'][0]['intent']
    if intent == 'get_status':
        m = get_status(wit, phone)
    elif intent == 'remind':
        entities = jso['outcomes'][0]['entities']
        date = dateutil.parser.parse(entities['time'][0]['value']['from'])
        text = entities['message'][0]['value']
        m = create_reminder(date, text, phone)
    else:
        m = "Hmm? Try again please :("
    return m

def text():
    b = request.form.get('Body','')
    phone = request.form.get('From','')
    m = do_wit(b, phone)

    # Send to wit.ai for processing
    resp = twilio.twiml.Response()
    resp.message(m)
    return str(resp)

def rec():
    print request.form.get('TranscriptionText','')
    m = do_wit(request.form.get('TranscriptionText',''),request.form.get('From',''))
    return ''
<commit_msg>Reply with response<commit_after>from flask import request
import requests
from ..utils.status import get_status
from ..utils.reminders import create_reminder
import twilio.twiml
from twilio.rest import TwilioRestClient
import json
import dateutil.parser

ACCOUNT_SID = "ayylmao"
AUTH_TOKEN = "ayylmao"

def call():
    resp = twilio.twiml.Response()
    resp.record(timeout=10, transcribe=True,
            transcribeCallback='http://queri.me/internal/rec', )
    return str(resp)

def do_wit(body, phone):
    wit = requests.get('https://api.wit.ai/message?v=20140905&q=%s' % body, headers={'Authorization':'Bearer L3VB34V6YTDFO4BRXNDQNAYMVOOF4BHB'}).text
    jso = json.loads(wit)
    print str(jso)
    intent = jso['outcomes'][0]['intent']
    if intent == 'get_status':
        m = get_status(wit, phone)
    elif intent == 'remind':
        entities = jso['outcomes'][0]['entities']
        date = dateutil.parser.parse(entities['time'][0]['value']['from'])
        text = entities['message'][0]['value']
        m = create_reminder(date, text, phone)
    else:
        m = "Hmm? Try again please :("
    return m

def text():
    b = request.form.get('Body','')
    phone = request.form.get('From','')
    m = do_wit(b, phone)

    # Send to wit.ai for processing
    resp = twilio.twiml.Response()
    resp.message(m)
    return str(resp)

def rec():
    print request.form.get('TranscriptionText','')
    m = do_wit(request.form.get('TranscriptionText',''),request.form.get('From',''))
    client = TwilioRestClient(account_sid, auth_token)
    message = client.messages.create(from_="+15172194225", to=request.form.get('From',''), body=m)
    return ''
<|endoftext|>


<commit_before>"""
RPM packages
============

This module provides tools to manage CentOS/RHEL/SL/Fedora packages
and repositories.

"""
from __future__ import with_statement

from fabric.api import hide, run, settings

from fabtools.utils import run_as_root

MANAGER = 'yum -y --color=never'

def update(kernel=False):
    """
    Upgrade all packages, skip obsoletes if ``obsoletes=0`` in ``yum.conf``.

    Exclude *kernel* upgrades by default.
    """
    manager = MANAGER
    cmds = {'yum -y --color=never': {False: '--exclude=kernel* update', True: 'update'}}
    cmd = cmds[manager][kernel]
    run_as_root("%(manager)s %(cmd)s" % locals())

def upgrade(kernel=False):
    """
    Upgrade all packages, including obsoletes.

    Exclude *kernel* upgrades by default.
    """
    manager = MANAGER
    cmds = {'yum -y --color=never': {False: '--exclude=kernel* upgrade', True: 'upgrade'}}
    cmd = cmds[manager][kernel]
    run_as_root("%(manager)s %(cmd)s" % locals())

def groupupdate(group, options=None):
    """
    Update an existing software group, skip obsoletes if ``obsoletes=1`` in ``yum.conf``.

    Extra *options* may be passed to ``yum`` if necessary.
    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s groupupdate "%(group)s"' % locals())

def is_installed(pkg_name):
    """
    Check if an RPM package is installed.
    """
    manager = MANAGER
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = run("%(manager)s list installed %(pkg_name)s" % locals())
        if res.succeeded:
            return True
        return False

def install(packages, repos=None, yes=None, options=None):
    """
    Install one or more RPM packages.

    Extra *repos* may be passed to ``yum`` to enable extra repositories at install time.

    Extra *yes* may be passed to ``yum`` to validate license if necessary.

    Extra *options* may be passed to ``yum`` if necessary
    (e.g. ``['--nogpgcheck', '--exclude=package']``).

    ::

        import fabtools

        # Install a single package, in an alternative install root
        fabtools.rpm.install('emacs', options='--installroot=/my/new/location')

        # Install multiple packages silently
        fabtools.rpm.install([
            'unzip',
            'nano'
        ], '--quiet')

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    if repos:
        for repo in repos:
            options.append('--enablerepo=%(repo)s' % locals())
    options = " ".join(options)
    if isinstance(yes, str):
        run_as_root('yes %(yes)s | %(manager)s %(options)s install %(packages)s' % locals())
    else:
        run_as_root('%(manager)s %(options)s install %(packages)s' % locals())

def groupinstall(group, options=None):
    """
    Install a group of packages.

    You can use ``yum grouplist`` to get the list of groups.

    Extra *options* may be passed to ``yum`` if necessary like
    (e.g. ``['--nogpgcheck', '--exclude=package']``).

    ::

        import fabtools

        # Install development packages
        fabtools.rpm.groupinstall('Development tools')

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s groupinstall "%(group)s"' % locals(), pty=False)

def uninstall(packages, options=None):
    """
    Remove one or more packages.

    Extra *options* may be passed to ``yum`` if necessary.

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s remove %(packages)s' % locals())

def groupuninstall(group, options=None):
    """
    Remove an existing software group.

    Extra *options* may be passed to ``yum`` if necessary.

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s groupremove "%(group)s"' % locals())

def repolist(status='', media=None):
    """
    Get the list of ``yum`` repositories.

    Returns enabled repositories by default. Extra *status* may be passed
    to list disabled repositories if necessary.

    Media and debug repositories are kept disabled, except if you pass *media*.

    ::

        import fabtools

        # Install a package that may be included in disabled repositories
        fabtools.rpm.install('vim', fabtools.rpm.repolist('disabled'))

    """
    manager = MANAGER
    with settings(hide('running', 'stdout')):
        if media:
            repos = run_as_root("%(manager)s repolist %(status)s | sed '$d' | sed -n '/repo id/,$p'" % locals())
        else:
            repos = run_as_root("%(manager)s repolist %(status)s | sed '/Media\\|Debug/d' | sed '$d' | sed -n '/repo id/,$p'" % locals())
        return map(lambda line: line.split(' ')[0], repos.splitlines()[1:])
<commit_msg>Use rpm instead of yum to check if RPM package is installed [#220]<commit_after>"""
RPM packages
============

This module provides tools to manage CentOS/RHEL/SL/Fedora packages
and repositories.

"""
from __future__ import with_statement

from fabric.api import hide, run, settings

from fabtools.utils import run_as_root

MANAGER = 'yum -y --color=never'

def update(kernel=False):
    """
    Upgrade all packages, skip obsoletes if ``obsoletes=0`` in ``yum.conf``.

    Exclude *kernel* upgrades by default.
    """
    manager = MANAGER
    cmds = {'yum -y --color=never': {False: '--exclude=kernel* update', True: 'update'}}
    cmd = cmds[manager][kernel]
    run_as_root("%(manager)s %(cmd)s" % locals())

def upgrade(kernel=False):
    """
    Upgrade all packages, including obsoletes.

    Exclude *kernel* upgrades by default.
    """
    manager = MANAGER
    cmds = {'yum -y --color=never': {False: '--exclude=kernel* upgrade', True: 'upgrade'}}
    cmd = cmds[manager][kernel]
    run_as_root("%(manager)s %(cmd)s" % locals())

def groupupdate(group, options=None):
    """
    Update an existing software group, skip obsoletes if ``obsoletes=1`` in ``yum.conf``.

    Extra *options* may be passed to ``yum`` if necessary.
    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s groupupdate "%(group)s"' % locals())

def is_installed(pkg_name):
    """
    Check if an RPM package is installed.
    """
    manager = MANAGER
    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
        res = run("rpm --query %(pkg_name)s" % locals())
        if res.succeeded:
            return True
        return False

def install(packages, repos=None, yes=None, options=None):
    """
    Install one or more RPM packages.

    Extra *repos* may be passed to ``yum`` to enable extra repositories at install time.

    Extra *yes* may be passed to ``yum`` to validate license if necessary.

    Extra *options* may be passed to ``yum`` if necessary
    (e.g. ``['--nogpgcheck', '--exclude=package']``).

    ::

        import fabtools

        # Install a single package, in an alternative install root
        fabtools.rpm.install('emacs', options='--installroot=/my/new/location')

        # Install multiple packages silently
        fabtools.rpm.install([
            'unzip',
            'nano'
        ], '--quiet')

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    if repos:
        for repo in repos:
            options.append('--enablerepo=%(repo)s' % locals())
    options = " ".join(options)
    if isinstance(yes, str):
        run_as_root('yes %(yes)s | %(manager)s %(options)s install %(packages)s' % locals())
    else:
        run_as_root('%(manager)s %(options)s install %(packages)s' % locals())

def groupinstall(group, options=None):
    """
    Install a group of packages.

    You can use ``yum grouplist`` to get the list of groups.

    Extra *options* may be passed to ``yum`` if necessary like
    (e.g. ``['--nogpgcheck', '--exclude=package']``).

    ::

        import fabtools

        # Install development packages
        fabtools.rpm.groupinstall('Development tools')

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s groupinstall "%(group)s"' % locals(), pty=False)

def uninstall(packages, options=None):
    """
    Remove one or more packages.

    Extra *options* may be passed to ``yum`` if necessary.

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    if not isinstance(packages, basestring):
        packages = " ".join(packages)
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s remove %(packages)s' % locals())

def groupuninstall(group, options=None):
    """
    Remove an existing software group.

    Extra *options* may be passed to ``yum`` if necessary.

    """
    manager = MANAGER
    if options is None:
        options = []
    elif isinstance(options, str):
        options = [options]
    options = " ".join(options)
    run_as_root('%(manager)s %(options)s groupremove "%(group)s"' % locals())

def repolist(status='', media=None):
    """
    Get the list of ``yum`` repositories.

    Returns enabled repositories by default. Extra *status* may be passed
    to list disabled repositories if necessary.

    Media and debug repositories are kept disabled, except if you pass *media*.

    ::

        import fabtools

        # Install a package that may be included in disabled repositories
        fabtools.rpm.install('vim', fabtools.rpm.repolist('disabled'))

    """
    manager = MANAGER
    with settings(hide('running', 'stdout')):
        if media:
            repos = run_as_root("%(manager)s repolist %(status)s | sed '$d' | sed -n '/repo id/,$p'" % locals())
        else:
            repos = run_as_root("%(manager)s repolist %(status)s | sed '/Media\\|Debug/d' | sed '$d' | sed -n '/repo id/,$p'" % locals())
        return map(lambda line: line.split(' ')[0], repos.splitlines()[1:])
<|endoftext|>