<commit_before># Copyright (C) 2006-2007 Jelmer Vernooij <jelmer@samba.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA

"""Marshalling for the svn_ra protocol."""

class literal:
    """A protocol literal."""

    def __init__(self, txt):
        self.txt = txt

    def __str__(self):
        return self.txt

    def __repr__(self):
        return self.txt

# 1. Syntactic structure
# ----------------------
#
# The Subversion protocol is specified in terms of the following
# syntactic elements, specified using ABNF [RFC 2234]:
#
#   item   = word / number / string / list
#   word   = ALPHA *(ALPHA / DIGIT / "-") space
#   number = 1*DIGIT space
#   string = 1*DIGIT ":" *OCTET space
#          ; digits give the byte count of the *OCTET portion
#   list   = "(" space *item ")" space
#   space  = 1*(SP / LF)
#

class MarshallError(Exception):
    """A Marshall error."""

class NeedMoreData(MarshallError):
    """More data needed."""

def marshall(x):
    """Marshall a Python data item.

    :param x: Data item
    :return: encoded string
    """
    if type(x) is int:
        return "%d " % x
    elif type(x) is list or type(x) is tuple:
        return "( " + "".join(map(marshall, x)) + ") "
    elif isinstance(x, literal):
        return "%s " % x
    elif type(x) is str:
        return "%d:%s " % (len(x), x)
    elif type(x) is unicode:
        return "%d:%s " % (len(x), x.encode("utf-8"))
    elif type(x) is bool:
        if x == True:
            return "true "
        elif x == False:
            return "false "
    raise MarshallError("Unable to marshall type %s" % x)

def unmarshall(x):
    """Unmarshall the next item from a text.

    :param x: Text to parse
    :return: tuple with unpacked item and remaining text
    """
    whitespace = ['\n', ' ']
    if len(x) == 0:
        raise NeedMoreData("Not enough data")
    if x[0] == "(": # list follows
        if len(x) <= 1:
            raise NeedMoreData("Missing whitespace")
        if x[1] != " ":
            raise MarshallError("missing whitespace after list start")
        x = x[2:]
        ret = []
        try:
            while x[0] != ")":
                (x, n) = unmarshall(x)
                ret.append(n)
        except IndexError:
            raise NeedMoreData("List not terminated")

        if len(x) <= 1:
            raise NeedMoreData("Missing whitespace")

        if not x[1] in whitespace:
            raise MarshallError("Expected space, got %c" % x[1])

        return (x[2:], ret)
    elif x[0].isdigit():
        num = ""
        # Check if this is a string or a number
        while x[0].isdigit():
            num += x[0]
            x = x[1:]
        num = int(num)

        if x[0] in whitespace:
            return (x[1:], num)
        elif x[0] == ":":
            if len(x) < num:
                raise NeedMoreData("Expected string of length %r" % num)
            return (x[num+2:], x[1:num+1])
        else:
            raise MarshallError("Expected whitespace or ':', got '%c" % x[0])
    elif x[0].isalpha():
        ret = ""
        # Parse literal
        try:
            while x[0].isalpha() or x[0].isdigit() or x[0] == '-':
                ret += x[0]
                x = x[1:]
        except IndexError:
            raise NeedMoreData("Expected literal")

        if not x[0] in whitespace:
            raise MarshallError("Expected whitespace, got %c" % x[0])

        return (x[1:], ret)
    else:
        raise MarshallError("Unexpected character '%c'" % x[0])

<commit_msg>Encode the number of UTF-8 bytes in subvertpy.marshall.marshall()<commit_after># Copyright (C) 2006-2007 Jelmer Vernooij <jelmer@samba.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA

"""Marshalling for the svn_ra protocol."""

class literal:
    """A protocol literal."""

    def __init__(self, txt):
        self.txt = txt

    def __str__(self):
        return self.txt

    def __repr__(self):
        return self.txt

# 1. Syntactic structure
# ----------------------
#
# The Subversion protocol is specified in terms of the following
# syntactic elements, specified using ABNF [RFC 2234]:
#
#   item   = word / number / string / list
#   word   = ALPHA *(ALPHA / DIGIT / "-") space
#   number = 1*DIGIT space
#   string = 1*DIGIT ":" *OCTET space
#          ; digits give the byte count of the *OCTET portion
#   list   = "(" space *item ")" space
#   space  = 1*(SP / LF)
#

class MarshallError(Exception):
    """A Marshall error."""

class NeedMoreData(MarshallError):
    """More data needed."""

def marshall(x):
    """Marshall a Python data item.

    :param x: Data item
    :return: encoded string
    """
    if type(x) is int:
        return "%d " % x
    elif type(x) is list or type(x) is tuple:
        return "( " + "".join(map(marshall, x)) + ") "
    elif isinstance(x, literal):
        return "%s " % x
    elif type(x) is str:
        return "%d:%s " % (len(x), x)
    elif type(x) is unicode:
        x = x.encode("utf-8")
        return "%d:%s " % (len(x), x)
    elif type(x) is bool:
        if x == True:
            return "true "
        elif x == False:
            return "false "
    raise MarshallError("Unable to marshall type %s" % x)

def unmarshall(x):
    """Unmarshall the next item from a text.

    :param x: Text to parse
    :return: tuple with unpacked item and remaining text
    """
    whitespace = ['\n', ' ']
    if len(x) == 0:
        raise NeedMoreData("Not enough data")
    if x[0] == "(": # list follows
        if len(x) <= 1:
            raise NeedMoreData("Missing whitespace")
        if x[1] != " ":
            raise MarshallError("missing whitespace after list start")
        x = x[2:]
        ret = []
        try:
            while x[0] != ")":
                (x, n) = unmarshall(x)
                ret.append(n)
        except IndexError:
            raise NeedMoreData("List not terminated")

        if len(x) <= 1:
            raise NeedMoreData("Missing whitespace")

        if not x[1] in whitespace:
            raise MarshallError("Expected space, got %c" % x[1])

        return (x[2:], ret)
    elif x[0].isdigit():
        num = ""
        # Check if this is a string or a number
        while x[0].isdigit():
            num += x[0]
            x = x[1:]
        num = int(num)

        if x[0] in whitespace:
            return (x[1:], num)
        elif x[0] == ":":
            if len(x) < num:
                raise NeedMoreData("Expected string of length %r" % num)
            return (x[num+2:], x[1:num+1])
        else:
            raise MarshallError("Expected whitespace or ':', got '%c" % x[0])
    elif x[0].isalpha():
        ret = ""
        # Parse literal
        try:
            while x[0].isalpha() or x[0].isdigit() or x[0] == '-':
                ret += x[0]
                x = x[1:]
        except IndexError:
            raise NeedMoreData("Expected literal")

        if not x[0] in whitespace:
            raise MarshallError("Expected whitespace, got %c" % x[0])

        return (x[1:], ret)
    else:
        raise MarshallError("Unexpected character '%c'" % x[0])

<|endoftext|>


<commit_before>
import ast
import errno
import os
import platform
import shutil
import subprocess
import sys

from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.sdist import sdist
from distutils.errors import DistutilsError

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno==errno.EEXIST and os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK):
            return
        raise

def rmtree(path):
    try:
        shutil.rmtree(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def exec_process(cmdline, silent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)
        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.decode("utf-8").rstrip("\n")))
    return stdout

def prepare_windows_env(env):
    if sys.version_info < (3, 3):
        env.pop('VS100COMNTOOLS', None)

    if env.get('PYTHON'):
        return  # Already manually set by user.

    if (2, 6) <= sys.version_info[:2] <= (2, 7):
        env['PYTHON'] = sys.executable
        return  # The current executable is fine.

    # Try if `python` on PATH is the right one. If we would execute
    # `python` directly the current executable might be used so we
    # delegate this to cmd.
    cmd = ['cmd.exe', '/C',  'python', '-c', 'import sys; '
           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
           'sys.stdout.flush()']
    try:
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        stdout, _ = sub.communicate()
        version = ast.literal_eval(stdout.decode('utf-8').strip())
        if (2, 6) <= version <= (2, 7):
            return  # Python on PATH is fine
    except OSError:
        pass

    # Check default install locations
    for v in ('26', '27'):
        path = os.path.join('%SYSTEMDRIVE%', 'Python%s' % v, 'python.exe')
        path = os.path.expandvars(path)
        if os.path.isfile(path):
            log.info('Using "%s" to build libuv...' % path)
            env['PYTHON'] = path
            return

    raise DistutilsError('No appropriate Python version found. An '
                         'installation of 2.6 or 2.7 is required to '
                         'build libuv. You can set the environment '
                         'variable "PYTHON" to point to a custom '
                         'installation location.')

class libuv_build_ext(build_ext):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = 'https://github.com/joyent/libuv.git'
    libuv_branch   = 'master'
    libuv_revision = '2f84a57'
    libuv_patches  = []

    user_options = build_ext.user_options
    user_options.extend([
        ("libuv-clean-compile", None, "Clean libuv tree before compilation"),
        ("libuv-force-fetch", None, "Remove libuv (if present) and fetch it again")
    ])
    boolean_options = build_ext.boolean_options
    boolean_options.extend(["libuv-clean-compile", "libuv-force-fetch"])

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.libuv_clean_compile = 0
        self.libuv_force_fetch = 0

    def build_extensions(self):
        if self.compiler.compiler_type == 'mingw32':
            # Dirty hack to avoid linking with more than one C runtime when using MinGW
            # Note that this hack forces the compilation to use the old MSVCT
            self.compiler.dll_libraries = [lib for lib in self.compiler.dll_libraries if not lib.startswith('msvcr')]
        self.force = self.libuv_force_fetch or self.libuv_clean_compile
        if self.compiler.compiler_type == 'msvc':
            self.libuv_lib = os.path.join(self.libuv_dir, 'Release', 'lib', 'libuv.lib')
        else:
            self.libuv_lib = os.path.join(self.libuv_dir, 'libuv.a')
        self.get_libuv()
        # Set compiler options
        if self.compiler.compiler_type == 'mingw32':
            self.compiler.add_library_dir(self.libuv_dir)
            self.compiler.add_library('uv')
        else:
            self.extensions[0].extra_objects.extend([self.libuv_lib])
        self.compiler.add_include_dir(os.path.join(self.libuv_dir, 'include'))
        if sys.platform.startswith('linux'):
            self.compiler.add_library('rt')
        elif sys.platform == 'darwin':
            self.extensions[0].extra_link_args.extend(['-framework', 'CoreServices'])
        elif sys.platform == 'win32':
            if self.compiler.compiler_type == 'msvc':
                self.extensions[0].extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
                self.compiler.add_library('advapi32')
            self.compiler.add_library('iphlpapi')
            self.compiler.add_library('psapi')
            self.compiler.add_library('ws2_32')
        build_ext.build_extensions(self)

    def get_libuv(self):
        #self.debug_mode =  bool(self.debug) or hasattr(sys, 'gettotalrefcount')
        win32_msvc = self.compiler.compiler_type=='msvc'
        def download_libuv():
            log.info('Downloading libuv...')
            makedirs(self.libuv_dir)
            exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
            exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        def patch_libuv():
            if self.libuv_patches:
                log.info('Patching libuv...')
                for patch_file in self.libuv_patches:
                    exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        def build_libuv():
            cflags = '-fPIC'
            env = os.environ.copy()
            env['CFLAGS'] = ' '.join(x for x in (cflags, env.get('CFLAGS', None)) if x)
            log.info('Building libuv...')
            if win32_msvc:
                prepare_windows_env(env)
                libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
                exec_process(['cmd.exe', '/C', 'vcbuild.bat', libuv_arch, 'release'], cwd=self.libuv_dir, env=env, shell=True)
            else:
                exec_process(['make', 'libuv.a'], cwd=self.libuv_dir, env=env)
        if self.libuv_force_fetch:
            rmtree('deps')
        if not os.path.exists(self.libuv_dir):
            try:
                download_libuv()
            except BaseException:
                rmtree('deps')
                raise
            patch_libuv()
            build_libuv()
        else:
            if self.libuv_clean_compile:
                if win32_msvc:
                    env = os.environ.copy()
                    prepare_windows_env(env)
                    exec_process(['cmd.exe', '/C', 'vcbuild.bat', 'clean'], cwd=self.libuv_dir, env=env, shell=True)
                    rmtree(os.path.join(self.libuv_dir, 'Release'))
                else:
                    exec_process(['make', 'clean'], cwd=self.libuv_dir)
            if not os.path.exists(self.libuv_lib):
                log.info('libuv needs to be compiled.')
                build_libuv()
            else:
                log.info('No need to build libuv.')

class libuv_sdist(sdist):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = libuv_build_ext.libuv_repo
    libuv_branch   = libuv_build_ext.libuv_branch
    libuv_revision = libuv_build_ext.libuv_revision
    libuv_patches  = libuv_build_ext.libuv_patches

    def initialize_options(self):
        sdist.initialize_options(self)
        rmtree('deps')
        makedirs(self.libuv_dir)
        log.info('Downloading libuv...')
        exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
        exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        if self.libuv_patches:
            log.info('Patching libuv...')
            for patch_file in self.libuv_patches:
                exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        rmtree(os.path.join(self.libuv_dir, '.git'))

<commit_msg>Fix miscalculation of struct sizes in Windows<commit_after>
import ast
import errno
import os
import platform
import shutil
import subprocess
import sys

from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.sdist import sdist
from distutils.errors import DistutilsError

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno==errno.EEXIST and os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK):
            return
        raise

def rmtree(path):
    try:
        shutil.rmtree(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def exec_process(cmdline, silent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)
        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.decode("utf-8").rstrip("\n")))
    return stdout

def prepare_windows_env(env):
    if sys.version_info < (3, 3):
        env.pop('VS100COMNTOOLS', None)

    if env.get('PYTHON'):
        return  # Already manually set by user.

    if (2, 6) <= sys.version_info[:2] <= (2, 7):
        env['PYTHON'] = sys.executable
        return  # The current executable is fine.

    # Try if `python` on PATH is the right one. If we would execute
    # `python` directly the current executable might be used so we
    # delegate this to cmd.
    cmd = ['cmd.exe', '/C',  'python', '-c', 'import sys; '
           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
           'sys.stdout.flush()']
    try:
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        stdout, _ = sub.communicate()
        version = ast.literal_eval(stdout.decode('utf-8').strip())
        if (2, 6) <= version <= (2, 7):
            return  # Python on PATH is fine
    except OSError:
        pass

    # Check default install locations
    for v in ('26', '27'):
        path = os.path.join('%SYSTEMDRIVE%', 'Python%s' % v, 'python.exe')
        path = os.path.expandvars(path)
        if os.path.isfile(path):
            log.info('Using "%s" to build libuv...' % path)
            env['PYTHON'] = path
            return

    raise DistutilsError('No appropriate Python version found. An '
                         'installation of 2.6 or 2.7 is required to '
                         'build libuv. You can set the environment '
                         'variable "PYTHON" to point to a custom '
                         'installation location.')

class libuv_build_ext(build_ext):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = 'https://github.com/joyent/libuv.git'
    libuv_branch   = 'master'
    libuv_revision = '2f84a57'
    libuv_patches  = []

    user_options = build_ext.user_options
    user_options.extend([
        ("libuv-clean-compile", None, "Clean libuv tree before compilation"),
        ("libuv-force-fetch", None, "Remove libuv (if present) and fetch it again")
    ])
    boolean_options = build_ext.boolean_options
    boolean_options.extend(["libuv-clean-compile", "libuv-force-fetch"])

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.libuv_clean_compile = 0
        self.libuv_force_fetch = 0

    def build_extensions(self):
        if self.compiler.compiler_type == 'mingw32':
            # Dirty hack to avoid linking with more than one C runtime when using MinGW
            # Note that this hack forces the compilation to use the old MSVCT
            self.compiler.dll_libraries = [lib for lib in self.compiler.dll_libraries if not lib.startswith('msvcr')]
        self.force = self.libuv_force_fetch or self.libuv_clean_compile
        if self.compiler.compiler_type == 'msvc':
            self.libuv_lib = os.path.join(self.libuv_dir, 'Release', 'lib', 'libuv.lib')
        else:
            self.libuv_lib = os.path.join(self.libuv_dir, 'libuv.a')
        self.get_libuv()
        # Set compiler options
        if self.compiler.compiler_type == 'mingw32':
            self.compiler.add_library_dir(self.libuv_dir)
            self.compiler.add_library('uv')
        else:
            self.extensions[0].extra_objects.extend([self.libuv_lib])
        self.compiler.add_include_dir(os.path.join(self.libuv_dir, 'include'))
        if sys.platform.startswith('linux'):
            self.compiler.add_library('rt')
        elif sys.platform == 'darwin':
            self.extensions[0].extra_link_args.extend(['-framework', 'CoreServices'])
        elif sys.platform == 'win32':
            self.extensions[0].define_macros.append(('WIN32', 1))
            if self.compiler.compiler_type == 'msvc':
                self.extensions[0].extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
                self.compiler.add_library('advapi32')
            self.compiler.add_library('iphlpapi')
            self.compiler.add_library('psapi')
            self.compiler.add_library('ws2_32')
        build_ext.build_extensions(self)

    def get_libuv(self):
        #self.debug_mode =  bool(self.debug) or hasattr(sys, 'gettotalrefcount')
        win32_msvc = self.compiler.compiler_type=='msvc'
        def download_libuv():
            log.info('Downloading libuv...')
            makedirs(self.libuv_dir)
            exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
            exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        def patch_libuv():
            if self.libuv_patches:
                log.info('Patching libuv...')
                for patch_file in self.libuv_patches:
                    exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        def build_libuv():
            cflags = '-fPIC'
            env = os.environ.copy()
            env['CFLAGS'] = ' '.join(x for x in (cflags, env.get('CFLAGS', None)) if x)
            log.info('Building libuv...')
            if win32_msvc:
                prepare_windows_env(env)
                libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
                exec_process(['cmd.exe', '/C', 'vcbuild.bat', libuv_arch, 'release'], cwd=self.libuv_dir, env=env, shell=True)
            else:
                exec_process(['make', 'libuv.a'], cwd=self.libuv_dir, env=env)
        if self.libuv_force_fetch:
            rmtree('deps')
        if not os.path.exists(self.libuv_dir):
            try:
                download_libuv()
            except BaseException:
                rmtree('deps')
                raise
            patch_libuv()
            build_libuv()
        else:
            if self.libuv_clean_compile:
                if win32_msvc:
                    env = os.environ.copy()
                    prepare_windows_env(env)
                    exec_process(['cmd.exe', '/C', 'vcbuild.bat', 'clean'], cwd=self.libuv_dir, env=env, shell=True)
                    rmtree(os.path.join(self.libuv_dir, 'Release'))
                else:
                    exec_process(['make', 'clean'], cwd=self.libuv_dir)
            if not os.path.exists(self.libuv_lib):
                log.info('libuv needs to be compiled.')
                build_libuv()
            else:
                log.info('No need to build libuv.')

class libuv_sdist(sdist):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = libuv_build_ext.libuv_repo
    libuv_branch   = libuv_build_ext.libuv_branch
    libuv_revision = libuv_build_ext.libuv_revision
    libuv_patches  = libuv_build_ext.libuv_patches

    def initialize_options(self):
        sdist.initialize_options(self)
        rmtree('deps')
        makedirs(self.libuv_dir)
        log.info('Downloading libuv...')
        exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
        exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        if self.libuv_patches:
            log.info('Patching libuv...')
            for patch_file in self.libuv_patches:
                exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        rmtree(os.path.join(self.libuv_dir, '.git'))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Option persistence."""
import re
import collections
import numpy as np

from .. import logger
from conf import Conf
import cPickle as pickle
from ..milang import Scriptable
import common_proxy
from .option import ao

def dictRecursiveModel(base):
    """BUild a dictionary configuration tree from ConfigurationProxy `base`"""
    out = {}
    for path, obj in base.iteritems():
        if path == 'self':
            out[path] = obj['name']
            continue
        out[path] = dictRecursiveModel(obj)
    return out

def print_tree(tree, level=0):
    """Pretty-print a dictionary configuration `tree`"""
    pre = '   ' * level
    msg = ''
    for k, v in tree.iteritems():
        if k == 'self':
            msg += print_tree(v, level)
            continue
        # Detect subdevice
        if isinstance(v, dict) and 'self' in v:
            msg += pre + '|++> ' + k + '\n'
            msg += print_tree(v, level + 1)
            continue
        v = repr(v['current'])
        if len(v) > 50:
            v = v[:46] + ' ...'
        msg += '{}|: {} = {}\n'.format(pre, k, v)
    return msg

# TODO: parametrize in user conf
# 0=always visible; 1=user ; 2=expert ; 3=advanced ; 4=technician ;
# 5=developer; 6=never visible
class ConfigurationProxy(Scriptable, Conf):

    """A configuration object behaving like a live server"""
    separator = '/'
    _readLevel = 5
    _writeLevel = 5
    _rmodel = False

    def print_tree(self):
        print print_tree(self.tree())

    def __init__(self, desc=collections.OrderedDict({'self': {}}),
                 name='MAINSERVER', parent=False, readLevel=2, writeLevel=5, kid_base='/'):
        Scriptable.__init__(self)
        self.log = logger.BaseLogger()
        self.kid_base = kid_base
        Conf.__init__(self, desc['self'])
        self._readLevel = readLevel
        self._writeLevel = writeLevel
        self.children = desc.copy()
        """Child configuration dictionaries"""
        self.children_obj = {}
        """Instantiated children configuration proxies"""
        # Remove myself from children...
        del self.children['self']
        self.name = name
        self._parent = parent
        if not parent:
            self._Method__name = name
        else:
            self._Method__name = parent._Method__name + self.separator + name
#           if parent._parent:
#               self._Method__name=parent._Method__name+self.separator+name
#           else:
#               self._Method__name=name
        if self.has_key('devpath'):
            self['devpath'] = name
        self.autosort()

    def get(self, *a, **k):
        return self.__getitem__(*a, **k)

    def set(self, *a, **k):
        return self.__setitem__(*a, **k)

    @property
    def root(self):
        if not self._parent:
            return self
        return self._parent.root

    @property
    def naturalName(self):
        return self.name

    def get_fullpath(self):
        po = self
        path = []
        while True:
            if po.name == 'MAINSERVER':
                break
            path.append(po.name)
            po = po.parent()
            if po in [None, False]:
                break
        # Revert to get the top-down path
        path.reverse()
        return '/' + '/'.join(path) + '/'

    def _update_from_children(self):
        """ Retrieve current configuration from instantiated child objects"""
        for key, obj in self.children_obj.iteritems():
            d = {'self': obj.desc}
            obj._update_from_children()
            d.update(obj.children)
            self.children[key] = d

    def rmodel(self):
        out = dictRecursiveModel(self.children)
        out['self'] = self.name
        return out

    def tree(self):
        """Build a full configuration tree suitable for serialization and saving on a file"""
        self._update_from_children()
        tree = self.children.copy()
        tree['self'] = self.desc.copy()
        return tree

    def __len__(self):
        return len(self.desc)

    def iteritems(self):
        return self.desc.iteritems()

    def itervalues(self):
        return self.desc.itervalues()

    def values(self):
        return self.desc.values()

    def iterkeys(self):
        return self.desc.iterkeys()

    def has_key(self, k):
        return self.desc.has_key(k)

    def dumps(self):
        d = {'self': self.desc.copy()}
        self._update_from_children()
        d.update(self.children)
        return pickle.dumps(d)

    def describe(self, *a, **kw):
        return self.desc

    def connect(self, *a, **k):
        pass

    def connection(self, *a, **k):
        return True

    def paste(self, obj):
        self.desc = obj.desc.copy()
        self.children = obj.children.copy()
        self.children_obj = obj.children_obj
        self._parent = obj._parent
        self._Method__name = obj._Method__name
        self._readLevel = obj._readLevel
        self._writeLevel = obj._writeLevel

    def copy(self):
        p = ConfigurationProxy()
        p.paste(self)
        return p

    def __nonzero__(self):
        return 1

    def __getitem__(self, key, *a):
        if key == 'fullpath':
            return self.get_fullpath()
        if len(a)==1 and not self.desc.has_key(key):
            return a[0]
        return self.desc[key]['current']

    def __setitem__(self, key, val):
        if not self.desc.has_key(key):
            print 'Impossible to set key', key, val
            return False
        self.desc[key]['current'] = val
        return True

    def gettype(self, key):
        return self.desc[key]['type']

    def multiget(self, keys):
        r = {}
        for k in keys:
            r[k] = self[k]
        return r

    def __getattr__(self, path):
        if path.startswith('_'):
            return object.__getattribute__(self, path)
        elif path in dir(self):
            return object.__getattribute__(self, path)
        return self.child(path)

    def gete(self, key):
        return self.desc[key]

    def sete(self, key, val):
        if not val.has_key('priority') or val['priority']==-1:
            priorities = [opt.get('priority',0) for opt in self.desc.values()]
            if len(priorities):
                val['priority'] = max(priorities) + 1
        self.desc[key] = val

    def add_option(self, *args, **kwargs):
        """Creates a new option using the ao() utility function.
        Migrate old one if existing."""
        out = {}
        overwrite = True
        if kwargs.has_key('overwrite'):
            overwrite = kwargs.pop('overwrite')
        ao(out, *args, **kwargs)
        out = out.values()[0]
        key = out['handle']
        if out['priority'] == 0:
            out['priority'] = -1
        # If option was already defined, update old one with new values
        if self.has_key(key):
            # Do not do anything if not overwriting
            if not overwrite:
                return out
            origin = self.gete(key).entry
            origin.update(out)
            out = origin

        self.sete(out['handle'], out)
        return out

    def autosort(self):
        def sorter(item):
            key, val = item
            m = [int(s) for s in key.split() if s.isdigit()]
            if not len(m):
                return key
            return m[-1]
        self.children = collections.OrderedDict(sorted(self.children.items(), key=sorter))

    def add_child(self, name, desc, overwrite=False):
        """Inserts a sub-object `name` with object tree dictionary `desc`.
        Returns a ConfigurationProxy to the new child."""
        # If desc is the description dictionary and not the object tree dictionary,
        # encapsulate it in a 'self' dict
        if not 'self' in desc:
            desc = {'self': desc}
        if overwrite or name not in self.children:
            self.children[name] = desc
        else:
            for key, val in desc['self'].iteritems():
                self.children[name]['self'][key] = val
        self.autosort()
        return self.child(name)

    def getFlags(self, opt):
        if not self.desc[opt].has_key('flags'):
            return {}
        return self.desc[opt]['flags']

    def list(self):
        return [(c, c) for c in self.children.iterkeys()]

    def has_child(self, name):
        """Returns if `name` is a child"""
        return self.children.has_key(name)

    def child(self, name):
        """Return child ConfigurationProxy object by `name`"""
        if not self.children.has_key(name):
            return None
        if not self.children_obj.has_key(name):
            kb = self.kid_base + name + self.separator
            self.children_obj[name] = ConfigurationProxy(
                self.children[name], name=name, parent=self, kid_base=kb)
        return self.children_obj[name]

    def calc_aggregate(self, aggregation, handle=False):
        function_name = re.search("(.+?)\(", aggregation).group(1)
        targets = re.search("\((.+?)\)", aggregation)
        if targets is None:
            targets = [handle]
        else:
            targets = targets.group(1).replace(' ','').split(',')
        values = collections.defaultdict(list)
        for child_name in self.children.iterkeys():
            child = self.child(child_name)
            pack = collections.defaultdict(list)
            for target in targets:
                # Ensure all targets exist
                if not child.has_key(target):
                    pack = False
                    break
                pack[target].append(child[target])
            if pack:
                for t in targets:
                    values[t]+=pack[t]
        result = None
        error = None
        #TODO: calc stdev here
        if function_name == 'mean':
            v = np.array(values[targets[0]]).astype(np.float32)
            result = float(v.mean())
            error = float(v.std())
        elif function_name == 'sum':
            result = float(np.array(values[targets[0]]).astype(np.float32).sum())
        elif function_name == 'prod':
            result = float(np.array(values[targets[0]]).astype(np.float32).prod())
        elif function_name == 'table':
            # Should calculate also table header?
            result = [self[handle][0]]
            for i, x in enumerate(values[targets[0]]):
                row = []
                for t in targets:
                    row.append(values[t][i])
                result.append(row)
        else:
            self.log.error('Aggregate function not found:', function_name, aggregation)
        return result, error

    def update_aggregates(self, recursive=1):
        """Updates aggregate options. recursive==1, upward; -1, downward"""
        for handle, opt in self.desc.iteritems():
            if not opt.has_key('aggregate'):
                continue
            aggregation = opt['aggregate']
            result, error = self.calc_aggregate(aggregation, handle)
            if result is not None:
                self[handle] = result
                if error is not None and opt.has_key('error'):
                    self[opt['error']] = error
            else:
                self.log.error('Aggregation failed for ', handle, aggregation)
        if recursive>0 and self.parent():
            self.parent().update_aggregates(recursive=True)
        elif recursive<0:
            for k in self.children.keys():
                self.child(k).update_aggregates(recursive=-1)

    @property
    def devices(self):
        return [self.child(name) for name in self.children.keys()]

    def parent(self):
        return self._parent

    def toPath(self, lst):
        """Returns a copy of the object at the path expressed in list lst"""
        if isinstance(lst, str):
            lst = lst.split(self.separator)
            if lst[0] in ['server', '']:
                lst.pop(0)
            if lst[-1] == '':
                lst.pop(-1)
        obj = self.copy()
        for p in lst:
            if obj is None:
                return None
            obj = obj.child(p)
        return obj

    def from_column(self, col0):
        return common_proxy.from_column(col0, self.root)

    def toMethodName(self, name):
        if name == 'MAINSERVER':
            return
        lst = name.split(self.separator)
        return self.toPath(lst)

    def searchPath(self, path):
        """Search a child with the corresponding devpath=`path` and returns its name"""
        fp = self.get_fullpath().replace('//', '/')
        if not path.startswith(fp):
            print 'searchPath: not corresponding!', path, fp
            return False
        # Cut itself from the path
        if not path.endswith('/'):
            path += '/'
        vpath = path[len(fp):].split('/')[:-1]

        # Verify existence
        obj = self
        for p in vpath:
            obj = obj.child(p)
            if obj is None:
                print 'searchPath: not found', repr(p), repr(path)
        return self.separator.join(vpath)

    def listPresets(self):
        return ['default']

    def iterprint(self, base=False, pre=''):
        """Iterative printing for debug purposes"""
        if not base:
            base = self
        for k, v in base.iteritems():
            print pre, k, v['current']
        for sub in base.children.iterkeys():
            base.child(sub).iterprint(pre=pre + '\t%s: ' % sub)

    @property
    def samples(self):
        if not self.measure.has_key('nSamples'):
            print 'no nSample option!'
            return []
        n = self.measure['nSamples']
        if n == 0:
            print 'no samples defined!'
            return []
        out = []
        for i in range(n):
            # Search direct child (instrument)
            child = self.child('sample{}'.format(i))
            if child:
                out.append(child)
                continue
            # Search referred sample (from device)
            s = 'smp{}'.format(i)
            if not self.has_key(s):
                print 'sample not found', s
                break
            # Get fullpath of the referred sample object
            s = self[s][0]
            # Get the actual object
            obj = self.root.toPath(s)
            if obj is None:
                print 'sample object not found', s
                continue
            out.append(obj)
        print 'returning samples', out
        return out

    def role2dev(self, opt):
        """Return the device object associated with role option `opt`"""
        p = self[opt]
        if p is False:
            return False
        p = p[0]
        if p in ('None', None):
            return False
        obj = self.root.toPath(p)
        if not obj:
            return False
        return obj

    def check_read(self, opt):
        return self.desc[opt].get('readLevel', 0) <= self._readLevel

    def check_write(self, opt):
        return self.desc[opt].get('writeLevel', 0) <= self._writeLevel
<commit_msg>Filter out zeros while calc mean/sum/prod/stdev, FLTD-201<commit_after># -*- coding: utf-8 -*-
"""Option persistence."""
import re
import collections
import numpy as np

from .. import logger
from conf import Conf
import cPickle as pickle
from ..milang import Scriptable
import common_proxy
from .option import ao

def dictRecursiveModel(base):
    """BUild a dictionary configuration tree from ConfigurationProxy `base`"""
    out = {}
    for path, obj in base.iteritems():
        if path == 'self':
            out[path] = obj['name']
            continue
        out[path] = dictRecursiveModel(obj)
    return out

def print_tree(tree, level=0):
    """Pretty-print a dictionary configuration `tree`"""
    pre = '   ' * level
    msg = ''
    for k, v in tree.iteritems():
        if k == 'self':
            msg += print_tree(v, level)
            continue
        # Detect subdevice
        if isinstance(v, dict) and 'self' in v:
            msg += pre + '|++> ' + k + '\n'
            msg += print_tree(v, level + 1)
            continue
        v = repr(v['current'])
        if len(v) > 50:
            v = v[:46] + ' ...'
        msg += '{}|: {} = {}\n'.format(pre, k, v)
    return msg

# TODO: parametrize in user conf
# 0=always visible; 1=user ; 2=expert ; 3=advanced ; 4=technician ;
# 5=developer; 6=never visible
class ConfigurationProxy(Scriptable, Conf):

    """A configuration object behaving like a live server"""
    separator = '/'
    _readLevel = 5
    _writeLevel = 5
    _rmodel = False

    def print_tree(self):
        print print_tree(self.tree())

    def __init__(self, desc=collections.OrderedDict({'self': {}}),
                 name='MAINSERVER', parent=False, readLevel=2, writeLevel=5, kid_base='/'):
        Scriptable.__init__(self)
        self.log = logger.BaseLogger()
        self.kid_base = kid_base
        Conf.__init__(self, desc['self'])
        self._readLevel = readLevel
        self._writeLevel = writeLevel
        self.children = desc.copy()
        """Child configuration dictionaries"""
        self.children_obj = {}
        """Instantiated children configuration proxies"""
        # Remove myself from children...
        del self.children['self']
        self.name = name
        self._parent = parent
        if not parent:
            self._Method__name = name
        else:
            self._Method__name = parent._Method__name + self.separator + name
#           if parent._parent:
#               self._Method__name=parent._Method__name+self.separator+name
#           else:
#               self._Method__name=name
        if self.has_key('devpath'):
            self['devpath'] = name
        self.autosort()

    def get(self, *a, **k):
        return self.__getitem__(*a, **k)

    def set(self, *a, **k):
        return self.__setitem__(*a, **k)

    @property
    def root(self):
        if not self._parent:
            return self
        return self._parent.root

    @property
    def naturalName(self):
        return self.name

    def get_fullpath(self):
        po = self
        path = []
        while True:
            if po.name == 'MAINSERVER':
                break
            path.append(po.name)
            po = po.parent()
            if po in [None, False]:
                break
        # Revert to get the top-down path
        path.reverse()
        return '/' + '/'.join(path) + '/'

    def _update_from_children(self):
        """ Retrieve current configuration from instantiated child objects"""
        for key, obj in self.children_obj.iteritems():
            d = {'self': obj.desc}
            obj._update_from_children()
            d.update(obj.children)
            self.children[key] = d

    def rmodel(self):
        out = dictRecursiveModel(self.children)
        out['self'] = self.name
        return out

    def tree(self):
        """Build a full configuration tree suitable for serialization and saving on a file"""
        self._update_from_children()
        tree = self.children.copy()
        tree['self'] = self.desc.copy()
        return tree

    def __len__(self):
        return len(self.desc)

    def iteritems(self):
        return self.desc.iteritems()

    def itervalues(self):
        return self.desc.itervalues()

    def values(self):
        return self.desc.values()

    def iterkeys(self):
        return self.desc.iterkeys()

    def has_key(self, k):
        return self.desc.has_key(k)

    def dumps(self):
        d = {'self': self.desc.copy()}
        self._update_from_children()
        d.update(self.children)
        return pickle.dumps(d)

    def describe(self, *a, **kw):
        return self.desc

    def connect(self, *a, **k):
        pass

    def connection(self, *a, **k):
        return True

    def paste(self, obj):
        self.desc = obj.desc.copy()
        self.children = obj.children.copy()
        self.children_obj = obj.children_obj
        self._parent = obj._parent
        self._Method__name = obj._Method__name
        self._readLevel = obj._readLevel
        self._writeLevel = obj._writeLevel

    def copy(self):
        p = ConfigurationProxy()
        p.paste(self)
        return p

    def __nonzero__(self):
        return 1

    def __getitem__(self, key, *a):
        if key == 'fullpath':
            return self.get_fullpath()
        if len(a)==1 and not self.desc.has_key(key):
            return a[0]
        return self.desc[key]['current']

    def __setitem__(self, key, val):
        if not self.desc.has_key(key):
            print 'Impossible to set key', key, val
            return False
        self.desc[key]['current'] = val
        return True

    def gettype(self, key):
        return self.desc[key]['type']

    def multiget(self, keys):
        r = {}
        for k in keys:
            r[k] = self[k]
        return r

    def __getattr__(self, path):
        if path.startswith('_'):
            return object.__getattribute__(self, path)
        elif path in dir(self):
            return object.__getattribute__(self, path)
        return self.child(path)

    def gete(self, key):
        return self.desc[key]

    def sete(self, key, val):
        if not val.has_key('priority') or val['priority']==-1:
            priorities = [opt.get('priority',0) for opt in self.desc.values()]
            if len(priorities):
                val['priority'] = max(priorities) + 1
        self.desc[key] = val

    def add_option(self, *args, **kwargs):
        """Creates a new option using the ao() utility function.
        Migrate old one if existing."""
        out = {}
        overwrite = True
        if kwargs.has_key('overwrite'):
            overwrite = kwargs.pop('overwrite')
        ao(out, *args, **kwargs)
        out = out.values()[0]
        key = out['handle']
        if out['priority'] == 0:
            out['priority'] = -1
        # If option was already defined, update old one with new values
        if self.has_key(key):
            # Do not do anything if not overwriting
            if not overwrite:
                return out
            origin = self.gete(key).entry
            origin.update(out)
            out = origin

        self.sete(out['handle'], out)
        return out

    def autosort(self):
        def sorter(item):
            key, val = item
            m = [int(s) for s in key.split() if s.isdigit()]
            if not len(m):
                return key
            return m[-1]
        self.children = collections.OrderedDict(sorted(self.children.items(), key=sorter))

    def add_child(self, name, desc, overwrite=False):
        """Inserts a sub-object `name` with object tree dictionary `desc`.
        Returns a ConfigurationProxy to the new child."""
        # If desc is the description dictionary and not the object tree dictionary,
        # encapsulate it in a 'self' dict
        if not 'self' in desc:
            desc = {'self': desc}
        if overwrite or name not in self.children:
            self.children[name] = desc
        else:
            for key, val in desc['self'].iteritems():
                self.children[name]['self'][key] = val
        self.autosort()
        return self.child(name)

    def getFlags(self, opt):
        if not self.desc[opt].has_key('flags'):
            return {}
        return self.desc[opt]['flags']

    def list(self):
        return [(c, c) for c in self.children.iterkeys()]

    def has_child(self, name):
        """Returns if `name` is a child"""
        return self.children.has_key(name)

    def child(self, name):
        """Return child ConfigurationProxy object by `name`"""
        if not self.children.has_key(name):
            return None
        if not self.children_obj.has_key(name):
            kb = self.kid_base + name + self.separator
            self.children_obj[name] = ConfigurationProxy(
                self.children[name], name=name, parent=self, kid_base=kb)
        return self.children_obj[name]

    def calc_aggregate(self, aggregation, handle=False):
        function_name = re.search("(.+?)\(", aggregation).group(1)
        targets = re.search("\((.+?)\)", aggregation)
        if targets is None:
            targets = [handle]
        else:
            targets = targets.group(1).replace(' ','').split(',')
        values = collections.defaultdict(list)
        for child_name in self.children.iterkeys():
            child = self.child(child_name)
            pack = collections.defaultdict(list)
            for target in targets:
                # Ensure all targets exist
                if not child.has_key(target):
                    pack = False
                    break
                pack[target].append(child[target])
            if pack:
                for t in targets:
                    values[t]+=pack[t]
        result = None
        error = None
        #TODO: calc stdev here
        if function_name == 'mean':
            v = np.array(values[targets[0]]).astype(np.float32)
            #FIXME: hack to filter out zeros
            v = v[v!=0]
            result = float(v.mean())
            error = float(v.std())
        elif function_name == 'sum':
            result = float(np.array(values[targets[0]]).astype(np.float32).sum())
        elif function_name == 'prod':
            result = float(np.array(values[targets[0]]).astype(np.float32).prod())
        elif function_name == 'table':
            # Should calculate also table header?
            result = [self[handle][0]]
            for i, x in enumerate(values[targets[0]]):
                row = []
                for t in targets:
                    row.append(values[t][i])
                result.append(row)
        else:
            self.log.error('Aggregate function not found:', function_name, aggregation)
        return result, error

    def update_aggregates(self, recursive=1):
        """Updates aggregate options. recursive==1, upward; -1, downward"""
        for handle, opt in self.desc.iteritems():
            if not opt.has_key('aggregate'):
                continue
            aggregation = opt['aggregate']
            result, error = self.calc_aggregate(aggregation, handle)
            if result is not None:
                self[handle] = result
                if error is not None and opt.has_key('error'):
                    self[opt['error']] = error
            else:
                self.log.error('Aggregation failed for ', handle, aggregation)
        if recursive>0 and self.parent():
            self.parent().update_aggregates(recursive=True)
        elif recursive<0:
            for k in self.children.keys():
                self.child(k).update_aggregates(recursive=-1)

    @property
    def devices(self):
        return [self.child(name) for name in self.children.keys()]

    def parent(self):
        return self._parent

    def toPath(self, lst):
        """Returns a copy of the object at the path expressed in list lst"""
        if isinstance(lst, str):
            lst = lst.split(self.separator)
            if lst[0] in ['server', '']:
                lst.pop(0)
            if lst[-1] == '':
                lst.pop(-1)
        obj = self.copy()
        for p in lst:
            if obj is None:
                return None
            obj = obj.child(p)
        return obj

    def from_column(self, col0):
        return common_proxy.from_column(col0, self.root)

    def toMethodName(self, name):
        if name == 'MAINSERVER':
            return
        lst = name.split(self.separator)
        return self.toPath(lst)

    def searchPath(self, path):
        """Search a child with the corresponding devpath=`path` and returns its name"""
        fp = self.get_fullpath().replace('//', '/')
        if not path.startswith(fp):
            print 'searchPath: not corresponding!', path, fp
            return False
        # Cut itself from the path
        if not path.endswith('/'):
            path += '/'
        vpath = path[len(fp):].split('/')[:-1]

        # Verify existence
        obj = self
        for p in vpath:
            obj = obj.child(p)
            if obj is None:
                print 'searchPath: not found', repr(p), repr(path)
        return self.separator.join(vpath)

    def listPresets(self):
        return ['default']

    def iterprint(self, base=False, pre=''):
        """Iterative printing for debug purposes"""
        if not base:
            base = self
        for k, v in base.iteritems():
            print pre, k, v['current']
        for sub in base.children.iterkeys():
            base.child(sub).iterprint(pre=pre + '\t%s: ' % sub)

    @property
    def samples(self):
        if not self.measure.has_key('nSamples'):
            print 'no nSample option!'
            return []
        n = self.measure['nSamples']
        if n == 0:
            print 'no samples defined!'
            return []
        out = []
        for i in range(n):
            # Search direct child (instrument)
            child = self.child('sample{}'.format(i))
            if child:
                out.append(child)
                continue
            # Search referred sample (from device)
            s = 'smp{}'.format(i)
            if not self.has_key(s):
                print 'sample not found', s
                break
            # Get fullpath of the referred sample object
            s = self[s][0]
            # Get the actual object
            obj = self.root.toPath(s)
            if obj is None:
                print 'sample object not found', s
                continue
            out.append(obj)
        print 'returning samples', out
        return out

    def role2dev(self, opt):
        """Return the device object associated with role option `opt`"""
        p = self[opt]
        if p is False:
            return False
        p = p[0]
        if p in ('None', None):
            return False
        obj = self.root.toPath(p)
        if not obj:
            return False
        return obj

    def check_read(self, opt):
        return self.desc[opt].get('readLevel', 0) <= self._readLevel

    def check_write(self, opt):
        return self.desc[opt].get('writeLevel', 0) <= self._writeLevel
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from __future__ import print_function

import os
import subprocess
import sys

# Root of the Git repository
ROOT = subprocess.check_output([
    'git', 'rev-parse', '--show-toplevel']).decode('ascii').strip()

def fprint(*args, **kwargs):
    if kwargs.get('file') == sys.stderr:
        kwargs['file'] = sys.stderr.flush()
    else:
        kwargs['file'] = sys.stdout.flush()
    print(*args, **kwargs)

def changed_files(*args):
    """
    Returns a set of changed files in a given commit range.

    :param commit_range: Arguments to pass to ``git diff``.
    """
    files = set()
    command = ['git', 'diff', '--name-only'] + list(args)
    diff_output = subprocess.check_output(command).decode('ascii')
    for line in diff_output.splitlines():
        filepath = line.strip()
        if filepath:
            files.add(filepath)
    return files

def are_there_job_relevant_changes(changed_files, task):
    """
    Given a list of changed files and a Make task, are there any reasons
    why the output of this task might have changed?

    e.g. if the loris/ directory changes, we should re-run tasks associated
         with Loris.

    Returns a list of reasons why the task should be run.
    """
    reasons = []

    if task == 'check-format':
        reasons.append('Linting/formatting tasks always run')

    if 'Makefile' in changed_files:
        reasons.append('Changes to the Makefile')

    if any(f.startswith('.travis') for f in changed_files):
        reasons.append('Changes to the Travis config')

    if any(f.startswith('builds/') for f in changed_files):
        reasons.append('Changes to the build scripts')

    if task.startswith('sbt-'):
        reasons.extend(_are_there_sbt_relevant_changes(changed_files, task))

    for project in os.listdir(ROOT):
        if task.startswith(project):
            if any(f.startswith('%s/' % project) for f in changed_files):
                reasons.append('Changes to %s/' % project)

    return reasons

def _are_there_sbt_relevant_changes(changed_files, task):
    """
    Return a list of reasons we might want to re-run an sbt task.
    """
    reasons = []

    if 'build.sbt' in changed_files:
        reasons.append('Changes to build.sbt')

    sbt_project = task.split('-')[-1]
    for dirname in ['common', 'project', sbt_project]:
        if any(f.startswith('%s/' % dirname) for f in changed_files):
            reasons.append('Changes to %s/' % dirname)

    return reasons

def _are_there_docker_relevant_changes(changed_files, task):
    """
    Return a list of reasons we might want to re-run a task in the
    docker directory.
    """
    image = task.split('-')[0]
    if any(f.startswith('docker/%s/' % image) for f in changed_files):
        return ['Changes to docker/%s' % image]
    else:
        return []

def make_decision(changed_files, task, action):
    reasons = are_there_job_relevant_changes(
        changed_files=changed_files, task=task
    )
    if reasons:
        print('*** Reasons to %s for this change:' % action)
        for r in reasons:
            print('***   - %s' % r)
        return True
    else:
        print('*** No reasons to %s for this change!' % action)
        return False

def make(task):
    subprocess.check_call(['make', task])

def rreplace(string, old, new, count=None):
    """
    Replace ``old`` with ``new``, starting from the right.
    """
    # Split on the occurrences of ``old``, starting from the right
    parts = string.rsplit(old, count)

    # Then join back together with ``new``
    return new.join(parts)
<commit_msg>Travis tooling auto-detects sbt projects<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from __future__ import print_function

import os
import re
import subprocess
import sys

# Root of the Git repository
ROOT = subprocess.check_output([
    'git', 'rev-parse', '--show-toplevel']).decode('ascii').strip()

def fprint(*args, **kwargs):
    if kwargs.get('file') == sys.stderr:
        kwargs['file'] = sys.stderr.flush()
    else:
        kwargs['file'] = sys.stdout.flush()
    print(*args, **kwargs)

def changed_files(*args):
    """
    Returns a set of changed files in a given commit range.

    :param commit_range: Arguments to pass to ``git diff``.
    """
    files = set()
    command = ['git', 'diff', '--name-only'] + list(args)
    diff_output = subprocess.check_output(command).decode('ascii')
    for line in diff_output.splitlines():
        filepath = line.strip()
        if filepath:
            files.add(filepath)
    return files

def are_there_job_relevant_changes(changed_files, task):
    """
    Given a list of changed files and a Make task, are there any reasons
    why the output of this task might have changed?

    e.g. if the loris/ directory changes, we should re-run tasks associated
         with Loris.

    Returns a list of reasons why the task should be run.
    """
    reasons = []

    if task == 'check-format':
        reasons.append('Linting/formatting tasks always run')

    if 'Makefile' in changed_files:
        reasons.append('Changes to the Makefile')

    if any(f.startswith('.travis') for f in changed_files):
        reasons.append('Changes to the Travis config')

    if any(f.startswith('builds/') for f in changed_files):
        reasons.append('Changes to the build scripts')

    if any(task.startswith(p) for p in _sbt_projects()):
        reasons.extend(_are_there_sbt_relevant_changes(changed_files, task))

    for project in os.listdir(ROOT):
        if task.startswith(project):
            if any(f.startswith('%s/' % project) for f in changed_files):
                reasons.append('Changes to %s/' % project)

    return reasons

def _sbt_projects():
    """Returns a list of sbt project names."""
    for line in open(os.path.join(ROOT, 'build.sbt')):
        m = re.match(r'lazy val (?P<project>[a-z_]+)', line)
        if (m is not None) and (m.group('project') != 'root'):
            yield m.group('project')

def _are_there_sbt_relevant_changes(changed_files, task):
    """
    Return a list of reasons we might want to re-run an sbt task.
    """
    reasons = []

    if 'build.sbt' in changed_files:
        reasons.append('Changes to build.sbt')

    sbt_project = task.split('-')[-1]
    for dirname in ['common', 'project', sbt_project]:
        if any(f.startswith('%s/' % dirname) for f in changed_files):
            reasons.append('Changes to %s/' % dirname)

    return reasons

def _are_there_docker_relevant_changes(changed_files, task):
    """
    Return a list of reasons we might want to re-run a task in the
    docker directory.
    """
    image = task.split('-')[0]
    if any(f.startswith('docker/%s/' % image) for f in changed_files):
        return ['Changes to docker/%s' % image]
    else:
        return []

def make_decision(changed_files, task, action):
    reasons = are_there_job_relevant_changes(
        changed_files=changed_files, task=task
    )
    if reasons:
        print('*** Reasons to %s for this change:' % action)
        for r in reasons:
            print('***   - %s' % r)
        return True
    else:
        print('*** No reasons to %s for this change!' % action)
        return False

def make(task):
    subprocess.check_call(['make', task])

def rreplace(string, old, new, count=None):
    """
    Replace ``old`` with ``new``, starting from the right.
    """
    # Split on the occurrences of ``old``, starting from the right
    parts = string.rsplit(old, count)

    # Then join back together with ``new``
    return new.join(parts)
<|endoftext|>


<commit_before>import multiprocessing
from multiprocessing import Process
import time
import htcondor
import json
import logging
import config
import socket

from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.automap import automap_base

# condor likes to return extra keys not defined in the projection
# this function will trim the extra ones so that we can use kwargs
# to initiate a valid table row based on the data returned
def trim_keys(dict_to_trim, key_list):
    keys_to_trim = []
    for key in dict_to_trim:
        if key not in key_list:
            keys_to_trim.append(key)
    for key in keys_to_trim:
        dict_to_trim.pop(key, None)
    return dict_to_trim

def job_producer():
    multiprocessing.current_process().name = "Poller"

    sleep_interval = config.job_collection_interval
    job_attributes = ["group_name", "TargetClouds", "JobStatus", "RequestMemory", "GlobalJobId", "RequestDisk", "Requirements",
                     "JobPrio", "ClusterId", "User", "VMInstanceType", "VMNetwork", "VMImage", "VMKeepAlive", "VMMaximumPrice", "VMUserData",
                     "VMJobPerCore", "EnteredCurrentStatus", "QDate"]
    # Thus far the attributes from this list that are available in my tests are: ClusterId, RequestDisk, User,
    # Requirements, JobStatus, JobPrio, RequestMemory, Iwd, Cmd, GlobalJobId
    # Not in the list that seem to be returned always: FileSystemDomian, MyType, ServerTime, TargetType,
    last_poll_time = 0
    while(True):
        try:
            job_dict_list = []

            condor_s = htcondor.Schedd()
            new_poll_time = time.time()

            #
            # Part 1 - Get new jobs
            #
            if(last_poll_time == 0):
                #first poll since starting up, get everything
                job_list = condor_s.query(attr_list=job_attributes)
            else:
                #regular polling cycle: Get all new jobs
                    ## constraint='JobStatus=?=1 && QDate>=' + last_poll_time, attr_list=job_attributes
                job_list = condor_s.query(constraint='QDate>=%d' % last_poll_time, attr_list=job_attributes)

            #Process job data & Insert/update jobs in Database
            Base = automap_base()
            engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
            Base.prepare(engine, reflect=True)
            Job = Base.classes.condor_jobs
            session = Session(engine)
            for job_ad in job_list:
                job_dict = dict(job_ad)
                if "Requirements" in job_dict:
                    job_dict['Requirements'] = str(job_dict['Requirements'])
                job_dict = trim_keys(job_dict, job_attributes)
                logging.info("Adding job %s" % job_dict["GlobalJobId"])
                new_job = Job(**job_dict)
                session.merge(new_job)
            session.commit()

            session = Session(engine)
            #
            # Part 2 - Detect any job status changes
            #
            if(last_poll_time != 0):
                # get all jobs who've had status changes since last poll excluding brand new jobs since they would have been updated above
                status_changed_job_list = condor_s.query(constraint='EnteredCurrentStatus>=%d && QDate<=%d' % (last_poll_time, last_poll_time), attr_list=job_attributes)

                for job_ad in status_changed_job_list:
                    job_dict = dict(job_ad)
                    if "Requirements" in job_dict:
                        job_dict['Requirements'] = str(job_dict['Requirements'])
                    job_dict = trim_keys(job_dict, job_attributes)
                    new_job = Job(**job_dict)
                    session.merge(new_job)
                session.commit()

            last_poll_time = new_poll_time

            time.sleep(sleep_interval)

        except Exception as e:
            logging.error(e)
            logging.error("Failure contacting condor...")
            time.sleep(sleep_interval)

        except(SystemExit, KeyboardInterrupt):
            return

def job_command_consumer(testrun=False):
    multiprocessing.current_process().name = "Cmd Consumer"
    sleep_interval = config.command_sleep_interval

    while(True):
        try:
            #Make database engine
            Base = automap_base()
            engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host+ ":" + str(config.db_port) + "/" + config.db_name)
            Base.prepare(engine, reflect=True)
            Job = Base.classes.condor_jobs
            session = Session(engine)
            #Query database for any entries that have a command flag
            for job in session.query(Job).filter(Job.hold_job==1):
                #execute condor hold on the jobs returned
                logging.info("Holding %s" % job.GlobalJobId)
                try:
                    s = htcondor.Schedd()
                    local_job_id = job.GlobalJobId.split('#')[1]
                    s.edit([local_job_id,], "JobStatus", "5")
                    #update job so that it is held, need to finalize encoding here.
                    job.JobStatus=5
                    job.hold_job=2 # Null/0 normal, 1 means needs to be held, 2 means job has been held
                    session.merge(job)
                except:
                    logging.error("Failed to hold job %s" % job.GlobalJobId)
                    continue
            #commit updates enteries
            session.commit()

            logging.debug("No more jobs to hold, begining sleep interval...")
            time.sleep(sleep_interval)

        except Exception as e:
            logging.error("Failure connecting to database or executing condor command, begining sleep interval...")
            logging.error(e)
            if(testrun):
                return False
            time.sleep(sleep_interval)

        except(SystemExit, KeyboardInterrupt):
            return

def cleanUp():
    multiprocessing.current_process().name = "Cleanup"
    while(True):
        # Setup condor classes and database connctions
        # this stuff may be able to be moved outside the while loop, but i think its better to re-mirror the
        # database each time for the sake on consistency.
        condor_s = htcondor.Schedd()
        Base = automap_base()
        local_hostname = socket.gethostname()
        engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
        Base.prepare(engine, reflect=True)
        session = Session(engine)
        #setup database objects
        Job = Base.classes.condor_jobs
        archJob = Base.classes.archived_condor_jobs

        # Clean up job ads
        condor_job_list = condor_s.query()
        # this query asks for only jobs that contain the local hostname as part of their JobID
        db_job_list = session.query(Job).filter(Job.GlobalJobId.like("%" + local_hostname+ "%"))
        #loop through the condor data and make a list of GlobalJobId
        #then loop through db list checking if they are in the aforementioned list
        condor_name_list = []
        for ad in condor_job_list:
            ad_dict = dict(ad)
            condor_name_list.append(ad_dict['GlobalJobId'])
        for job in db_job_list:
            if job.GlobalJobId not in condor_name_list:
                #job is missing from condor, clean it up
                logging.info("Found Job missing from condor: %s, cleaning up." % job.GlobalJobId)
                job_dict = job.__dict__
                logging.info(job_dict)
                session.delete(job)
                job_dict.pop('_sa_instance_state', None) # metadata not relevent to the job ad, must trim to init with kwargs
                new_arch_job = archJob(**job_dict)
                session.merge(new_arch_job)

        session.commit()
        time.sleep(config.cleanup_sleep_interval) #sleep 2 mins, should probably add this as a config option

if __name__ == '__main__':

    logging.basicConfig(filename=config.job_log_file,level=config.log_level, format='%(asctime)s - %(processName)-12s - %(levelname)s - %(message)s')
    processes = []
    # job polling proccess
    p_job_producer = Process(target=job_producer)
    processes.append(p_job_producer)
    # command executer proccess
    #p_command_consumer = Process(target=job_command_consumer)
    #processes.append(p_command_consumer)
    # cleanUp proccess
    p_cleanup = Process(target=cleanUp)
    processes.append(p_cleanup)

    # Wait for keyboard input to exit
    try:
        for process in processes:
            process.start()
        while(True):
            for process in processes:
                if not process.is_alive():
                    logging.error("%s process died!" % process.name)
                    logging.error("Restarting %s process...")
                    process.start()
                time.sleep(1)
            time.sleep(10)
    except (SystemExit, KeyboardInterrupt):
        logging.error("Caught KeyboardInterrupt, shutting down threads and exiting...")

    for process in processes:
        try:
            process.join()
        except:
            logging.error("failed to join process %s" % process.name)

<commit_msg>add group name logic to csjobs<commit_after>import multiprocessing
from multiprocessing import Process
import time
import htcondor
import json
import logging
import config
import socket

from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.automap import automap_base

# condor likes to return extra keys not defined in the projection
# this function will trim the extra ones so that we can use kwargs
# to initiate a valid table row based on the data returned
def trim_keys(dict_to_trim, key_list):
    keys_to_trim = []
    for key in dict_to_trim:
        if key not in key_list:
            keys_to_trim.append(key)
    for key in keys_to_trim:
        dict_to_trim.pop(key, None)
    # need to change GroupName (condor attribute) to group_name (db attribute)
    dict_to_trim["group_name"] = dict_to_trim["GroupName"]
    dict_to_trim.pop("GroupName", None)
    return dict_to_trim

def build_user_group_dict(db_list):
    user_group_dict = {}
    for entry in db_list:
        if user_group_dict[entry.username]:
            user_group_dict[entry.username].append(entry.group_name)
        else:
            user_group_dict[entry.username] = [entry.group_name]
    return user_group_dict

def job_producer():
    multiprocessing.current_process().name = "Poller"

    sleep_interval = config.job_collection_interval
    job_attributes = ["GroupName", "TargetClouds", "JobStatus", "RequestMemory", "GlobalJobId", "RequestDisk", "Requirements",
                     "JobPrio", "ClusterId", "User", "VMInstanceType", "VMNetwork", "VMImage", "VMKeepAlive", "VMMaximumPrice", "VMUserData",
                     "VMJobPerCore", "EnteredCurrentStatus", "QDate"]
    # Thus far the attributes from this list that are available in my tests are: ClusterId, RequestDisk, User,
    # Requirements, JobStatus, JobPrio, RequestMemory, Iwd, Cmd, GlobalJobId
    # Not in the list that seem to be returned always: FileSystemDomian, MyType, ServerTime, TargetType,
    last_poll_time = 0
    while(True):
        try:
            #
            # Setup - initialize condor and database objects and build user-group list
            #
            job_dict_list = []

            condor_s = htcondor.Schedd()
            Base = automap_base()
            engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
            Base.prepare(engine, reflect=True)
            Job = Base.classes.condor_jobs
            User_Groups = Base.classes.csv2_user_groups
            session = Session(engine)

            db_user_grps = session.query(User_Groups)
            user_group_dict = build_user_group_dict(db_user_grps)

            new_poll_time = time.time()

            #
            # Part 1 - Get new jobs
            #
            if(last_poll_time == 0):
                #first poll since starting up, get everything
                job_list = condor_s.query(attr_list=job_attributes)
            else:
                #regular polling cycle: Get all new jobs
                    ## constraint='JobStatus=?=1 && QDate>=' + last_poll_time, attr_list=job_attributes
                job_list = condor_s.query(constraint='QDate>=%d' % last_poll_time, attr_list=job_attributes)

            #Process job data & Insert/update jobs in Database
            for job_ad in job_list:
                job_dict = dict(job_ad)
                if "Requirements" in job_dict:
                    job_dict['Requirements'] = str(job_dict['Requirements'])

                #
                # check if there is a group_name
                # if not, try and assign one, if default is ambiguos ignore ad
                # if a group is found update job ad in condor before adding to database
                #
                if job_dict["GroupName"]:
                    # if there is a grp name check that it is a valid one.
                    if job_dict["GroupName"] not in user_group_dict[job_dict["User"]]:
                        logging.info("Job ad: %s has invalid group_name, ignoring..." % job_dict["GlobalJobId"])
                        # Invalid group name
                        # IGNORE
                        continue

                else:
                    # else if there is no group name try to assign one
                    logging.info("Job ad: %s has no group_name, attemping to resolve..." % job_dict["GlobalJobId"])
                    job_user = job_dict["User"]
                    if len(user_group_dict[job_user]) = 1:
                        job_dict["group_name"] = user_group_dict[job_user][0]
                        #UPDATE CLASSAD
                        cluster = job_dict["ClusterId"]
                        proc = job_dict["ProcId"]
                        expr = str(cluster) + "." + str(proc)
                        condor_s.edit([expr], "GroupName", user_group_dict[job_user][0])
                    else:
                        #AMBIGUOUS GROUP NAME, IGNORE
                        logging.info("Could not automatically resolve group_name for: %s, ignoring... " % job_dict["GlobalJobId"])
                        continue

                job_dict = trim_keys(job_dict, job_attributes)

                logging.info("Adding job %s" % job_dict["GlobalJobId"])
                new_job = Job(**job_dict)
                session.merge(new_job)
            session.commit()

            session = Session(engine)
            #
            # Part 2 - Detect any job status changes
            #
            if(last_poll_time != 0):
                # get all jobs who've had status changes since last poll excluding brand new jobs since they would have been updated above
                status_changed_job_list = condor_s.query(constraint='EnteredCurrentStatus>=%d && QDate<=%d' % (last_poll_time, last_poll_time), attr_list=job_attributes)

                for job_ad in status_changed_job_list:
                    job_dict = dict(job_ad)
                    if "Requirements" in job_dict:
                        job_dict['Requirements'] = str(job_dict['Requirements'])
                    job_dict = trim_keys(job_dict, job_attributes)
                    new_job = Job(**job_dict)
                    session.merge(new_job)
                session.commit()

            last_poll_time = new_poll_time

            time.sleep(sleep_interval)

        except Exception as e:
            logging.error(e)
            logging.error("Failure contacting condor...")
            time.sleep(sleep_interval)

        except(SystemExit, KeyboardInterrupt):
            return

def job_command_consumer(testrun=False):
    multiprocessing.current_process().name = "Cmd Consumer"
    sleep_interval = config.command_sleep_interval

    while(True):
        try:
            #Make database engine
            Base = automap_base()
            engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host+ ":" + str(config.db_port) + "/" + config.db_name)
            Base.prepare(engine, reflect=True)
            Job = Base.classes.condor_jobs
            session = Session(engine)
            #Query database for any entries that have a command flag
            for job in session.query(Job).filter(Job.hold_job==1):
                #execute condor hold on the jobs returned
                logging.info("Holding %s" % job.GlobalJobId)
                try:
                    s = htcondor.Schedd()
                    local_job_id = job.GlobalJobId.split('#')[1]
                    s.edit([local_job_id,], "JobStatus", "5")
                    #update job so that it is held, need to finalize encoding here.
                    job.JobStatus=5
                    job.hold_job=2 # Null/0 normal, 1 means needs to be held, 2 means job has been held
                    session.merge(job)
                except:
                    logging.error("Failed to hold job %s" % job.GlobalJobId)
                    continue
            #commit updates enteries
            session.commit()

            logging.debug("No more jobs to hold, begining sleep interval...")
            time.sleep(sleep_interval)

        except Exception as e:
            logging.error("Failure connecting to database or executing condor command, begining sleep interval...")
            logging.error(e)
            if(testrun):
                return False
            time.sleep(sleep_interval)

        except(SystemExit, KeyboardInterrupt):
            return

def cleanUp():
    multiprocessing.current_process().name = "Cleanup"
    while(True):
        # Setup condor classes and database connctions
        # this stuff may be able to be moved outside the while loop, but i think its better to re-mirror the
        # database each time for the sake on consistency.
        condor_s = htcondor.Schedd()
        Base = automap_base()
        local_hostname = socket.gethostname()
        engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
        Base.prepare(engine, reflect=True)
        session = Session(engine)
        #setup database objects
        Job = Base.classes.condor_jobs
        archJob = Base.classes.archived_condor_jobs

        # Clean up job ads
        condor_job_list = condor_s.query()
        # this query asks for only jobs that contain the local hostname as part of their JobID
        db_job_list = session.query(Job).filter(Job.GlobalJobId.like("%" + local_hostname+ "%"))
        #loop through the condor data and make a list of GlobalJobId
        #then loop through db list checking if they are in the aforementioned list
        condor_name_list = []
        for ad in condor_job_list:
            ad_dict = dict(ad)
            condor_name_list.append(ad_dict['GlobalJobId'])
        for job in db_job_list:
            if job.GlobalJobId not in condor_name_list:
                #job is missing from condor, clean it up
                logging.info("Found Job missing from condor: %s, cleaning up." % job.GlobalJobId)
                job_dict = job.__dict__
                logging.info(job_dict)
                session.delete(job)
                job_dict.pop('_sa_instance_state', None) # metadata not relevent to the job ad, must trim to init with kwargs
                new_arch_job = archJob(**job_dict)
                session.merge(new_arch_job)

        session.commit()
        time.sleep(config.cleanup_sleep_interval) #sleep 2 mins, should probably add this as a config option

if __name__ == '__main__':

    logging.basicConfig(filename=config.job_log_file,level=config.log_level, format='%(asctime)s - %(processName)-12s - %(levelname)s - %(message)s')
    processes = []
    # job polling proccess
    p_job_producer = Process(target=job_producer)
    processes.append(p_job_producer)
    # command executer proccess
    #p_command_consumer = Process(target=job_command_consumer)
    #processes.append(p_command_consumer)
    # cleanUp proccess
    p_cleanup = Process(target=cleanUp)
    processes.append(p_cleanup)

    # Wait for keyboard input to exit
    try:
        for process in processes:
            process.start()
        while(True):
            for process in processes:
                if not process.is_alive():
                    logging.error("%s process died!" % process.name)
                    logging.error("Restarting %s process...")
                    process.start()
                time.sleep(1)
            time.sleep(10)
    except (SystemExit, KeyboardInterrupt):
        logging.error("Caught KeyboardInterrupt, shutting down threads and exiting...")

    for process in processes:
        try:
            process.join()
        except:
            logging.error("failed to join process %s" % process.name)

<|endoftext|>


<commit_before>from django.core.cache import caches
from rest_framework import throttling

THROTTLE_CACHE_ALIAS = 'api_throttle'

throttle_cache = caches[THROTTLE_CACHE_ALIAS]

class UserRateThrottle(throttling.UserRateThrottle):
    cache = throttle_cache

class HostRateThrottle(throttling.SimpleRateThrottle):
    cache = throttle_cache
    scope = 'host'

    def get_cache_key(self, request, view):
        ident = self.get_ident(request)

        return self.cache_format % {
            'scope': self.scope,
            'ident': ident
        }
<commit_msg>important comment about global throttle cache<commit_after>from django.core.cache import caches
from rest_framework import throttling

THROTTLE_CACHE_ALIAS = 'api_throttle'

# django caches are thread local, including their connection pools
# therefore, different threads cannot reuse each other's connections
# this cache and its connection pool are created at import time and are therefore global
throttle_cache = caches[THROTTLE_CACHE_ALIAS]

class UserRateThrottle(throttling.UserRateThrottle):
    cache = throttle_cache

class HostRateThrottle(throttling.SimpleRateThrottle):
    cache = throttle_cache
    scope = 'host'

    def get_cache_key(self, request, view):
        ident = self.get_ident(request)

        return self.cache_format % {
            'scope': self.scope,
            'ident': ident
        }
<|endoftext|>