<commit_before># pylint: disable=invalid-name
import re
import time
import os
import datetime
from collections import OrderedDict
from ..rfb_logger import rfb_log
from ..rfb_utils import filepath_utils
from .prefs_utils import get_pref
import bpy

COUNTERS = None

def time_this(f):
    """Function that can be used as a decorator to time any method."""
    def timed(*args, **kw):
        global COUNTERS
        ts = time.time()
        result = f(*args, **kw)
        te = time.time()
        elapsed = (te - ts) * 1000.0
        if COUNTERS is None:
            print('  _ %0.3f ms: %r args:[%r, %r]' % (elapsed, f.__name__,
                                                      args, kw))
        else:
            if not f.__name__ in COUNTERS:
                COUNTERS[f.__name__] = (1, elapsed)
            else:
                COUNTERS[f.__name__] = (COUNTERS[f.__name__][0] + 1,
                                        COUNTERS[f.__name__][1] + elapsed)
        return result

    return timed

STD_TOKENS = OrderedDict()
STD_TOKENS['ws'] = '<ws>:\tpath to current project'
STD_TOKENS['pwd'] = '<pwd>:\tpath to current directory'
STD_TOKENS['file'] = '<file>:\tscene file name'
STD_TOKENS['scene'] = '<scene>:\tscene name'
STD_TOKENS['camera'] = '<camera>:\trender camera'
STD_TOKENS['aov'] = '<aov>:\tAOV display name'
STD_TOKENS['aovdir'] = '''<aovdir>:\tAOV directory name (without '_variance' if denoised)'''
STD_TOKENS['ext'] = '<ext>:\timage file extension'
STD_TOKENS['layer'] = '<layer>:\trender layer name'
STD_TOKENS['shape'] = '<shape>:\tshort shape name'
STD_TOKENS['shapepath'] = '<shapepath>:\tlong shape name'
STD_TOKENS['frame'] = '<frame>:\tdecimal frame number'
STD_TOKENS['f'] = '<f>:\tframe number'
STD_TOKENS['f2'] = '<f2>:\tzero-padded frame number'
STD_TOKENS['f3'] = '<f3>:\tzero-padded frame number'
STD_TOKENS['f4'] = '<f4>:\tzero-padded frame number'
STD_TOKENS['jobid'] = '<jobid>:\tunique job identifier'
STD_TOKENS['date'] = '<date>:\tcurrent date'
STD_TOKENS['time'] = '<time>:\tcurrent time'
STD_TOKENS['version'] = '<version>:\tscene version number'
STD_TOKENS['take'] = '<take>:\tscene take number'
STD_TOKENS['assetlib'] = '<assetlib>:\tpath to the standard RenderManAssetLibrary'
STD_TOKENS['imagedir'] = '<imagedir>:\timages output path defined in rmanGlobals'
STD_TOKENS['integrator'] = '<integrator>:\tcurrent integrator'

PAD_FMT = ['%d', '%01d', '%02d', '%03d', '%04d']

# split the token into 3 groups:
#   1 - the main token
#   2 - the token formatting string
#   3 - an environment variable
#
# Examples:
#   '<shape.fogColor>'  ->  ('shape.fogColor', None, None)
#   '<shape.fogColor:%g %g %g>'  ->  ('shape.fogColor', ':%g %g %g', None)
#   '$RMANTREE/bin' ->  (None, None, '$RMANTREE')
#
PARSING_EXPR = re.compile(r'<([\w\:]*[\w\.]+)(\?[\w\d\._-]+)*'  # token
                          r'(:[^>]+)*>|'                        # formatter
                          r'\$\{?([A-Z0-9_]{3,})\}?')           # env var

class StringExpression(object):

    # @time_this
    def __init__(self, tokens=None, bl_scene=None):
        self.bl_scene = bpy.context.scene
        if bl_scene:
            self.bl_scene = bl_scene
        self.tokens = {}
        self.update_blend_tokens()
        self.tokens['pwd'] = ''
        ts = datetime.datetime.fromtimestamp(time.time())
        self.tokens['jobid'] = ts.strftime('%y%m%d%H%M%S')
        self.tokens['date'] = ts.strftime('%y_%m_%d')
        self.tokens['time'] = ts.strftime('%H-%M-%S')
        self.tokens['layer'] = ''
        self.tokens['renderlayer'] = self.tokens['layer']
        self.tokens['ext'] = 'exr'
        self.tokens['aov'] = ''
        self.tokens['aovdir'] = 'beauty'
        self.tokens['file'] = self.tokens['blend']
        self.tokens['ws'] = self.tokens['OUT']

        for k, v in self.tokens.items():
            # print '%s = %s' % (k, repr(v))
            # suppress any trailing slash or anti-slash for consistency.
            if v and v[-1] in ['/', os.sep]:
                self.tokens[k] = v[:-1]

        if tokens is not None:
            for k, v in tokens.items():
                tokens[k] = self.expand(v)
            self.tokens.update(tokens)

    def update_blend_tokens(self):
        scene = self.bl_scene
        rm = scene.renderman
        for i in range(len(rm.user_tokens)):
            user_token = rm.user_tokens[i]
            self.tokens[user_token.name] = user_token.value

        self.tokens['OUT'] = self.expand(get_pref('env_vars').out)
        unsaved = True if not bpy.data.filepath else False
        if unsaved:
            self.tokens['blend'] = 'UNTITLED'
            self.tokens['blend_dir'] = self.tokens['OUT']
        else:
            self.tokens['blend_dir'] = os.path.split(bpy.data.filepath)[0]
            self.tokens['blend'] = os.path.splitext(os.path.split(bpy.data.filepath)[1])[0]

        self.tokens['scene'] = scene.name
        self.set_frame_context(scene.frame_current)

        vp = get_pref('rman_scene_version_padding', default=3)
        tp = get_pref('rman_scene_take_padding', default=1)
        self.tokens['version'] = PAD_FMT[vp] % rm.version_token
        self.tokens['take'] = PAD_FMT[tp] % rm.take_token

    # @time_this
    def set_frame_context(self, frame):
        """Set the scene frame for the next subst() call."""
        self.tokens['frame'] = str(frame)
        iframe = int(frame)
        self.tokens['f'] = str(iframe)
        self.tokens['f2'] = '{:0>2d}'.format(iframe)
        self.tokens['f3'] = '{:0>3d}'.format(iframe)
        self.tokens['f4'] = '{:0>4d}'.format(iframe)
        self.tokens['F'] = str(iframe)
        self.tokens['F2'] = '{:0>2d}'.format(iframe)
        self.tokens['F3'] = '{:0>3d}'.format(iframe)
        self.tokens['F4'] = '{:0>4d}'.format(iframe)

    # @time_this
    def expand(self, expr, objTokens={}, asFilePath=False):
        """handle the '<token>' format"""

        # early exit: around 0.003ms
        if '<' not in expr and '$' not in expr:
            return expr

        toks = dict(self.tokens)
        toks.update(objTokens)
        # print toks
        pos = 0
        result = ''
        for m in re.finditer(PARSING_EXPR, expr):
            # print 'GROUPS: %s' % str(m.groups())
            if m.group(1):
                # Token case
                tok = m.group(1)
                tok_val = None
                # print '  |__ TOKEN: %02d-%02d: %s' % (m.start(), m.end(), tok)
                try:
                    tok_val = toks[tok]
                    # result += expr[pos:m.start()] + toks[tok]
                except KeyError:
                    try:
                        # forced lower-case version if first attempts failed.
                        tok_val = toks[tok.lower()]
                    except KeyError:
                        # the token REALLY doesn't exist...
                        try:
                            # if it contains 'shape.' it may be a getAttr for an
                            # attribute on the current shape.
                            tok = tok.replace('shape.', '%s.' % toks['shape'])
                        except KeyError:
                            pass
                        try:
                            # if it contains 'shapepath.' it may be a getAttr for an
                            # attribute on the current shape.
                            tok = tok.replace('shapepath.', '%s.' % toks['shapepath'])
                        except KeyError:
                            pass
                        try:
                            # could be a getAttr node.attr ?
                            tok_val = '' #mc.getAttr(tok)
                        except ValueError as err:
                            # the node or attribute desn't exist.
                            if m.group(2):
                                # a default value has been provided
                                tok_val = m.group(2)[1:]
                            else:
                                # the node or/and attribute doesn't exist, pass
                                # un-substituted: that's better for debugging.
                                rfb_log().debug('%s  (maya plug = %r)', err, tok)
                                tok_val = m.group(0)
                        except TypeError as err:
                            # invalid plug string, just append it. Tokens like
                            # <udim> will be substituted by the renderer.
                            tok_val = m.group(0)
                            rfb_log().debug('%s  (maya plug = %r)', err, tok)
                            # print '   un-processed token: %s' % m.group(0)
                        else:
                            pass
                            # getAttr(tok) succeeded: store the result
                            """
                            if mc.getAttr(tok, mi=True) is None:
                                if mc.getAttr(tok, type=True) == 'float3':
                                    # maya returns [(x, x, x)] and we just want (x, x, x)
                                    tok_val = tok_val[0]
                            else:
                                rfb_log().warning('Array attribute lookups are '
                                                  'not supported ! (%s)', tok)
                                continue
                            """
                # optional formating
                if m.group(3):
                    if isinstance(tok_val, basestring) and tok_val:
                        try:
                            tok_val = eval(tok_val)
                        except (NameError, SyntaxError, TypeError) as err:
                            rfb_log().debug('Eval failed: %s  -> %r', err, tok_val)
                            result += expr[pos:m.start()] +  tok_val
                        else:
                            result += expr[pos:m.start()] +  m.group(3)[1:] % tok_val
                    else:
                        result += expr[pos:m.start()] +  m.group(3)[1:] % tok_val
                else:
                    result += '%s%s' % (expr[pos:m.start()], tok_val)
            else:
                # Environment variable case
                tok = m.group(4)
                # print '  |__ ENV VAR: %02d-%02d: %s' % (m.start(), m.end(), tok)
                try:
                    result += expr[pos: m.start()] + os.environ[tok]
                except KeyError:
                    result += expr[pos: m.start()] + m.group(0)

            pos = m.end()
        # append the end of the string. If no match (despite the presence of
        # a < or $), sets result to the original expression.
        result += expr[pos:]

        if asFilePath:
            # If this is meant to be a file path, substitute : with _
            # Can not have ':' after the drive descriptor on windows. Allow
            # for a leading space before the drive letter
            result = re.sub(r'((?<!^[A-Z])(?<!^[ ][A-Z]))\:', '_', result)
            result = result.replace(' ', '_')

            # get the real path
            result = filepath_utils.get_real_path(result)

            dirname = os.path.dirname(result)
            if not os.path.exists(dirname):
                os.makedirs(dirname, exist_ok=True)

        return result

def fixup_file_name(inputNm):
    result = inputNm
    # replace repeated underscores with one underscore
    # these come about if a var like <layer> is empty
    result = re.sub(r'_+', '_', result)
    # eg.  test_.0001.exr -> test.0001.exr
    result = re.sub(r'_\.', '.', result)
    # strip underscore if it's at end of string
    result = result.rstrip('_')
    return result<commit_msg>Fix issue where we would return an empty string for unknown tokens , example <udim>. Also, fix issue where OUT wasn't expanded correctly.<commit_after># pylint: disable=invalid-name
import re
import time
import os
import datetime
from collections import OrderedDict
from ..rfb_logger import rfb_log
from ..rfb_utils import filepath_utils
from .prefs_utils import get_pref
import bpy

COUNTERS = None

def time_this(f):
    """Function that can be used as a decorator to time any method."""
    def timed(*args, **kw):
        global COUNTERS
        ts = time.time()
        result = f(*args, **kw)
        te = time.time()
        elapsed = (te - ts) * 1000.0
        if COUNTERS is None:
            print('  _ %0.3f ms: %r args:[%r, %r]' % (elapsed, f.__name__,
                                                      args, kw))
        else:
            if not f.__name__ in COUNTERS:
                COUNTERS[f.__name__] = (1, elapsed)
            else:
                COUNTERS[f.__name__] = (COUNTERS[f.__name__][0] + 1,
                                        COUNTERS[f.__name__][1] + elapsed)
        return result

    return timed

STD_TOKENS = OrderedDict()
STD_TOKENS['ws'] = '<ws>:\tpath to current project'
STD_TOKENS['pwd'] = '<pwd>:\tpath to current directory'
STD_TOKENS['file'] = '<file>:\tscene file name'
STD_TOKENS['scene'] = '<scene>:\tscene name'
STD_TOKENS['camera'] = '<camera>:\trender camera'
STD_TOKENS['aov'] = '<aov>:\tAOV display name'
STD_TOKENS['aovdir'] = '''<aovdir>:\tAOV directory name (without '_variance' if denoised)'''
STD_TOKENS['ext'] = '<ext>:\timage file extension'
STD_TOKENS['layer'] = '<layer>:\trender layer name'
STD_TOKENS['shape'] = '<shape>:\tshort shape name'
STD_TOKENS['shapepath'] = '<shapepath>:\tlong shape name'
STD_TOKENS['frame'] = '<frame>:\tdecimal frame number'
STD_TOKENS['f'] = '<f>:\tframe number'
STD_TOKENS['f2'] = '<f2>:\tzero-padded frame number'
STD_TOKENS['f3'] = '<f3>:\tzero-padded frame number'
STD_TOKENS['f4'] = '<f4>:\tzero-padded frame number'
STD_TOKENS['jobid'] = '<jobid>:\tunique job identifier'
STD_TOKENS['date'] = '<date>:\tcurrent date'
STD_TOKENS['time'] = '<time>:\tcurrent time'
STD_TOKENS['version'] = '<version>:\tscene version number'
STD_TOKENS['take'] = '<take>:\tscene take number'
STD_TOKENS['assetlib'] = '<assetlib>:\tpath to the standard RenderManAssetLibrary'
STD_TOKENS['imagedir'] = '<imagedir>:\timages output path defined in rmanGlobals'
STD_TOKENS['integrator'] = '<integrator>:\tcurrent integrator'

PAD_FMT = ['%d', '%01d', '%02d', '%03d', '%04d']

# split the token into 3 groups:
#   1 - the main token
#   2 - the token formatting string
#   3 - an environment variable
#
# Examples:
#   '<shape.fogColor>'  ->  ('shape.fogColor', None, None)
#   '<shape.fogColor:%g %g %g>'  ->  ('shape.fogColor', ':%g %g %g', None)
#   '$RMANTREE/bin' ->  (None, None, '$RMANTREE')
#
PARSING_EXPR = re.compile(r'<([\w\:]*[\w\.]+)(\?[\w\d\._-]+)*'  # token
                          r'(:[^>]+)*>|'                        # formatter
                          r'\$\{?([A-Z0-9_]{3,})\}?')           # env var

class StringExpression(object):

    # @time_this
    def __init__(self, tokens=None, bl_scene=None):
        self.bl_scene = bpy.context.scene
        if bl_scene:
            self.bl_scene = bl_scene
        self.tokens = {}
        self.update_blend_tokens()
        self.tokens['pwd'] = ''
        ts = datetime.datetime.fromtimestamp(time.time())
        self.tokens['jobid'] = ts.strftime('%y%m%d%H%M%S')
        self.tokens['date'] = ts.strftime('%y_%m_%d')
        self.tokens['time'] = ts.strftime('%H-%M-%S')
        self.tokens['layer'] = ''
        self.tokens['renderlayer'] = self.tokens['layer']
        self.tokens['ext'] = 'exr'
        self.tokens['aov'] = ''
        self.tokens['aovdir'] = 'beauty'
        self.tokens['file'] = self.tokens['blend']
        self.tokens['ws'] = self.tokens['OUT']

        for k, v in self.tokens.items():
            # print '%s = %s' % (k, repr(v))
            # suppress any trailing slash or anti-slash for consistency.
            if v and v[-1] in ['/', os.sep]:
                self.tokens[k] = v[:-1]

        if tokens is not None:
            for k, v in tokens.items():
                tokens[k] = self.expand(v)
            self.tokens.update(tokens)

    def update_blend_tokens(self):
        scene = self.bl_scene
        rm = scene.renderman
        for i in range(len(rm.user_tokens)):
            user_token = rm.user_tokens[i]
            self.tokens[user_token.name] = user_token.value

        unsaved = True if not bpy.data.filepath else False
        if unsaved:
            self.tokens['blend'] = 'UNTITLED'
            self.tokens['blend_dir'] = ''
        else:
            self.tokens['blend_dir'] = os.path.split(bpy.data.filepath)[0]
            self.tokens['blend'] = os.path.splitext(os.path.split(bpy.data.filepath)[1])[0]

        self.tokens['OUT'] = self.expand(get_pref('env_vars').out)
        if self.tokens['blend_dir'] == '':
            self.tokens['blend_dir'] = self.tokens['OUT']
        self.tokens['scene'] = scene.name
        self.set_frame_context(scene.frame_current)

        vp = get_pref('rman_scene_version_padding', default=3)
        tp = get_pref('rman_scene_take_padding', default=1)
        self.tokens['version'] = PAD_FMT[vp] % rm.version_token
        self.tokens['take'] = PAD_FMT[tp] % rm.take_token

    # @time_this
    def set_frame_context(self, frame):
        """Set the scene frame for the next subst() call."""
        self.tokens['frame'] = str(frame)
        iframe = int(frame)
        self.tokens['f'] = str(iframe)
        self.tokens['f2'] = '{:0>2d}'.format(iframe)
        self.tokens['f3'] = '{:0>3d}'.format(iframe)
        self.tokens['f4'] = '{:0>4d}'.format(iframe)
        self.tokens['F'] = str(iframe)
        self.tokens['F2'] = '{:0>2d}'.format(iframe)
        self.tokens['F3'] = '{:0>3d}'.format(iframe)
        self.tokens['F4'] = '{:0>4d}'.format(iframe)

    # @time_this
    def expand(self, expr, objTokens={}, asFilePath=False):
        """handle the '<token>' format"""

        # early exit: around 0.003ms
        if '<' not in expr and '$' not in expr:
            return expr

        toks = dict(self.tokens)
        toks.update(objTokens)
        # print toks
        pos = 0
        result = ''
        for m in re.finditer(PARSING_EXPR, expr):
            # print 'GROUPS: %s' % str(m.groups())
            if m.group(1):
                # Token case
                tok = m.group(1)
                tok_val = None
                # print '  |__ TOKEN: %02d-%02d: %s' % (m.start(), m.end(), tok)
                try:
                    tok_val = toks[tok]
                    # result += expr[pos:m.start()] + toks[tok]
                except KeyError:
                    try:
                        # forced lower-case version if first attempts failed.
                        tok_val = toks[tok.lower()]
                    except KeyError:
                        # the token REALLY doesn't exist...
                        tok_val = '<%s>' % tok

                # optional formating
                if m.group(3):
                    if isinstance(tok_val, basestring) and tok_val:
                        try:
                            tok_val = eval(tok_val)
                        except (NameError, SyntaxError, TypeError) as err:
                            rfb_log().debug('Eval failed: %s  -> %r', err, tok_val)
                            result += expr[pos:m.start()] +  tok_val
                        else:
                            result += expr[pos:m.start()] +  m.group(3)[1:] % tok_val
                    else:
                        result += expr[pos:m.start()] +  m.group(3)[1:] % tok_val
                else:
                    result += '%s%s' % (expr[pos:m.start()], tok_val)
            else:
                # Environment variable case
                tok = m.group(4)
                # print '  |__ ENV VAR: %02d-%02d: %s' % (m.start(), m.end(), tok)
                try:
                    result += expr[pos: m.start()] + os.environ[tok]
                except KeyError:
                    result += expr[pos: m.start()] + m.group(0)

            pos = m.end()
        # append the end of the string. If no match (despite the presence of
        # a < or $), sets result to the original expression.
        result += expr[pos:]

        if asFilePath:
            # If this is meant to be a file path, substitute : with _
            # Can not have ':' after the drive descriptor on windows. Allow
            # for a leading space before the drive letter
            result = re.sub(r'((?<!^[A-Z])(?<!^[ ][A-Z]))\:', '_', result)
            result = result.replace(' ', '_')

            # get the real path
            result = filepath_utils.get_real_path(result)

            dirname = os.path.dirname(result)
            if not os.path.exists(dirname):
                os.makedirs(dirname, exist_ok=True)

        return result

def fixup_file_name(inputNm):
    result = inputNm
    # replace repeated underscores with one underscore
    # these come about if a var like <layer> is empty
    result = re.sub(r'_+', '_', result)
    # eg.  test_.0001.exr -> test.0001.exr
    result = re.sub(r'_\.', '.', result)
    # strip underscore if it's at end of string
    result = result.rstrip('_')
    return result<|endoftext|>


<commit_before>from oauth2_provider.models import AbstractApplication

from django.db import models

class MyApplication(AbstractApplication):
    """
    Custom Application model which adds description field
    """
    description = models.TextField()
<commit_msg>description optional<commit_after>from oauth2_provider.models import AbstractApplication

from django.db import models

class MyApplication(AbstractApplication):
    """
    Custom Application model which adds description field
    """
    description = models.TextField(blank=True)
<|endoftext|>


<commit_before>import serial
import time
powerOff = (0xAA, 0x11, 0xFE, 0x01, 0x01, 0x11)
ser = serial.Serial(
                    port='COM3',
                    baudrate = 9600,
                    parity=serial.PARITY_NONE,
                    stopbits=serial.STOPBITS_ONE,
                    bytesize=serial.EIGHTBITS)  # abrir y configurar puerto serie

time.sleep(1)            #para que le de tiempo a abrir
for n in powerOff:
    ser.write(chr(n))   # lanxzamos el array de uno en uno

ser.close()              # cerramos puerto
<commit_msg>Update encender.py<commit_after>import serial
import time
powerOff = (0xAA, 0x11, 0xFE, 0x01, 0x01, 0x11)
ser = serial.Serial(
                    port='COM3',
                    baudrate = 9600,
                    parity=serial.PARITY_NONE,
                    stopbits=serial.STOPBITS_ONE,
                    bytesize=serial.EIGHTBITS)  # abrir y configurar puerto serie

time.sleep(0.1)            #para que le de tiempo a abrir
for n in powerOff:
    ser.write(chr(n))   # lanxzamos el array de uno en uno

ser.close()              # cerramos puerto
<|endoftext|>


<commit_before>from future.utils import raise_from

from rightshift import Transformer, TransformationException, Chain

__author__ = 'adam.jorgensen.za@gmail.com'

class ExtractorException(TransformationException): pass

class Extractor(Transformer): pass

class ItemChain(Chain):
    """
    TODO: Document
    """
    def __getattr__(self, item_name):
        """
        :param item_name: A valid item name value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return self.__getitem__(item_name)

    def __getitem__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return ItemChain(self, Item(item_or_slice))

class Item(Extractor):
    """
    An Item instances expects to be called with a value that will be
    accessed as if it were a container type in order to retrieve the item or
    slice that the ItemExtract was instantiated with.

    If retrieval of the item fails from the value then an ExtractorException
    is raised.
    """
    def __init__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        """
        self.item_or_slice = item_or_slice

    def __call__(self, value, **flags):
        """
        :param value: The value to attempt extraction from
        :param flags: A dictionary of flags
        :return: The extracted value
        :raise: ExtractorException
        """
        try:
            return value[self.item_or_slice]
        except Exception as e:
            raise_from(ExtractorException, e)

    def __getattr__(self, item_name):
        """
        :param item_name: A valid item name value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return self.__getitem__(item_name)

    def __getitem__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return ItemChain(self, Item(item_or_slice))

class _ItemCreator(object):
    """
    The _ItemCreator is a private class that is instantiated once
    and assigned to the `item` variable in the rightshift.extractors module.

    This allows syntax of the form item['key'], item.key and item('key') to be
    used in place of the less compact Item('key') form.
    """
    def __call__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        :return: an Item instance
        :rtype: Item
        """
        return self.__getitem__(item_or_slice)

    def __getattr__(self, item_name):
        """
        :param item_name: A valid item name value
        :return: an Item instance
        :rtype: Item
        """
        return self.__getitem__(item_name)

    def __getitem__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        :return: an Item instance
        :rtype: Item
        """

        return Item(item_or_slice)

item = _ItemCreator()
"""
item is a special shortcut to enable working with the Item class to
feel more natural. item is an instance of the private _ItemCreator
class which mirrors the functionality of the Item class but is not
actually an instance of Item itself. This allows item to be used to
generate natural looking item extraction expressions.

Examples:

item['x'] is equivalent to Item('x')
item['x']['y'] is equivalent to Item('x')['y']
"""

class AttributeChain(Chain):
    """
    TODO: Document
    """
    def __getattr__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return self.__getitem__(attribute)

    def __getitem__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return AttributeChain(self, Attribute(attribute))

class Attribute(Extractor):
    """
    An Attribute instance can be called with a value in order to
    attempt to retrieve an attribute on that value.

    If retrieval of the attribute fails from the value then an ExtractorException
    is raised.
    """
    def __init__(self, attribute):
        """
        :param attribute: A valid attribute name value
        """
        self.attribute = attribute

    def __call__(self, value, **flags):
        """
        :param value: The value to attempt extraction from
        :param flags: A dictionary of flags
        :return: The extracted value
        :raise: ExtractorException
        """
        if hasattr(value, self.attribute):
            return getattr(value, self.attribute)
        else:
            raise ExtractorException('{} has no attribute `{}`'.format(value, self.attribute))

    def __getitem__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return self.__getattr__(attribute)

    def __getattr__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return AttributeChain(self, Attribute(attribute))

class _AttributeCreator(object):
    """
    The _AttributeCreator is a private class that is instantiated once
    and assigned to the `attr` variable in the rightshift.extractors module.

    This allows syntax of the form item['key'], item.key and item('key') to be
    used in place of the less compact Item('key') form.
    """
    def __call__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeExtractor instance
        :rtype: Attribute
        """
        return self.__getattr__(attribute)

    def __getitem__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeExtractor instance
        :rtype: Attribute
        """
        return self.__getattr__(attribute)

    def __getattr__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeExtractor instance
        :rtype: Attribute
        """

        return Attribute(attribute)

    def __setattr__(self, key, value):
        """
        :raise: NotImplementedError
        """
        raise NotImplementedError

attr = prop = _AttributeCreator()
"""
attr is a special shortcut to enable working with the Attribute class
to feel more natural. attr is an instance of the private _AttributeCreator
class which mirrors the functionality of the Attribute class but is not
actually an instance of Attribute itself. This allows attr to be used
to generate natural looking attr extraction expressions.

Examples:

attr.x is equivalent to Item('x')
attr.x.y is equivalent to Item('x').y
"""

class PatternGroup(Extractor):
    """
    A PatternGroup can be called with a string in order to attempt to extract a
    new string from that string using a regular expression.
    """
    def __init__(self, pattern, group=1, search=True):
        """
        :param pattern: A string or compiled Regular Expression pattern
        :param group: A string or numeric group value
        :param search: A boolean value indicating whether the search or match
                       method should be used
        """
        from past.builtins import basestring
        if isinstance(pattern, basestring):
            from re import compile
            pattern = compile(pattern)
        self.pattern = pattern
        self.group = group
        self.search = search

    def __call__(self, value, **flags):
        try:
            method = self.pattern.search if flags.get('pattern_group__search', self.search) else self.pattern.match
            match = method(value)
            if match is None:
                raise ExtractorException
            return match.group(flags.get('pattern_group__group', self.group))
        except Exception as e:
            raise_from(ExtractorException, e)

pattern_group = PatternGroup
"""
TODO: Document
"""

class CoerceTo(Extractor):
    """
    A CoerceTo instance will attempt to transform the type of an input value.
    """
    def __init__(self, type, coercer=None):
        if type is None and coercer is None:
            coercer = lambda v: None
        if coercer is None:
            coercer = type
        self.type = type
        self.coercer = coercer

    def __call__(self, value, **flags):
        try:
            value = self.coercer(value)
            if not isinstance(value, self.type):
                raise ExtractorException('Unable to coerce {} to {}'.format(
                    value, self.type))
        except ExtractorException:
            raise
        except Exception as e:
            raise_from(ExtractorException, e)
        return value

coerce_to = CoerceTo

<commit_msg>#25: Leveraged IndexOrAccessToInstantiate metaclass to substantially simplify code for Item and Attribute extractors.<commit_after>from future.utils import raise_from

from rightshift import Transformer, TransformationException, Chain
from rightshift.magic import IndexOrAccessToInstantiate

__author__ = 'adam.jorgensen.za@gmail.com'

class ExtractorException(TransformationException):
    """
    Base class for all Exceptions thrown by components of this module.
    """

class Extractor(Transformer):
    """
    Base class for all Transformers defined in this module. If you are writing
    a new Transformer that that fits the general description of an Extractor
    then it should inherit from this class.
    """

class ItemChain(Chain):
    """
    TODO: Document
    """
    def __getattr__(self, item_name):
        """
        :param item_name: A valid item name value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return self.__getitem__(item_name)

    def __getitem__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return ItemChain(self, Item(item_or_slice))

class Item(Extractor):
    """
    An Item instances expects to be called with a value that will be
    accessed as if it were a container type in order to retrieve the item or
    slice that the ItemExtract was instantiated with.

    If retrieval of the item fails from the value then an ExtractorException
    is raised.

    Examples:

    Item('x')
    Item.x
    Item['x']
    Item.x.y
    Item['x']['y']
    Item[variable]
    Item[42]
    """
    __metaclass__ = IndexOrAccessToInstantiate

    def __init__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        """
        self.item_or_slice = item_or_slice

    def __call__(self, value, **flags):
        """
        :param value: The value to attempt extraction from
        :param flags: A dictionary of flags
        :return: The extracted value
        :raise: ExtractorException
        """
        try:
            return value[self.item_or_slice]
        except Exception as e:
            raise_from(ExtractorException, e)

    def __getattr__(self, item_name):
        """
        :param item_name: A valid item name value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return self.__getitem__(item_name)

    def __getitem__(self, item_or_slice):
        """
        :param item_or_slice: A valid item name or slice value
        :return: an ItemChain instance
        :rtype: ItemChain
        """
        return ItemChain(self, Item(item_or_slice))

item = Item
"""
item is an alias to the Item class.
"""

class AttributeChain(Chain):
    """
    TODO: Document
    """
    def __getattr__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return self.__getitem__(attribute)

    def __getitem__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return AttributeChain(self, Attribute(attribute))

class Attribute(Extractor):
    """
    An Attribute instance can be called with a value in order to
    attempt to retrieve an attribute on that value.

    If retrieval of the attribute fails from the value then an ExtractorException
    is raised.

    Examples:

    Attribute('x')
    Attribute.x
    Attribute['x']
    Attribute('x').y
    Attribute.x.y
    Attribute['x']['y']
    Attribute[variable]
    """
    __metaclass__ = IndexOrAccessToInstantiate

    def __init__(self, attribute):
        """
        :param attribute: A valid attribute name value
        """
        self.attribute = attribute

    def __call__(self, value, **flags):
        """
        :param value: The value to attempt extraction from
        :param flags: A dictionary of flags
        :return: The extracted value
        :raise: ExtractorException
        """
        if hasattr(value, self.attribute):
            return getattr(value, self.attribute)
        else:
            raise ExtractorException('{} has no attribute `{}`'.format(value, self.attribute))

    def __getitem__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return self.__getattr__(attribute)

    def __getattr__(self, attribute):
        """
        :param attribute: A valid attribute name value
        :return: an AttributeChain instance
        :rtype: AttributeChain
        """
        return AttributeChain(self, Attribute(attribute))

attr = prop = Attribute
"""
attr and prop are aliases to the Attribute class.
"""

class PatternGroup(Extractor):
    """
    A PatternGroup can be called with a string in order to attempt to extract a
    new string from that string using a regular expression.
    """
    def __init__(self, pattern, group=1, search=True):
        """
        :param pattern: A string or compiled Regular Expression pattern
        :param group: A string or numeric group value
        :param search: A boolean value indicating whether the search or match
                       method should be used
        """
        from past.builtins import basestring
        if isinstance(pattern, basestring):
            from re import compile
            pattern = compile(pattern)
        self.pattern = pattern
        self.group = group
        self.search = search

    def __call__(self, value, **flags):
        try:
            method = self.pattern.search if flags.get('pattern_group__search', self.search) else self.pattern.match
            match = method(value)
            if match is None:
                raise ExtractorException
            return match.group(flags.get('pattern_group__group', self.group))
        except Exception as e:
            raise_from(ExtractorException, e)

pattern_group = PatternGroup
"""
TODO: Document
"""

class CoerceTo(Extractor):
    """
    A CoerceTo instance will attempt to transform the type of an input value.
    """
    def __init__(self, type, coercer=None):
        if type is None and coercer is None:
            coercer = lambda v: None
        if coercer is None:
            coercer = type
        self.type = type
        self.coercer = coercer

    def __call__(self, value, **flags):
        try:
            value = self.coercer(value)
            if not isinstance(value, self.type):
                raise ExtractorException('Unable to coerce {} to {}'.format(
                    value, self.type))
        except ExtractorException:
            raise
        except Exception as e:
            raise_from(ExtractorException, e)
        return value

coerce_to = CoerceTo

<|endoftext|>


<commit_before>from __future__ import absolute_import
import subprocess
import tarfile
import os
import zipfile
import six
import sys
import traceback

from billiard import Process, Queue

from django.utils.text import get_valid_filename
from django.core.files import File
from django.conf import settings

from celery import Task
from celery import app
from celery.schedules import crontab
from celery.signals import worker_process_init

from . import settings as wooey_settings

try:
    from Queue import Empty
except ImportError:
    from queue import Empty  # python 3.x

ON_POSIX = 'posix' in sys.builtin_module_names
ON_WINDOWS = os.name == 'nt'

if ON_WINDOWS:
    import msvcrt
    from billiard.reduction import steal_handle

celery_app = app.app_or_default()

def enqueue_output(out, q):
    if ON_WINDOWS:
        new_handle = steal_handle(os.getppid(), out)
        fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY | os.O_BINARY)
        out = open(fd, 'rb')
    for line in iter(out.readline, b''):
        q.put(line.decode('utf-8'))
    try:
        out.close()
    except IOError:
        pass

def output_monitor_queue(queue, out):
    p = Process(target=enqueue_output, args=(msvcrt.get_osfhandle(out.fileno()) if ON_WINDOWS else out, queue))
    p.start()
    return p

def update_from_output_queue(queue, out):
    lines = []
    while True:
        try:
            line = queue.get_nowait()
            lines.append(line)
        except Empty:
            break

    out += ''.join(map(str, lines))
    return out

@worker_process_init.connect
def configure_workers(*args, **kwargs):
    # this sets up Django on nodes started by the worker daemon.
    import django
    django.setup()

class WooeyTask(Task):
    pass

    # def after_return(self, status, retval, task_id, args, kwargs, einfo):
    #     job, created = WooeyJob.objects.get_or_create(wooey_celery_id=task_id)
    #     job.content_type.wooey_celery_state = status
    #     job.save()

@celery_app.task(base=WooeyTask)
def submit_script(**kwargs):
    job_id = kwargs.pop('wooey_job')
    resubmit = kwargs.pop('wooey_resubmit', False)
    from .backend import utils
    from .models import WooeyJob, UserFile
    job = WooeyJob.objects.get(pk=job_id)

    command = utils.get_job_commands(job=job)
    if resubmit:
        # clone ourselves, setting pk=None seems hackish but it works
        job.pk = None

    # This is where the script works from -- it is what is after the media_root since that may change between
    # setups/where our user uploads are stored.
    cwd = job.get_output_path()

    abscwd = os.path.abspath(os.path.join(settings.MEDIA_ROOT, cwd))
    job.command = ' '.join(command)
    job.save_path = cwd

    utils.mkdirs(abscwd)
    # make sure we have the script, otherwise download it. This can happen if we have an ephemeral file system or are
    # executing jobs on a worker node.
    script_path = job.script_version.script_path
    if not utils.get_storage(local=True).exists(script_path.path):
        utils.get_storage(local=True).save(script_path.path, script_path.file)

    job.status = WooeyJob.RUNNING
    job.save()

    stdout, stderr = '', ''
    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=abscwd, bufsize=0)

    # We need to use subprocesses to capture the IO, otherwise they will block one another
    # i.e. a check against stderr will sit waiting on stderr before returning
    # we use Queues to communicate
    qout, qerr = Queue(), Queue()
    pout = output_monitor_queue(qout, proc.stdout)
    perr = output_monitor_queue(qerr, proc.stderr)

    prev_std = None

    def check_output(job, stdout, stderr, prev_std):
        # Check for updates from either (non-blocking)
        stdout = update_from_output_queue(qout, stdout)
        stderr = update_from_output_queue(qerr, stderr)

        # If there are changes, update the db
        if (stdout, stderr) != prev_std:
            job.update_realtime(stdout=stdout, stderr=stderr)
            prev_std = (stdout, stderr)

        return stdout, stderr, prev_std

    # Loop until the process is complete + both stdout/stderr have EOFd
    while proc.poll() is None or pout.is_alive() or perr.is_alive():
        stdout, stderr, prev_std = check_output(job, stdout, stderr, prev_std)

    # Catch any remaining output
    try:
        proc.stdout.flush()
    except ValueError:  # Handle if stdout is closed
        pass
    stdout, stderr, prev_std = check_output(job, stdout, stderr, prev_std)

    # tar/zip up the generated content for bulk downloads
    def get_valid_file(cwd, name, ext):
        out = os.path.join(cwd, name)
        index = 0
        while os.path.exists(six.u('{}.{}').format(out, ext)):
            index += 1
            out = os.path.join(cwd, six.u('{}_{}').format(name, index))
        return six.u('{}.{}').format(out, ext)

    # fetch the job again in case the database connection was lost during the job or something else changed.
    job = WooeyJob.objects.get(pk=job_id)
    # if there are files generated, make zip/tar files for download
    if len(os.listdir(abscwd)):
        tar_out = get_valid_file(abscwd, get_valid_filename(job.job_name), 'tar.gz')
        tar = tarfile.open(tar_out, "w:gz")
        tar_name = os.path.splitext(os.path.splitext(os.path.split(tar_out)[1])[0])[0]
        tar.add(abscwd, arcname=tar_name)
        tar.close()

        zip_out = get_valid_file(abscwd, get_valid_filename(job.job_name), 'zip')
        zip = zipfile.ZipFile(zip_out, "w")
        arcname = os.path.splitext(os.path.split(zip_out)[1])[0]
        zip.write(abscwd, arcname=arcname)
        for root, folders, filenames in os.walk(os.path.split(zip_out)[0]):
            for filename in filenames:
                path = os.path.join(root, filename)
                if path == tar_out:
                    continue
                if path == zip_out:
                    continue
                try:
                    zip.write(path, arcname=os.path.join(arcname, filename))
                except:
                    stderr = '{}\n{}'.format(stderr, traceback.format_exc())
        try:
            zip.close()
        except:
            stderr = '{}\n{}'.format(stderr, traceback.format_exc())

        # save all the files generated as well to our default storage for ephemeral storage setups
        if wooey_settings.WOOEY_EPHEMERAL_FILES:
            for root, folders, files in os.walk(abscwd):
                for filename in files:
                    filepath = os.path.join(root, filename)
                    s3path = os.path.join(root[root.find(cwd):], filename)
                    remote = utils.get_storage(local=False)
                    exists = remote.exists(s3path)
                    filesize = remote.size(s3path) if exists else 0
                    if not exists or (exists and filesize == 0):
                        if exists:
                            remote.delete(s3path)
                        remote.save(s3path, File(open(filepath, 'rb')))
    utils.create_job_fileinfo(job)

    job.stdout = stdout
    job.stderr = stderr
    job.status = WooeyJob.COMPLETED
    job.update_realtime(delete=True)
    job.save()

    return (stdout, stderr)

@celery_app.task(base=WooeyTask)
def cleanup_wooey_jobs(**kwargs):
    from django.utils import timezone
    from .models import WooeyJob

    cleanup_settings = wooey_settings.WOOEY_JOB_EXPIRATION
    anon_settings = cleanup_settings.get('anonymous')
    now = timezone.now()
    if anon_settings:
        WooeyJob.objects.filter(user=None, created_date__lte=now-anon_settings).delete()
    user_settings = cleanup_settings.get('user')
    if user_settings:
        WooeyJob.objects.filter(user__isnull=False, created_date__lte=now-user_settings).delete()

celery_app.conf.update(
    CELERYBEAT_SCHEDULE={
        'cleanup-old-jobs': {
            'task': 'wooey.tasks.cleanup_wooey_jobs',
            'schedule': crontab(hour=0, minute=0),  # cleanup at midnight each day
        },
    }
)
<commit_msg>fix steal_handle import<commit_after>from __future__ import absolute_import
import subprocess
import tarfile
import os
import zipfile
import six
import sys
import traceback

from billiard import Process, Queue

from django.utils.text import get_valid_filename
from django.core.files import File
from django.conf import settings

from celery import Task
from celery import app
from celery.schedules import crontab
from celery.signals import worker_process_init

from . import settings as wooey_settings

try:
    from Queue import Empty
except ImportError:
    from queue import Empty  # python 3.x

ON_POSIX = 'posix' in sys.builtin_module_names
ON_WINDOWS = os.name == 'nt'

if ON_WINDOWS:
    import msvcrt
    from multiprocessing.reduction import steal_handle

celery_app = app.app_or_default()

def enqueue_output(out, q):
    if ON_WINDOWS:
        new_handle = steal_handle(os.getppid(), out)
        fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY | os.O_BINARY)
        out = open(fd, 'rb')
    for line in iter(out.readline, b''):
        q.put(line.decode('utf-8'))
    try:
        out.close()
    except IOError:
        pass

def output_monitor_queue(queue, out):
    p = Process(target=enqueue_output, args=(msvcrt.get_osfhandle(out.fileno()) if ON_WINDOWS else out, queue))
    p.start()
    return p

def update_from_output_queue(queue, out):
    lines = []
    while True:
        try:
            line = queue.get_nowait()
            lines.append(line)
        except Empty:
            break

    out += ''.join(map(str, lines))
    return out

@worker_process_init.connect
def configure_workers(*args, **kwargs):
    # this sets up Django on nodes started by the worker daemon.
    import django
    django.setup()

class WooeyTask(Task):
    pass

    # def after_return(self, status, retval, task_id, args, kwargs, einfo):
    #     job, created = WooeyJob.objects.get_or_create(wooey_celery_id=task_id)
    #     job.content_type.wooey_celery_state = status
    #     job.save()

@celery_app.task(base=WooeyTask)
def submit_script(**kwargs):
    job_id = kwargs.pop('wooey_job')
    resubmit = kwargs.pop('wooey_resubmit', False)
    from .backend import utils
    from .models import WooeyJob, UserFile
    job = WooeyJob.objects.get(pk=job_id)

    command = utils.get_job_commands(job=job)
    if resubmit:
        # clone ourselves, setting pk=None seems hackish but it works
        job.pk = None

    # This is where the script works from -- it is what is after the media_root since that may change between
    # setups/where our user uploads are stored.
    cwd = job.get_output_path()

    abscwd = os.path.abspath(os.path.join(settings.MEDIA_ROOT, cwd))
    job.command = ' '.join(command)
    job.save_path = cwd

    utils.mkdirs(abscwd)
    # make sure we have the script, otherwise download it. This can happen if we have an ephemeral file system or are
    # executing jobs on a worker node.
    script_path = job.script_version.script_path
    if not utils.get_storage(local=True).exists(script_path.path):
        utils.get_storage(local=True).save(script_path.path, script_path.file)

    job.status = WooeyJob.RUNNING
    job.save()

    stdout, stderr = '', ''
    proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=abscwd, bufsize=0)

    # We need to use subprocesses to capture the IO, otherwise they will block one another
    # i.e. a check against stderr will sit waiting on stderr before returning
    # we use Queues to communicate
    qout, qerr = Queue(), Queue()
    pout = output_monitor_queue(qout, proc.stdout)
    perr = output_monitor_queue(qerr, proc.stderr)

    prev_std = None

    def check_output(job, stdout, stderr, prev_std):
        # Check for updates from either (non-blocking)
        stdout = update_from_output_queue(qout, stdout)
        stderr = update_from_output_queue(qerr, stderr)

        # If there are changes, update the db
        if (stdout, stderr) != prev_std:
            job.update_realtime(stdout=stdout, stderr=stderr)
            prev_std = (stdout, stderr)

        return stdout, stderr, prev_std

    # Loop until the process is complete + both stdout/stderr have EOFd
    while proc.poll() is None or pout.is_alive() or perr.is_alive():
        stdout, stderr, prev_std = check_output(job, stdout, stderr, prev_std)

    # Catch any remaining output
    try:
        proc.stdout.flush()
    except ValueError:  # Handle if stdout is closed
        pass
    stdout, stderr, prev_std = check_output(job, stdout, stderr, prev_std)

    # tar/zip up the generated content for bulk downloads
    def get_valid_file(cwd, name, ext):
        out = os.path.join(cwd, name)
        index = 0
        while os.path.exists(six.u('{}.{}').format(out, ext)):
            index += 1
            out = os.path.join(cwd, six.u('{}_{}').format(name, index))
        return six.u('{}.{}').format(out, ext)

    # fetch the job again in case the database connection was lost during the job or something else changed.
    job = WooeyJob.objects.get(pk=job_id)
    # if there are files generated, make zip/tar files for download
    if len(os.listdir(abscwd)):
        tar_out = get_valid_file(abscwd, get_valid_filename(job.job_name), 'tar.gz')
        tar = tarfile.open(tar_out, "w:gz")
        tar_name = os.path.splitext(os.path.splitext(os.path.split(tar_out)[1])[0])[0]
        tar.add(abscwd, arcname=tar_name)
        tar.close()

        zip_out = get_valid_file(abscwd, get_valid_filename(job.job_name), 'zip')
        zip = zipfile.ZipFile(zip_out, "w")
        arcname = os.path.splitext(os.path.split(zip_out)[1])[0]
        zip.write(abscwd, arcname=arcname)
        for root, folders, filenames in os.walk(os.path.split(zip_out)[0]):
            for filename in filenames:
                path = os.path.join(root, filename)
                if path == tar_out:
                    continue
                if path == zip_out:
                    continue
                try:
                    zip.write(path, arcname=os.path.join(arcname, filename))
                except:
                    stderr = '{}\n{}'.format(stderr, traceback.format_exc())
        try:
            zip.close()
        except:
            stderr = '{}\n{}'.format(stderr, traceback.format_exc())

        # save all the files generated as well to our default storage for ephemeral storage setups
        if wooey_settings.WOOEY_EPHEMERAL_FILES:
            for root, folders, files in os.walk(abscwd):
                for filename in files:
                    filepath = os.path.join(root, filename)
                    s3path = os.path.join(root[root.find(cwd):], filename)
                    remote = utils.get_storage(local=False)
                    exists = remote.exists(s3path)
                    filesize = remote.size(s3path) if exists else 0
                    if not exists or (exists and filesize == 0):
                        if exists:
                            remote.delete(s3path)
                        remote.save(s3path, File(open(filepath, 'rb')))
    utils.create_job_fileinfo(job)

    job.stdout = stdout
    job.stderr = stderr
    job.status = WooeyJob.COMPLETED
    job.update_realtime(delete=True)
    job.save()

    return (stdout, stderr)

@celery_app.task(base=WooeyTask)
def cleanup_wooey_jobs(**kwargs):
    from django.utils import timezone
    from .models import WooeyJob

    cleanup_settings = wooey_settings.WOOEY_JOB_EXPIRATION
    anon_settings = cleanup_settings.get('anonymous')
    now = timezone.now()
    if anon_settings:
        WooeyJob.objects.filter(user=None, created_date__lte=now-anon_settings).delete()
    user_settings = cleanup_settings.get('user')
    if user_settings:
        WooeyJob.objects.filter(user__isnull=False, created_date__lte=now-user_settings).delete()

celery_app.conf.update(
    CELERYBEAT_SCHEDULE={
        'cleanup-old-jobs': {
            'task': 'wooey.tasks.cleanup_wooey_jobs',
            'schedule': crontab(hour=0, minute=0),  # cleanup at midnight each day
        },
    }
)
<|endoftext|>


<commit_before>#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

"""
The dispatcher is responsible for moving a job through the build workflow.

It creates jobs, assigns them to nodes, and monitors the progress of the jobs.
Status updates are routed back to clients and to the database.
"""

import copy
import errno
import logging
import os
import stat
from rmake.core.config import DispatcherConfig
from rmake.core.handler import getHandlerClass
from rmake.db import database
from rmake.errors import RmakeError
from rmake.lib import dbpool
from rmake.lib import rpc_pickle
from rmake.lib.apirpc import RPCServer, expose
from rmake.lib.daemon import setDebugHook
from rmake.lib.logger import logFailure
from rmake.lib.twisted_extras.ipv6 import TCP6Server
from rmake.lib.uuid import UUID
from rmake.messagebus import message
from rmake.messagebus.client import BusService
from rmake.messagebus.interact import InteractiveHandler
from twisted.application.internet import UNIXServer
from twisted.application.service import MultiService
from twisted.internet import defer
from twisted.web.resource import Resource
from twisted.web.server import Site

log = logging.getLogger(__name__)

class DispatcherBusService(BusService):

    role = 'dispatcher'
    description = 'rMake Dispatcher'

    def __init__(self, dispatcher, cfg):
        self.dispatcher = dispatcher
        BusService.__init__(self, cfg, other_handlers={
            'interactive': DispatcherInteractiveHandler(),
            })

    def messageReceived(self, msg):
        if isinstance(msg, message.TaskStatus):
            self.dispatcher.updateTask(msg.task)
        else:
            BusService.messageReceived(self, msg)

class DispatcherInteractiveHandler(InteractiveHandler):

    def interact_show(self, msg, words):
        what = words.pop(0)
        if what == 'job':
            uuids = []
            for uuid in words:
                try:
                    uuids.append(UUID(uuid))
                except ValueError:
                    return "Bad UUID '%s'" % (uuid,)

            d = self.parent.parent.getJobs(None, uuids)

            @d.addCallback
            def on_get(jobs):
                out = ['']
                for uuid, job in zip(uuids, jobs):
                    if job is not None:
                        out.append('Job %s ' % job.job_uuid)
                        out.append('  Type: %s' % job.job_type)
                        out.append('  Owner: %s' % job.owner)
                    else:
                        out.append('Job %s not found' % uuid)
                return '\n'.join(out)

            return d
        else:
            return "Usage: show job <uuid>"

class Dispatcher(MultiService, RPCServer):

    def __init__(self, cfg):
        MultiService.__init__(self)

        self.pool = dbpool.ConnectionPool(cfg.databaseUrl)
        self.db = database.Database(cfg.databaseUrl,
                dbpool.PooledDatabaseProxy(self.pool))

        self.bus = DispatcherBusService(self, cfg)
        self.bus.setServiceParent(self)

        self.jobs = {}

        root = Resource()
        root.putChild('picklerpc', rpc_pickle.PickleRPCResource(self))
        site = Site(root)
        if cfg.listenPath:
            try:
                st = os.lstat(cfg.listenPath)
            except OSError, err:
                if err.errno != errno.ENOENT:
                    raise
            else:
                if not stat.S_ISSOCK(st.st_mode):
                    raise RuntimeError("Path '%s' exists but is not a socket" %
                            (cfg.listenPath,))
                os.unlink(cfg.listenPath)

            UNIXServer(cfg.listenPath, site).setServiceParent(self)
        if cfg.listenPort:
            TCP6Server(cfg.listenPort, site, interface=cfg.listenAddress
                    ).setServiceParent(self)

    ## Client API

    @expose
    def getJobs(self, callData, job_uuids):
        return self.pool.runWithTransaction(self.db.core.getJobs, job_uuids)

    @expose
    def createJob(self, callData, job):
        #job = copy.deepcopy(job)
        try:
            handlerClass = getHandlerClass(job.job_type)
        except KeyError:
            raise RmakeError("Job type %r is unsupported" % job.job_type)

        d = self.pool.runWithTransaction(self.db.core.createJob, job)
        @d.addCallback
        def post_create(newJob):
            log.info("Job %s of type %r started", newJob.job_uuid,
                    newJob.job_type)
            handler = handlerClass(self, newJob)
            self.jobs[newJob.job_uuid] = handler
            handler.start()
            return newJob
        return d

    # Job handler API

    def jobDone(self, job_uuid):
        if job_uuid in self.jobs:
            status = self.jobs[job_uuid].job.status
            if status.completed:
                result = 'done'
            elif status.failed:
                result = 'failed'
            else:
                result = 'finished'
            log.info("Job %s %s: %s", job_uuid, result, status.text)
            del self.jobs[job_uuid]
        else:
            log.warning("Tried to remove job %s but it is already gone.",
                    job_uuid)

    def _createJob(self, job):
        job = self.db.core.createJob(job)
        return job

    def updateJob(self, job, frozen=None):
        # TODO: pass update params directly because deepcopy is super expensive
        job = copy.deepcopy(job)
        return self.pool.runWithTransaction(self._updateJob, job,
                frozen=frozen)

    def _updateJob(self, job, frozen=None):
        self.db.core.updateJob(job, frozen=frozen)
        if job.status.final:
            self.jobDone(job.job_uuid)

    def createTask(self, task):
        job = self.jobs[task.job_uuid]
        if task.task_uuid in job.tasks:
            return defer.succeed(job.tasks[task.task_uuid])
        job.tasks[task.task_uuid] = task

        # Try to assign before saving so we can avoid a second trip to the DB
        self._assignTask(task)

        task = copy.deepcopy(task)
        d = self.pool.runWithTransaction(self.db.core.createTaskMaybe, task)
        @d.addCallback
        def post_create(newTask):
            job.tasks[newTask.task_uuid] = newTask
            return newTask
        return d

    ## Message bus API

    def updateTask(self, task):
        d = self.pool.runWithTransaction(self.db.core.updateTask, task)
        @d.addCallback
        def task_updated(newTask):
            handler = self.jobs.get(task.job_uuid)
            if not handler:
                return
            handler.taskUpdated(newTask)
        d.addErrback(logFailure)

    ## Task assignment

    def _assignTask(self, task):
        from twisted.words.protocols.jabber.jid import JID
        node = task.node_assigned = JID(
                'e962bdb07ee8bacc1087d38a8b07642f@dhcp196.eng.rpath.com/rmake')
        msg = message.StartTask(copy.deepcopy(task))
        msg.send(self.bus, node)

def main():
    import optparse
    import sys

    cfg = DispatcherConfig()
    parser = optparse.OptionParser()
    parser.add_option('--debug', action='store_true')
    parser.add_option('-c', '--config-file', action='callback', type='str',
            callback=lambda a, b, value, c: cfg.read(value))
    parser.add_option('--config', action='callback', type='str',
            callback=lambda a, b, value, c: cfg.configLine(value))
    options, args = parser.parse_args()
    if args:
        parser.error("No arguments expected")

    for name in ('xmppJID', 'xmppIdentFile'):
        if cfg[name] is None:
            sys.exit("error: Configuration option %r must be set." % name)

    from rmake.lib.logger import setupLogging
    setupLogging(consoleLevel=(options.debug and logging.DEBUG or
        logging.INFO), consoleFormat='file', withTwisted=True)
    setDebugHook()

    from twisted.internet import reactor
    service = Dispatcher(cfg)
    if options.debug:
        service.bus.logTraffic = True
    service.startService()
    reactor.run()

if __name__ == '__main__':
    main()
<commit_msg>Track connected worker nodes and expire when no heartbeat is received.<commit_after>#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

"""
The dispatcher is responsible for moving a job through the build workflow.

It creates jobs, assigns them to nodes, and monitors the progress of the jobs.
Status updates are routed back to clients and to the database.
"""

import copy
import errno
import logging
import os
import stat
import time
from rmake.core.config import DispatcherConfig
from rmake.core.handler import getHandlerClass
from rmake.db import database
from rmake.errors import RmakeError
from rmake.lib import dbpool
from rmake.lib import rpc_pickle
from rmake.lib.apirpc import RPCServer, expose
from rmake.lib.daemon import setDebugHook
from rmake.lib.logger import logFailure
from rmake.lib.twisted_extras.ipv6 import TCP6Server
from rmake.lib.uuid import UUID
from rmake.messagebus import message
from rmake.messagebus.client import BusService
from rmake.messagebus.interact import InteractiveHandler
from twisted.application.internet import TimerService, UNIXServer
from twisted.application.service import MultiService
from twisted.internet import defer
from twisted.web.resource import Resource
from twisted.web.server import Site
from twisted.words.protocols.jabber.jid import internJID

log = logging.getLogger(__name__)

class DispatcherBusService(BusService):

    role = 'dispatcher'
    description = 'rMake Dispatcher'

    def __init__(self, dispatcher, cfg):
        self.dispatcher = dispatcher
        BusService.__init__(self, cfg, other_handlers={
            'interactive': DispatcherInteractiveHandler(),
            })

    def messageReceived(self, msg):
        if isinstance(msg, message.TaskStatus):
            self.dispatcher.updateTask(msg.task)
        elif isinstance(msg, message.Heartbeat):
            sender = internJID(msg.info.sender)
            self.dispatcher.workerHeartbeat(sender, msg.caps, msg.tasks)
        else:
            BusService.messageReceived(self, msg)

    def onPresence(self, presence):
        if not presence.available:
            self.dispatcher.workerDown(presence.sender)

class DispatcherInteractiveHandler(InteractiveHandler):

    def interact_show(self, msg, words):
        what = words.pop(0)
        if what == 'job':
            uuids = []
            for uuid in words:
                try:
                    uuids.append(UUID(uuid))
                except ValueError:
                    return "Bad UUID '%s'" % (uuid,)

            d = self.parent.parent.getJobs(None, uuids)

            @d.addCallback
            def on_get(jobs):
                out = ['']
                for uuid, job in zip(uuids, jobs):
                    if job is not None:
                        out.append('Job %s ' % job.job_uuid)
                        out.append('  Type: %s' % job.job_type)
                        out.append('  Owner: %s' % job.owner)
                    else:
                        out.append('Job %s not found' % uuid)
                return '\n'.join(out)

            return d
        else:
            return "Usage: show job <uuid>"

class Dispatcher(MultiService, RPCServer):

    def __init__(self, cfg):
        MultiService.__init__(self)

        self.pool = dbpool.ConnectionPool(cfg.databaseUrl)
        self.db = database.Database(cfg.databaseUrl,
                dbpool.PooledDatabaseProxy(self.pool))

        self.bus = DispatcherBusService(self, cfg)
        self.bus.setServiceParent(self)

        self.jobs = {}
        self.workers = {}

        WorkerChecker(self).setServiceParent(self)

        root = Resource()
        root.putChild('picklerpc', rpc_pickle.PickleRPCResource(self))
        site = Site(root)
        if cfg.listenPath:
            try:
                st = os.lstat(cfg.listenPath)
            except OSError, err:
                if err.errno != errno.ENOENT:
                    raise
            else:
                if not stat.S_ISSOCK(st.st_mode):
                    raise RuntimeError("Path '%s' exists but is not a socket" %
                            (cfg.listenPath,))
                os.unlink(cfg.listenPath)

            UNIXServer(cfg.listenPath, site).setServiceParent(self)
        if cfg.listenPort:
            TCP6Server(cfg.listenPort, site, interface=cfg.listenAddress
                    ).setServiceParent(self)

    ## Client API

    @expose
    def getJobs(self, callData, job_uuids):
        return self.pool.runWithTransaction(self.db.core.getJobs, job_uuids)

    @expose
    def createJob(self, callData, job):
        #job = copy.deepcopy(job)
        try:
            handlerClass = getHandlerClass(job.job_type)
        except KeyError:
            raise RmakeError("Job type %r is unsupported" % job.job_type)

        d = self.pool.runWithTransaction(self.db.core.createJob, job)
        @d.addCallback
        def post_create(newJob):
            log.info("Job %s of type %r started", newJob.job_uuid,
                    newJob.job_type)
            handler = handlerClass(self, newJob)
            self.jobs[newJob.job_uuid] = handler
            handler.start()
            return newJob
        return d

    # Job handler API

    def jobDone(self, job_uuid):
        if job_uuid in self.jobs:
            status = self.jobs[job_uuid].job.status
            if status.completed:
                result = 'done'
            elif status.failed:
                result = 'failed'
            else:
                result = 'finished'
            log.info("Job %s %s: %s", job_uuid, result, status.text)
            del self.jobs[job_uuid]
        else:
            log.warning("Tried to remove job %s but it is already gone.",
                    job_uuid)

    def _createJob(self, job):
        job = self.db.core.createJob(job)
        return job

    def updateJob(self, job, frozen=None):
        # TODO: pass update params directly because deepcopy is super expensive
        job = copy.deepcopy(job)
        return self.pool.runWithTransaction(self._updateJob, job,
                frozen=frozen)

    def _updateJob(self, job, frozen=None):
        self.db.core.updateJob(job, frozen=frozen)
        if job.status.final:
            self.jobDone(job.job_uuid)

    def createTask(self, task):
        job = self.jobs[task.job_uuid]
        if task.task_uuid in job.tasks:
            return defer.succeed(job.tasks[task.task_uuid])
        job.tasks[task.task_uuid] = task

        # Try to assign before saving so we can avoid a second trip to the DB
        self._assignTask(task)

        task = copy.deepcopy(task)
        d = self.pool.runWithTransaction(self.db.core.createTaskMaybe, task)
        @d.addCallback
        def post_create(newTask):
            job.tasks[newTask.task_uuid] = newTask
            return newTask
        return d

    ## Message bus API

    def updateTask(self, task):
        d = self.pool.runWithTransaction(self.db.core.updateTask, task)
        @d.addCallback
        def task_updated(newTask):
            handler = self.jobs.get(task.job_uuid)
            if not handler:
                return
            handler.taskUpdated(newTask)
        d.addErrback(logFailure)

    def workerHeartbeat(self, jid, caps, tasks):
        worker = self.workers.get(jid)
        if worker is None:
            log.info("Worker %s connected: caps=%r", jid.full(), caps)
            worker = self.workers[jid] = WorkerInfo(jid)
        worker.setCaps(caps)
        self._assignTasks()

    def workerDown(self, jid):
        if jid in self.workers:
            log.info("Worker %s disconnected", jid.full())
            del self.workers[jid]

    ## Task assignment

    def _assignTasks(self):
        # FIXME
        pass

    def _assignTask(self, task):
        from twisted.words.protocols.jabber.jid import JID
        node = task.node_assigned = JID(
                'e962bdb07ee8bacc1087d38a8b07642f@dhcp196.eng.rpath.com/rmake')
        msg = message.StartTask(copy.deepcopy(task))
        msg.send(self.bus, node)

class WorkerChecker(TimerService):

    threshold = 4

    def __init__(self, dispatcher):
        TimerService.__init__(self, 5, self.checkWorkers)
        self.dispatcher = dispatcher

    def checkWorkers(self):
        for info in self.dispatcher.workers.values():
            info.expiring += 1
            if info.expiring > self.threshold:
                self.dispatcher.workerDown(info.jid)

class WorkerInfo(object):

    def __init__(self, jid):
        self.jid = jid
        self.caps = set()
        self.tasks = set()
        # expiring is incremented each time WorkerChecker runs and zeroed each
        # time the worker heartbeats. When it gets high enough, the worker is
        # assumed dead.
        self.expiring = 0

    def setCaps(self, caps):
        self.caps = caps
        self.expiring = 0

def main():
    import optparse
    import sys

    cfg = DispatcherConfig()
    parser = optparse.OptionParser()
    parser.add_option('--debug', action='store_true')
    parser.add_option('-c', '--config-file', action='callback', type='str',
            callback=lambda a, b, value, c: cfg.read(value))
    parser.add_option('--config', action='callback', type='str',
            callback=lambda a, b, value, c: cfg.configLine(value))
    options, args = parser.parse_args()
    if args:
        parser.error("No arguments expected")

    for name in ('xmppJID', 'xmppIdentFile'):
        if cfg[name] is None:
            sys.exit("error: Configuration option %r must be set." % name)

    from rmake.lib.logger import setupLogging
    setupLogging(consoleLevel=(options.debug and logging.DEBUG or
        logging.INFO), consoleFormat='file', withTwisted=True)
    setDebugHook()

    from twisted.internet import reactor
    service = Dispatcher(cfg)
    if options.debug:
        service.bus.logTraffic = True
    service.startService()
    reactor.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
import itertools
import time

from conary import conaryclient
from conary.conaryclient import resolve
from conary.lib import log
from conary.local import database
from conary.repository import trovesource

from rmake.lib import apiutils
from rmake.lib.apiutils import register, freeze, thaw
from rmake.worker import resolvesource

class ResolveResult(object):
    def __init__(self, inCycle=False):
        self.success = False
        self.buildReqs = []
        self.missingBuildReqs = []
        self.missingDeps = []
        self.inCycle = inCycle

    def getBuildReqs(self):
        assert(self.success)
        return self.buildReqs

    def getMissingBuildReqs(self):
        return self.missingBuildReqs

    def getMissingDeps(self):
        return self.missingDeps

    def hasMissingBuildReqs(self):
        return bool(self.missingBuildReqs)

    def hasMissingDeps(self):
        return bool(self.missingDeps)

    def troveResolved(self, buildReqs):
        self.success = True
        self.buildReqs = buildReqs

    def troveMissingBuildReqs(self, buildReqs):
        self.success = False
        self.missingBuildReqs = buildReqs

    def troveMissingDependencies(self, missingDeps):
        self.success = False
        self.missingDeps = missingDeps

    def __freeze__(self):
        d = self.__dict__.copy()
        d['buildReqs'] = freeze('installJobList', self.buildReqs)
        d['missingDeps'] = freeze('dependencyList', self.missingDeps)
        return d

    @classmethod
    def __thaw__(class_, d):
        self = class_()
        self.__dict__.update(d)
        self.buildReqs = thaw('installJobList', self.buildReqs)
        self.missingDeps = thaw('dependencyList', self.missingDeps)
        return self
register(ResolveResult)

class DependencyResolver(object):
    """
        Resolves dependencies for one trove.
    """
    def __init__(self, logger, repos=None):
        self.logger = logger
        self.repos = repos

    def getSources(self, resolveJob):
        cfg = resolveJob.getConfig()
        builtTroves = self.repos.getTroves(resolveJob.getBuiltTroves(),
                                           withFiles=False)
        builtTroveSource = resolvesource.BuiltTroveSource(builtTroves)
        if cfg.resolveTroveTups:
            return self.getSourcesWithResolveTroves(cfg, cfg.resolveTroveTups,
                                                    builtTroveSource)

        searchSource = resolvesource.DepHandlerSource(builtTroveSource,
                                                      [], self.repos)
        resolveSource = resolvesource.DepResolutionByTroveLists(cfg, None, [])
        return searchSource, resolveSource

    def getSourcesWithResolveTroves(self, cfg, resolveTroveTups,
                                    builtTroveSource):
        resolveTroves = []
        searchSourceTroves = []
        allResolveTroveTups = list(itertools.chain(*cfg.resolveTroveTups))
        allResolveTroves = self.repos.getTroves(allResolveTroveTups,
                                                withFiles=False)
        resolveTrovesByTup = dict((x.getNameVersionFlavor(), x)
                                  for x in allResolveTroves)

        for resolveTupList in cfg.resolveTroveTups:
            resolveTroves = [ resolveTrovesByTup[x]
                              for x in resolveTupList ]
            searchSourceTroves.append(resolveTroves)

        searchSource = resolvesource.DepHandlerSource(builtTroveSource,
                           searchSourceTroves,
                           self.repos,
                           useInstallLabelPath=not cfg.resolveTrovesOnly)
        resolveSource = resolvesource.DepResolutionByTroveLists(cfg, None,
                                                        searchSourceTroves)
        return searchSource, resolveSource

    def resolve(self, resolveJob):
        """
            Find the set of troves that must be installed for the set
            of buildreqs associated with this trove.

            Searches for build req and runtime req solutions in the following
            order:

            1. Search the group, w/o consideration of order
            2. Search the label of the trove we're building, followed
               by the reset of the labelPath
            2. Search the label path.
        """
        log.setMinVerbosity(log.DEBUG)
        trv = resolveJob.getTrove()
        cfg = resolveJob.getConfig()
        client = conaryclient.ConaryClient(cfg)
        if not self.repos:
            self.repos = client.repos
        else:
            client.repos = self.repos

        if cfg.resolveTrovesOnly:
            installLabelPath = None
            searchFlavor = None
        else:
            installLabelPath = cfg.installLabelPath
            searchFlavor = cfg.flavor

        searchSource, resolveSource = self.getSources(resolveJob)

        self.logger.debug('attempting to resolve buildreqs for %s=%s[%s]' % resolveJob.getTrove().getNameVersionFlavor())

        resolveResult = ResolveResult(inCycle=resolveJob.inCycle)

        if not trv.getBuildRequirements():
            resolveResult.troveResolved([])
            return resolveResult

        finalToInstall = {}

        # we allow build requirements to be matched against anywhere on the
        # install label.  Create a list of all of this trove's labels,
        # from latest on branch to earliest to use as search labels.
        self.logger.debug('   finding buildreqs for %s....' % trv.getName())
        start = time.time()

        result = searchSource.findTroves(installLabelPath,
                                         trv.getBuildRequirementSpecs(),
                                         searchFlavor, allowMissing=True,
                                         acrossLabels=False)

        # don't follow redirects when resolving buildReqs
        result = searchSource.findTroves(installLabelPath,
                                     trv.getBuildRequirementSpecs(),
                                     searchFlavor, allowMissing=True,
                                     acrossLabels=False,
                                     troveTypes=trovesource.TROVE_QUERY_NORMAL)
        okay = True

        buildReqTups = []
        missingBuildReqs = []
        for troveSpec in trv.getBuildRequirementSpecs():
            solutions = result.get(troveSpec, [])
            if not solutions:
                missingBuildReqs.append(troveSpec)
                okay = False
            else:
                sol = _findBestSolution(trv, troveSpec, solutions, searchFlavor)
                buildReqTups.append(sol)

        if not okay:
            self.logger.debug('could not find all buildreqs')
            resolveResult.troveMissingBuildReqs(missingBuildReqs)
            return resolveResult

        self.logger.debug('   resolving deps for %s...' % trv.getName())
        start = time.time()
        itemList = [ (x[0], (None, None), (x[1], x[2]), True)
                                                for x in buildReqTups ]

        resolveSource.setLabelPath(installLabelPath)

        uJob = database.UpdateJob(None)
        uJob.setSearchSource(searchSource)
        jobSet = client._updateChangeSet(itemList, uJob, useAffinity=False)
        (depList, suggMap, cannotResolve, splitJob, keepList) = \
        client.resolver.resolveDependencies(uJob, jobSet,
                                            resolveDeps=True,
                                            useRepos=False, split=False,
                                            resolveSource=resolveSource)

        jobSet.update((x[0], (None, None), (x[1], x[2]), False)
                      for x in itertools.chain(*suggMap.itervalues()))
        if cannotResolve or depList:
            self.logger.debug('Failed - unresolved deps - took %s seconds' % (time.time() - start))
            resolveResult.troveMissingDependencies(depList + cannotResolve)
            return resolveResult

        self._addPackages(searchSource, jobSet)
        self.logger.debug('   took %s seconds' % (time.time() - start))
        resolveResult.troveResolved(jobSet)
        return resolveResult

    def _addPackages(self, searchSource, jobSet):
        """
            Adds packages for buildReqs we're installing (if they're
            not already added)
        """
        neededPackages = set((x[0].split(':')[0], x[2][0], x[2][1])
                             for x in jobSet)
        neededPackages.difference_update((x[0], x[2][0], x[2][1])
                                         for x in jobSet)
        hasTroves = searchSource.hasTroves(neededPackages)
        if isinstance(hasTroves, list):
            hasTroves = dict(itertools.izip(neededPackages, hasTroves))
        jobSet.update([(x[0][0], (None, None), (x[0][1], x[0][2]), False)
                       for x in hasTroves.items() if x[1]])

def _findBestSolution(trove, (name, versionSpec, flavorSpec),
                      solutions, searchFlavor):
    """Given a trove, a buildRequirement troveSpec, and a set of troves
       that may match that buildreq, find the best trove.
    """
    if len(solutions) == 1:
        return solutions[0]
    # flavorSpec _should_ have been handled by findTroves.
    # However, in some cases it's not - for example, if two different
    # flavors of a trove are in the same group (glibc, e.g.).
    filter = resolve.DepResolutionMethod(None, None)
    result = None
    affDict = dict.fromkeys(x[0] for x in solutions)
    for flavor in searchFlavor:
        result = filter.selectResolutionTrove(trove, None, None,
                                              solutions, flavor,
                                              affDict)
        if result:
            break
    return result

<commit_msg>Output result of dep resolution in RESOLVE log file<commit_after>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
import itertools
import time

from conary import conaryclient
from conary.conaryclient import resolve
from conary.lib import log
from conary.local import database
from conary.repository import trovesource

from rmake.lib import apiutils
from rmake.lib.apiutils import register, freeze, thaw
from rmake.worker import resolvesource

class ResolveResult(object):
    def __init__(self, inCycle=False):
        self.success = False
        self.buildReqs = []
        self.missingBuildReqs = []
        self.missingDeps = []
        self.inCycle = inCycle

    def getBuildReqs(self):
        assert(self.success)
        return self.buildReqs

    def getMissingBuildReqs(self):
        return self.missingBuildReqs

    def getMissingDeps(self):
        return self.missingDeps

    def hasMissingBuildReqs(self):
        return bool(self.missingBuildReqs)

    def hasMissingDeps(self):
        return bool(self.missingDeps)

    def troveResolved(self, buildReqs):
        self.success = True
        self.buildReqs = buildReqs

    def troveMissingBuildReqs(self, buildReqs):
        self.success = False
        self.missingBuildReqs = buildReqs

    def troveMissingDependencies(self, missingDeps):
        self.success = False
        self.missingDeps = missingDeps

    def __freeze__(self):
        d = self.__dict__.copy()
        d['buildReqs'] = freeze('installJobList', self.buildReqs)
        d['missingDeps'] = freeze('dependencyList', self.missingDeps)
        return d

    @classmethod
    def __thaw__(class_, d):
        self = class_()
        self.__dict__.update(d)
        self.buildReqs = thaw('installJobList', self.buildReqs)
        self.missingDeps = thaw('dependencyList', self.missingDeps)
        return self
register(ResolveResult)

class DependencyResolver(object):
    """
        Resolves dependencies for one trove.
    """
    def __init__(self, logger, repos=None):
        self.logger = logger
        self.repos = repos

    def getSources(self, resolveJob):
        cfg = resolveJob.getConfig()
        builtTroves = self.repos.getTroves(resolveJob.getBuiltTroves(),
                                           withFiles=False)
        builtTroveSource = resolvesource.BuiltTroveSource(builtTroves)
        if cfg.resolveTroveTups:
            return self.getSourcesWithResolveTroves(cfg, cfg.resolveTroveTups,
                                                    builtTroveSource)

        searchSource = resolvesource.DepHandlerSource(builtTroveSource,
                                                      [], self.repos)
        resolveSource = resolvesource.DepResolutionByTroveLists(cfg, None, [])
        return searchSource, resolveSource

    def getSourcesWithResolveTroves(self, cfg, resolveTroveTups,
                                    builtTroveSource):
        resolveTroves = []
        searchSourceTroves = []
        allResolveTroveTups = list(itertools.chain(*cfg.resolveTroveTups))
        allResolveTroves = self.repos.getTroves(allResolveTroveTups,
                                                withFiles=False)
        resolveTrovesByTup = dict((x.getNameVersionFlavor(), x)
                                  for x in allResolveTroves)

        for resolveTupList in cfg.resolveTroveTups:
            resolveTroves = [ resolveTrovesByTup[x]
                              for x in resolveTupList ]
            searchSourceTroves.append(resolveTroves)

        searchSource = resolvesource.DepHandlerSource(builtTroveSource,
                           searchSourceTroves,
                           self.repos,
                           useInstallLabelPath=not cfg.resolveTrovesOnly)
        resolveSource = resolvesource.DepResolutionByTroveLists(cfg, None,
                                                        searchSourceTroves)
        return searchSource, resolveSource

    def resolve(self, resolveJob):
        """
            Find the set of troves that must be installed for the set
            of buildreqs associated with this trove.

            Searches for build req and runtime req solutions in the following
            order:

            1. Search the group, w/o consideration of order
            2. Search the label of the trove we're building, followed
               by the reset of the labelPath
            2. Search the label path.
        """
        log.setMinVerbosity(log.DEBUG)
        trv = resolveJob.getTrove()
        cfg = resolveJob.getConfig()
        client = conaryclient.ConaryClient(cfg)
        if not self.repos:
            self.repos = client.repos
        else:
            client.repos = self.repos

        if cfg.resolveTrovesOnly:
            installLabelPath = None
            searchFlavor = None
        else:
            installLabelPath = cfg.installLabelPath
            searchFlavor = cfg.flavor

        searchSource, resolveSource = self.getSources(resolveJob)

        self.logger.debug('attempting to resolve buildreqs for %s=%s[%s]' % resolveJob.getTrove().getNameVersionFlavor())

        resolveResult = ResolveResult(inCycle=resolveJob.inCycle)

        if not trv.getBuildRequirements():
            resolveResult.troveResolved([])
            return resolveResult

        finalToInstall = {}

        # we allow build requirements to be matched against anywhere on the
        # install label.  Create a list of all of this trove's labels,
        # from latest on branch to earliest to use as search labels.
        self.logger.debug('   finding buildreqs for %s....' % trv.getName())
        start = time.time()

        result = searchSource.findTroves(installLabelPath,
                                         trv.getBuildRequirementSpecs(),
                                         searchFlavor, allowMissing=True,
                                         acrossLabels=False)

        # don't follow redirects when resolving buildReqs
        result = searchSource.findTroves(installLabelPath,
                                     trv.getBuildRequirementSpecs(),
                                     searchFlavor, allowMissing=True,
                                     acrossLabels=False,
                                     troveTypes=trovesource.TROVE_QUERY_NORMAL)
        okay = True

        buildReqTups = []
        missingBuildReqs = []
        for troveSpec in trv.getBuildRequirementSpecs():
            solutions = result.get(troveSpec, [])
            if not solutions:
                missingBuildReqs.append(troveSpec)
                okay = False
            else:
                sol = _findBestSolution(trv, troveSpec, solutions, searchFlavor)
                buildReqTups.append(sol)

        if not okay:
            self.logger.debug('could not find all buildreqs: %s' % (missingBuildReqs,))
            resolveResult.troveMissingBuildReqs(missingBuildReqs)
            return resolveResult

        self.logger.debug('   resolving deps for %s...' % trv.getName())
        start = time.time()
        itemList = [ (x[0], (None, None), (x[1], x[2]), True)
                                                for x in buildReqTups ]

        resolveSource.setLabelPath(installLabelPath)

        uJob = database.UpdateJob(None)
        uJob.setSearchSource(searchSource)
        jobSet = client._updateChangeSet(itemList, uJob, useAffinity=False)
        (depList, suggMap, cannotResolve, splitJob, keepList) = \
        client.resolver.resolveDependencies(uJob, jobSet,
                                            resolveDeps=True,
                                            useRepos=False, split=False,
                                            resolveSource=resolveSource)

        jobSet.update((x[0], (None, None), (x[1], x[2]), False)
                      for x in itertools.chain(*suggMap.itervalues()))
        if cannotResolve or depList:
            self.logger.debug('Failed - unresolved deps - took %s seconds' % (time.time() - start))
            self.logger.debug('Missing: %s' % ((depList + cannotResolve),))
            resolveResult.troveMissingDependencies(depList + cannotResolve)
            return resolveResult

        self._addPackages(searchSource, jobSet)
        self.logger.debug('   took %s seconds' % (time.time() - start))
        self.logger.info('   Resolved troves:')
        self.logger.info('\n    '.join('%s=%s[%s]' % (x[0], x[2][0], x[2][1])
                               for x in sorted(jobSet)))
        resolveResult.troveResolved(jobSet)
        return resolveResult

    def _addPackages(self, searchSource, jobSet):
        """
            Adds packages for buildReqs we're installing (if they're
            not already added)
        """
        neededPackages = set((x[0].split(':')[0], x[2][0], x[2][1])
                             for x in jobSet)
        neededPackages.difference_update((x[0], x[2][0], x[2][1])
                                         for x in jobSet)
        hasTroves = searchSource.hasTroves(neededPackages)
        if isinstance(hasTroves, list):
            hasTroves = dict(itertools.izip(neededPackages, hasTroves))
        jobSet.update([(x[0][0], (None, None), (x[0][1], x[0][2]), False)
                       for x in hasTroves.items() if x[1]])

def _findBestSolution(trove, (name, versionSpec, flavorSpec),
                      solutions, searchFlavor):
    """Given a trove, a buildRequirement troveSpec, and a set of troves
       that may match that buildreq, find the best trove.
    """
    if len(solutions) == 1:
        return solutions[0]
    # flavorSpec _should_ have been handled by findTroves.
    # However, in some cases it's not - for example, if two different
    # flavors of a trove are in the same group (glibc, e.g.).
    filter = resolve.DepResolutionMethod(None, None)
    result = None
    affDict = dict.fromkeys(x[0] for x in solutions)
    for flavor in searchFlavor:
        result = filter.selectResolutionTrove(trove, None, None,
                                              solutions, flavor,
                                              affDict)
        if result:
            break
    return result

<|endoftext|>


<commit_before>
"""
argh
"""
from roamer.entry import Entry

class EditDirectory(object):
    """
    argh
    """
    def __init__(self, path, content):
        self.path = path
        self.entries = {}

        for line in content.splitlines():
            name, digest = process_line(line)
            entry = Entry(name, path, digest)
            if digest in self.entries:
                self.entries[digest].append(entry)
            else:
                self.entries[digest] = [entry]

    def find(self, digest):
        return self.entries.get(digest)

def process_line(line):
    columns = line.split('|')
    name = columns[0]
    if name[-1] == ' ':
        name = name[:-1]

    if len(columns) == 1:
        digest = None
    else:
        digest = columns[1].replace(' ', '')
        if digest == '':
            digest = None
    return name, digest
<commit_msg>Handle empty lines<commit_after>
"""
argh
"""
from roamer.entry import Entry

class EditDirectory(object):
    """
    argh
    """
    def __init__(self, path, content):
        self.path = path
        self.entries = {}

        for line in content.splitlines():
            name, digest = process_line(line)
            if name is None:
                continue
            entry = Entry(name, path, digest)
            if digest in self.entries:
                self.entries[digest].append(entry)
            else:
                self.entries[digest] = [entry]

    def find(self, digest):
        return self.entries.get(digest)

def process_line(line):
    columns = line.split('|')
    name = columns[0]
    if name[-1] == ' ':
        name = name[:-1]
    if name.isspace() or name == '':
        name = None

    if len(columns) == 1:
        digest = None
    else:
        digest = columns[1].replace(' ', '')
        if digest == '':
            digest = None
    return name, digest
<|endoftext|>