<commit_before># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents
"""

from __future__ import absolute_import
import logging
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty

logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

def split_and_html_strip(string):
    """ Compute words from transcriptions """
    words = []
    START = '<'
    END = '>'
    skip = False
    word = ""
    for char in string:
        if char == START:
            skip = True
            continue
        elif char == END:
            skip = False
            continue
        if skip:
            continue
        if char.isalpha():
            word += char
        elif word != "" and len(word) > 3:
            words.append(word)  # word.lower())
            word = ""

    return set(words)

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
    # Fix transcriptions
                words = set()
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({pkey: record, 'images': images}).run()
        logger.info("Insert of record '%s'" % record)

def convert_search():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t3)
    qt2 = query.get_table_query(tin)
    qtin = query.get_table_query(t2in)

    pkey = 'record'
    q = query.get_query()

    if t2in in list(q.table_list().run()):
        q.table_drop(t2in).run()
    q.table_create(t2in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t2in)

    # Query
    res = qt1.group('recordid').order_by('step').run()

    for record, rows in res.items():
        steps = []
        for row in rows:

            # Compose back the elements...
            index = 0
            elements = []
            for myhash in row['hashes']:
                # Query with lambda. COOL!
                cursor = qt2.filter(
                    lambda x: x['fields']['original_hash'].contains(myhash)
                    ).run()
                fields = []
#if not query.is_empty().run():
                try:
                    fields = cursor.next()['fields']
                except DefaultCursorEmpty:
                    logger.warning("No original hash for '%s'" % myhash)

                for field in fields:
                    if field['original_hash'] == myhash:
                        value = None
                        try:
                            value = row['values'][index]
                        except:
                            pass
                        elements.append({
                            'name': field['name'],
                            'position': field['position'],
                            'hash': field['original_hash'],
                            'value': value,
                        })
                        break
                index += 1

            # Create a sane JSON to contain all the data for one step
            steps.append({
                'step': row['step'],
                # Extra info
                'latest_db_info': {
# WHAT ABOUT TIMESTAMP CONVERSION ALSO?
                    'timestamp': row['latest_timestamp'],
                    'ip': row['latest_ipaddress'],
                    'user_id': row['user'],
                },
                'data': elements,
            })

        # Save the record
        qtin.insert({'record': record, 'steps': steps}).run()
        logger.info("Worked off document '%s'" % record)

    # # Create indexes
    # indexes = ['record']
    # existing_indexes = list(qtin.index_list().run())
    # for index in indexes:
    #     if index not in existing_indexes:
    #         qtin.index_create(index).run()
    #         logger.info("Added index '%s'" % index)

def check_indexes(table):

    q = query.get_table_query(table)
    existing_indexes = list(q.index_list().run())
    for index in existing_indexes:
        print(list(q.index_status(index).run()))

def convert_submission():
    """ Convert schema for Steps Submission """

    qt1 = query.get_table_query(t1)
    qt2 = query.get_table_query(t2)
    qtin = query.get_table_query(tin)
    qtin.delete().run()

    # DO
    data = qt1.group("step").run()
    for step in sorted(list(data)):
        new = {"step": None, "fields": None}
        myfilter = {'step': step}
        logger.info("*** STEP: %s" % step)

        # Single step elements
        element = list(qt2.filter(myfilter).run()).pop()
        new['step'] = {"num": step, "name": element['label'],
                       "desc": element['description']}

        # Singles steps fields
        element = []
        fields = list(qt1.filter(myfilter).run())
        sorted_fields = sorted(fields, key=lambda k: k['position'])
        for row in sorted_fields:
            if 'extra' not in row:
                row['extra'] = None

            element.append({
                "name": row['field'],
                "position": row['position'],
                "required": row['required'],
                "type": row['type'],
                "options": row['extra'],
                "original_hash": row['hash'],
            })

        # A whole step
        new["fields"] = element

        # INSERT
        logger.debug("To insert!\n%s" % new)
        qtin.insert(new).run()
        logger.info("Added row")

def test_query():
    """ test queries on rdb """
    # q = query.get_table_query(t2in)
    q = query.get_table_query(t3in)

    # cursor = q \
    #     .concat_map(
    #         lambda doc:
    #             doc['images'].has_fields({'transcriptions': True}).concat_map(
    #                 lambda image: image['transcriptions_split'])) \
    #     .distinct() \
    #     .run()

    cursor = q \
        .concat_map(
            lambda doc: doc['images'].has_fields(
                {'transcriptions': True}).map(
                    lambda image: {
                        'word': image['transcriptions_split'],
                        'record': doc['record'],
                    }
                )).distinct() \
        .filter(lambda mapped: mapped['word'].contains('grati')) \
        .run()

    # print(len(list(cursor)))
    # exit(1)
    for obj in cursor:
        print("TEST", obj)
        exit(1)
    exit(1)

    # # http://stackoverflow.com/a/34647904/2114395
    # cursor = q \
    #     .concat_map(
    #         lambda doc: doc['steps']
    #         .concat_map(lambda step: step['data']
    #                     .concat_map(lambda data:
    #                     [{'record': doc['record'], 'step': data}]))) \
    #     .filter(lambda doc:
    #             doc['step']['value'].match('mog').
    #             and_(doc['step']['name'].match('Numero de page'))) \
    #     .run()

    # for obj in cursor:
    #     print("TEST", obj)
    #     exit(1)

# #TEST1
#     cursor = q \
#         .concat_map(
#             lambda x: x['steps']['data'].map(
#                 lambda item: item['value'])
#         ) \
#         .run()
#     for obj in cursor:
#         print("TEST", obj)
#         exit(1)
#     print(list(cursor))
#     exit(1)

#WORKING FOR RECOVERING DATA
    cursor = q \
        .concat_map(r.row['steps']) \
        .filter(
            lambda row: row['step'] == 3
            ) \
        .concat_map(r.row['data']) \
        .filter(
            lambda row: row['position'] == 1
        ).pluck('value').distinct()['value'].run()
    print(list(cursor))

def convert_schema():
    """ Do all ops """

    test_query()
    convert_submission()
    # check_indexes(t2in)
    convert_search()
    convert_docs()
<commit_msg>Fix conversion script<commit_after># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents
"""

from __future__ import absolute_import
import logging
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty

logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

def split_and_html_strip(string):
    """ Compute words from transcriptions """
    words = []
    START = '<'
    END = '>'
    skip = False
    word = ""
    for char in string:
        if char == START:
            skip = True
            continue
        elif char == END:
            skip = False
            continue
        if skip:
            continue
        if char.isalpha():
            word += char
        elif word != "" and len(word) > 3:
            words.append(word)  # word.lower())
            word = ""

    return set(words)

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
    # Fix transcriptions
                words = set()
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({pkey: record, 'images': images}).run()
        logger.info("Insert of record '%s'" % record)

def convert_search():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t3)
    qt2 = query.get_table_query(tin)
    qtin = query.get_table_query(t2in)

    pkey = 'record'
    q = query.get_query()

    if t2in in list(q.table_list().run()):
        q.table_drop(t2in).run()
    q.table_create(t2in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t2in)

    # Query
    res = qt1.group('recordid').order_by('step').run()

    for record, rows in res.items():
        steps = []
        for row in rows:

            # Compose back the elements...
            index = 0
            elements = []
            for myhash in row['hashes']:
                # Query with lambda. COOL!
                cursor = qt2.filter(
                    lambda x: x['fields']['original_hash'].contains(myhash)
                    ).run()
                fields = []
#if not query.is_empty().run():
                try:
                    fields = cursor.next()['fields']
                except DefaultCursorEmpty:
                    logger.warning("No original hash for '%s'" % myhash)

                for field in fields:
                    if field['original_hash'] == myhash:
                        value = None
                        try:
                            value = row['values'][index]
                        except:
                            pass
                        elements.append({
                            'name': field['name'],
                            'position': field['position'],
                            'hash': field['original_hash'],
                            'value': value,
                        })
                        break
                index += 1

            # Create a sane JSON to contain all the data for one step
            steps.append({
                'step': row['step'],
                # Extra info
                'latest_db_info': {
# WHAT ABOUT TIMESTAMP CONVERSION ALSO?
                    'timestamp': row['latest_timestamp'],
                    'ip': row['latest_ipaddress'],
                    'user_id': row['user'],
                },
                'data': elements,
            })

        # Save the record
        qtin.insert({'record': record, 'steps': steps}).run()
        logger.info("Worked off document '%s'" % record)

    # # Create indexes
    # indexes = ['record']
    # existing_indexes = list(qtin.index_list().run())
    # for index in indexes:
    #     if index not in existing_indexes:
    #         qtin.index_create(index).run()
    #         logger.info("Added index '%s'" % index)

def check_indexes(table):

    q = query.get_table_query(table)
    existing_indexes = list(q.index_list().run())
    for index in existing_indexes:
        print(list(q.index_status(index).run()))

def convert_submission():
    """ Convert schema for Steps Submission """

    qt1 = query.get_table_query(t1)
    qt2 = query.get_table_query(t2)
    qtin = query.get_table_query(tin)
    qtin.delete().run()

    # DO
    data = qt1.group("step").run()
    for step in sorted(list(data)):
        new = {"step": None, "fields": None}
        myfilter = {'step': step}
        logger.info("*** STEP: %s" % step)

        # Single step elements
        element = list(qt2.filter(myfilter).run()).pop()
        new['step'] = {"num": step, "name": element['label'],
                       "desc": element['description']}

        # Singles steps fields
        element = []
        fields = list(qt1.filter(myfilter).run())
        sorted_fields = sorted(fields, key=lambda k: k['position'])
        for row in sorted_fields:
            if 'extra' not in row:
                row['extra'] = None

            element.append({
                "name": row['field'],
                "position": row['position'],
                "required": row['required'],
                "type": row['type'],
                "options": row['extra'],
                "original_hash": row['hash'],
            })

        # A whole step
        new["fields"] = element

        # INSERT
        logger.debug("To insert!\n%s" % new)
        qtin.insert(new).run()
        logger.info("Added row")

def test_query():
    """ test queries on rdb """
    # q = query.get_table_query(t2in)
    q = query.get_table_query(t3in)

    # cursor = q \
    #     .concat_map(
    #         lambda doc:
    #             doc['images'].has_fields({'transcriptions': True}).concat_map(
    #                 lambda image: image['transcriptions_split'])) \
    #     .distinct() \
    #     .run()

    cursor = q \
        .concat_map(
            lambda doc: doc['images'].has_fields(
                {'transcriptions': True}).map(
                    lambda image: {
                        'word': image['transcriptions_split'],
                        'record': doc['record'],
                    }
                )).distinct() \
        .filter(lambda mapped: mapped['word'].contains('grati')) \
        .run()

    # print(len(list(cursor)))
    # exit(1)
    for obj in cursor:
        print("TEST", obj)
        exit(1)
    exit(1)

    # # http://stackoverflow.com/a/34647904/2114395
    # cursor = q \
    #     .concat_map(
    #         lambda doc: doc['steps']
    #         .concat_map(lambda step: step['data']
    #                     .concat_map(lambda data:
    #                     [{'record': doc['record'], 'step': data}]))) \
    #     .filter(lambda doc:
    #             doc['step']['value'].match('mog').
    #             and_(doc['step']['name'].match('Numero de page'))) \
    #     .run()

    # for obj in cursor:
    #     print("TEST", obj)
    #     exit(1)

# #TEST1
#     cursor = q \
#         .concat_map(
#             lambda x: x['steps']['data'].map(
#                 lambda item: item['value'])
#         ) \
#         .run()
#     for obj in cursor:
#         print("TEST", obj)
#         exit(1)
#     print(list(cursor))
#     exit(1)

#WORKING FOR RECOVERING DATA
    cursor = q \
        .concat_map(r.row['steps']) \
        .filter(
            lambda row: row['step'] == 3
            ) \
        .concat_map(r.row['data']) \
        .filter(
            lambda row: row['position'] == 1
        ).pluck('value').distinct()['value'].run()
    print(list(cursor))

def convert_schema():
    """ Do all ops """

    ######################
    # Make rethinkdb query tests
    #test_query()

    ######################
    # Conversion from old structure to the new one
    convert_submission()
    #check_indexes(t2in)
    convert_search()
    convert_docs()
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals
from django.core.exceptions import ImproperlyConfigured
from django.core.urlresolvers import get_callable
from django.utils import six
from django.test.signals import setting_changed

# List Editable Policy
# 0 do not save updated records, save others, show message to the user
# 1 abort whole transaction

CONCURRENCY_LIST_EDITABLE_POLICY_SILENT = 1
CONCURRENCY_LIST_EDITABLE_POLICY_ABORT_ALL = 2
CONCURRENCY_POLICY_RAISE = 4
CONCURRENCY_POLICY_CALLBACK = 8

CONFLICTS_POLICIES = [CONCURRENCY_POLICY_RAISE, CONCURRENCY_POLICY_CALLBACK]
LIST_EDITABLE_POLICIES = [CONCURRENCY_LIST_EDITABLE_POLICY_SILENT, CONCURRENCY_LIST_EDITABLE_POLICY_ABORT_ALL]

class AppSettings(object):
    """
    Class to manage application related settings
    How to use:

    >>> from django.conf import settings
    >>> settings.APP_OVERRIDE = 'overridden'
    >>> settings.MYAPP_CALLBACK = 100
    >>> class MySettings(AppSettings):
    ...     defaults = {'ENTRY1': 'abc', 'ENTRY2': 123, 'OVERRIDE': None, 'CALLBACK':10}
    ...     def set_CALLBACK(self, value):
    ...         setattr(self, 'CALLBACK', value*2)

    >>> conf = MySettings("APP")
    >>> conf.ENTRY1, settings.APP_ENTRY1
    ('abc', 'abc')
    >>> conf.OVERRIDE, settings.APP_OVERRIDE
    ('overridden', 'overridden')

    >>> conf = MySettings("MYAPP")
    >>> conf.ENTRY2, settings.MYAPP_ENTRY2
    (123, 123)
    >>> conf = MySettings("MYAPP")
    >>> conf.CALLBACK
    200

    """
    defaults = {
        'SANITY_CHECK': True,
        'FIELD_SIGNER': 'concurrency.forms.VersionFieldSigner',
        'POLICY': CONCURRENCY_LIST_EDITABLE_POLICY_SILENT | CONCURRENCY_POLICY_RAISE,
        'CALLBACK': 'concurrency.views.callback',
        'HANDLER409': 'concurrency.views.conflict'}

    def __init__(self, prefix):
        """
        Loads our settings from django.conf.settings, applying defaults for any
        that are omitted.
        """
        self.prefix = prefix
        from django.conf import settings

        for name, default in self.defaults.items():
            prefix_name = (self.prefix + '_' + name).upper()
            value = getattr(settings, prefix_name, default)
            self._set_attr(prefix_name, value)
            setattr(settings, prefix_name, value)
            setting_changed.send(self, setting=prefix_name, value=value)

        setting_changed.connect(self._handler)

    def _check_config(self):
        list_editable_policy = self.POLICY | sum(LIST_EDITABLE_POLICIES)
        if list_editable_policy == sum(LIST_EDITABLE_POLICIES):
            raise ImproperlyConfigured("Invalid value for `CONCURRENCY_POLICY`: "
                                       "Use only one of `CONCURRENCY_LIST_EDITABLE_*` flags")

        conflict_policy = self.POLICY | sum(CONFLICTS_POLICIES)
        if conflict_policy == sum(CONFLICTS_POLICIES):
            raise ImproperlyConfigured("Invalid value for `CONCURRENCY_POLICY`: "
                                       "Use only one of `CONCURRENCY_POLICY_*` flags")

    def _set_attr(self, prefix_name, value):
        name = prefix_name[len(self.prefix) + 1:]
        if name == 'CALLBACK':
            if isinstance(value, six.string_types):
                func = get_callable(value)
            elif callable(value):
                func = value
            else:
                raise ImproperlyConfigured("`CALLBACK` must be a callable or a fullpath to callable")
            self._callback = func

        setattr(self, name, value)

    def _handler(self, sender, setting, value, **kwargs):
        """
            handler for ``setting_changed`` signal.

        @see :ref:`django:setting-changed`_
        """
        if setting.startswith(self.prefix):
            self._set_attr(setting, value)

conf = AppSettings('CONCURRENCY')
<commit_msg>add missed configuration for CALLBACK handler<commit_after>from __future__ import absolute_import, unicode_literals
from django.core.exceptions import ImproperlyConfigured
from django.core.urlresolvers import get_callable
from django.utils import six
from django.test.signals import setting_changed

# List Editable Policy
# 0 do not save updated records, save others, show message to the user
# 1 abort whole transaction

CONCURRENCY_LIST_EDITABLE_POLICY_SILENT = 1
CONCURRENCY_LIST_EDITABLE_POLICY_ABORT_ALL = 2
CONCURRENCY_POLICY_RAISE = 4
CONCURRENCY_POLICY_CALLBACK = 8

CONFLICTS_POLICIES = [CONCURRENCY_POLICY_RAISE, CONCURRENCY_POLICY_CALLBACK]
LIST_EDITABLE_POLICIES = [CONCURRENCY_LIST_EDITABLE_POLICY_SILENT, CONCURRENCY_LIST_EDITABLE_POLICY_ABORT_ALL]

class AppSettings(object):
    """
    Class to manage application related settings
    How to use:

    >>> from django.conf import settings
    >>> settings.APP_OVERRIDE = 'overridden'
    >>> settings.MYAPP_CALLBACK = 100
    >>> class MySettings(AppSettings):
    ...     defaults = {'ENTRY1': 'abc', 'ENTRY2': 123, 'OVERRIDE': None, 'CALLBACK':10}
    ...     def set_CALLBACK(self, value):
    ...         setattr(self, 'CALLBACK', value*2)

    >>> conf = MySettings("APP")
    >>> conf.ENTRY1, settings.APP_ENTRY1
    ('abc', 'abc')
    >>> conf.OVERRIDE, settings.APP_OVERRIDE
    ('overridden', 'overridden')

    >>> conf = MySettings("MYAPP")
    >>> conf.ENTRY2, settings.MYAPP_ENTRY2
    (123, 123)
    >>> conf = MySettings("MYAPP")
    >>> conf.CALLBACK
    200

    """
    defaults = {
        'ENABLED': True,
        'SANITY_CHECK': True,
        'FIELD_SIGNER': 'concurrency.forms.VersionFieldSigner',
        'POLICY': CONCURRENCY_LIST_EDITABLE_POLICY_SILENT | CONCURRENCY_POLICY_RAISE,
        'CALLBACK': 'concurrency.views.callback',
        'HANDLER409': 'concurrency.views.conflict'}

    def __init__(self, prefix):
        """
        Loads our settings from django.conf.settings, applying defaults for any
        that are omitted.
        """
        self.prefix = prefix
        from django.conf import settings

        for name, default in self.defaults.items():
            prefix_name = (self.prefix + '_' + name).upper()
            value = getattr(settings, prefix_name, default)
            self._set_attr(prefix_name, value)
            setattr(settings, prefix_name, value)
            setting_changed.send(self, setting=prefix_name, value=value)

        setting_changed.connect(self._handler)

    def _check_config(self):
        list_editable_policy = self.POLICY | sum(LIST_EDITABLE_POLICIES)
        if list_editable_policy == sum(LIST_EDITABLE_POLICIES):
            raise ImproperlyConfigured("Invalid value for `CONCURRENCY_POLICY`: "
                                       "Use only one of `CONCURRENCY_LIST_EDITABLE_*` flags")

        conflict_policy = self.POLICY | sum(CONFLICTS_POLICIES)
        if conflict_policy == sum(CONFLICTS_POLICIES):
            raise ImproperlyConfigured("Invalid value for `CONCURRENCY_POLICY`: "
                                       "Use only one of `CONCURRENCY_POLICY_*` flags")

    def _set_attr(self, prefix_name, value):
        name = prefix_name[len(self.prefix) + 1:]
        if name == 'CALLBACK':
            if isinstance(value, six.string_types):
                func = get_callable(value)
            elif callable(value):
                func = value
            else:
                raise ImproperlyConfigured("`CALLBACK` must be a callable or a fullpath to callable")
            self._callback = func

        setattr(self, name, value)

    def _handler(self, sender, setting, value, **kwargs):
        """
            handler for ``setting_changed`` signal.

        @see :ref:`django:setting-changed`_
        """
        if setting.startswith(self.prefix):
            self._set_attr(setting, value)

conf = AppSettings('CONCURRENCY')
<|endoftext|>


<commit_before>import io
import os
import time
import subprocess
import re
import cherrypy
import calendar
import datetime
from pydispatch import dispatcher

class FFMPEGError(Exception):
    pass

class FFMPEGTranscoderSubprocessTool(cherrypy.Tool):
    """
    This tool makes sure the ffmpeg subprocess is ended
    properly when a request is cancelled
    """
    def __init__(self):
        cherrypy.Tool.__init__(self, 'on_end_request',
                               self.end, priority=20)

    def end(self):
        if (hasattr(cherrypy.request, 'ffmpegtranscoder_subprocess') and
            cherrypy.request.ffmpegtranscoder_subprocess is not None):

            p = cherrypy.request.ffmpegtranscoder_subprocess
            p.stdout.read()
            p.wait()

        if (hasattr(cherrypy.request, 'transcoding_track') and
            cherrypy.request.transcoding_track is not None):

            track = cherrypy.request.transcoding_track
            dispatcher.send(signal='end_transcoding', sender=track)

class Transcoder:
    def transcode(self):
        raise NotImplementedError()

    @staticmethod
    def outputs():
        raise NotImplementedError()

class FFMPEGTranscoder(Transcoder):

    FNULL = open('/dev/null', 'w')

    def __init__(self, track):
        self.track = track

    def __enter__(self):
        filename = self.track.paths[0].path

        ext = os.path.splitext(os.path.basename(filename))[1].lower()[1:]

        artist = self.track.artist.name
        album = self.track.album.name
        title = self.track.name
        track_number = self.track.number if self.track.number is not None else 0

        args = (['ffmpeg'] +
            self.ffmpeg_input_args +
            ['-i', filename] +
            # always produce stereo output
            ['-ac', '2'] +
            # strip any video streams
            ['-vn'] +
            self.ffmpeg_output_args + [
                '-metadata', 'artist=%s' % artist,
                '-metadata', 'album=%s' % album,
                '-metadata', 'title=%s' % title,
                '-metadata', 'tracknumber=%s' % track_number,
                '-'
            ])

        for index, arg in enumerate(args):
            if not isinstance(arg, bytes):
                arg = arg.encode('utf8')

            args[index] = arg.replace(b'EXT', ext)

        self.process = subprocess.Popen(args, shell = False, stdout = subprocess.PIPE,
                                        stderr = self.FNULL,
                                        stdin = None)

        cherrypy.request.ffmpegtranscoder_subprocess = self.process
        cherrypy.request.transcoding_track = self.track

        return self.transcode

    def __exit__(self, type, value, traceback):
        self.process.wait()

        if self.process.returncode != 0:
            raise FFMPEGError('ffmpeg returned non-zero status "%d".' % self.process.returncode)

        cherrypy.request.ffmpegtranscoder_subprocess = None

    def transcode(self):

        # total seconds transcoded
        seconds = 0

        # how many seconds to stay ahead of the client
        seconds_keep_ahead = 10

        start_time = time.time()
        one_second = .95

        bitrate = self.bitrate()

        while True:
            data = self.process.stdout.read(bitrate)

            if len(data) == 0:
                break

            yield data

            seconds += 1

            # we pace the streaming so we don't have to send more than
            # the client can chew. this is also good for start/end_transcoding
            # events that lastfm uses so it can more accurately know how much
            # the client has actually played.
            #
            # what we do is try to stay seconds_keep_ahead seconds ahead of the
            # client, no more, no less...

            seconds_ahead = seconds * one_second - (time.time() - start_time)

            if seconds_ahead < seconds_keep_ahead:
                wait = one_second - (seconds_keep_ahead - seconds_ahead)
            else:
                wait = one_second

            if wait > 0:
                time.sleep(wait)

    def bitrate(self):
        """
        Should return the approximate bitrate this transcoder produces, so we
        can approximately pace the streaming properly
        """
        raise NotImplementedError()

class CopyFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "copy", "-f", "EXT"]
    ffmpeg_input_args = []

    def outputs():
        return None

    def bitrate(self):
        if self.track.bitrate is not None and self.track.bitrate != 0:
            return int(self.track.bitrate / 8)
        else:
            # default to a bitrate of 128kbit/s
            return int(128000 / 8)

class Mp3FFMPEGTranscoder(FFMPEGTranscoder):
    # -aq 0 should be v0
    ffmpeg_output_args = ["-acodec", "libmp3lame", "-f", "mp3", "-aq", "0"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/mp3'

    def bitrate(self):
        # lame v0's target bitrate is 245kbit/s (but is of course VBR)
        return int(245000 / 8)

class OggFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "libvorbis", "-f", "ogg", "-aq", "6"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/ogg'

    def bitrate(self):
        # -aq 6 is about 192kbit/s
        return int(192000 / 8)

class Transcoding:
    # list of players that don't supply proper Accept headers, basically...
    # don't have audio/flac here, always transcode it seeing as it's lossless...
    players = [
        ['Music Player Daemon', ['audio/ogg', 'audio/mp3']],
        ['foobar2000', ['audio/ogg', 'audio/mp3']],
        ['Windows.*Chrome', ['audio/ogg', 'audio/mp3']],
        ['Linux.*Chrome', ['audio/ogg']],
        ['Windows-Media-Player', ['audio/mp3']],
        ['NSPlayer', ['audio/mp3']],
        ['WinampMPEG', ['audio/mp3']],
        ['iTunes.*Windows', ['audio/mp3']],
        ['iTunes.*Macintosh', ['audio/mp3']],
        ['VLC', ['audio/ogg', 'audio/mp3']],
        # This matches Nokia N9s default player
        ['GStreamer', ['audio/ogg', 'audio/mp3']]
    ]

    transcoders = [Mp3FFMPEGTranscoder, OggFFMPEGTranscoder]

    def determine_transcoder(self, track, user_agent, accepts):
        if not (len(accepts) == 0 or len(accepts) == 1 and accepts[0] == '*/*'):
            transcoder, format = self._determine_transcoder(track, accepts)
            if transcoder is not None:
                return transcoder, format

        for player, formats in self.players:
            if re.search(player, user_agent):
                transcoder, format = self._determine_transcoder(track, formats)
                if transcoder is not None:
                    return transcoder, format
                break

        if track.format == 'audio/ogg':
            return CopyFFMPEGTranscoder, 'audio/ogg'
        else:
            return OggFFMPEGTranscoder, 'audio/ogg'

    def _determine_transcoder(self, track, formats):
        if track.format in formats:
            return CopyFFMPEGTranscoder, track.format

        for format in formats:
            for transcoder in self.transcoders:
                if transcoder.outputs() == format:
                    return transcoder, format

        return None, None

    def transcode(self, tracks, transcoder = None):
        if transcoder is None:
            transcoder = CopyFFMPEGTranscoder

        for track in tracks:

            start_time = time.time()

            if isinstance(transcoder, Transcoder):
                raise Exception("transcoder must be an instance of Transcoder")

            dispatcher.send(signal='start_transcoding', sender=track)

            with transcoder(track) as transcode:
                for data in transcode():
                    yield data

            total_time = int(time.time() - start_time)

transcoding = Transcoding()

<commit_msg>add ipad to player list<commit_after>import io
import os
import time
import subprocess
import re
import cherrypy
import calendar
import datetime
from pydispatch import dispatcher

class FFMPEGError(Exception):
    pass

class FFMPEGTranscoderSubprocessTool(cherrypy.Tool):
    """
    This tool makes sure the ffmpeg subprocess is ended
    properly when a request is cancelled
    """
    def __init__(self):
        cherrypy.Tool.__init__(self, 'on_end_request',
                               self.end, priority=20)

    def end(self):
        if (hasattr(cherrypy.request, 'ffmpegtranscoder_subprocess') and
            cherrypy.request.ffmpegtranscoder_subprocess is not None):

            p = cherrypy.request.ffmpegtranscoder_subprocess
            p.stdout.read()
            p.wait()

        if (hasattr(cherrypy.request, 'transcoding_track') and
            cherrypy.request.transcoding_track is not None):

            track = cherrypy.request.transcoding_track
            dispatcher.send(signal='end_transcoding', sender=track)

class Transcoder:
    def transcode(self):
        raise NotImplementedError()

    @staticmethod
    def outputs():
        raise NotImplementedError()

class FFMPEGTranscoder(Transcoder):

    FNULL = open('/dev/null', 'w')

    def __init__(self, track):
        self.track = track

    def __enter__(self):
        filename = self.track.paths[0].path

        ext = os.path.splitext(os.path.basename(filename))[1].lower()[1:]

        artist = self.track.artist.name
        album = self.track.album.name
        title = self.track.name
        track_number = self.track.number if self.track.number is not None else 0

        args = (['ffmpeg'] +
            self.ffmpeg_input_args +
            ['-i', filename] +
            # always produce stereo output
            ['-ac', '2'] +
            # strip any video streams
            ['-vn'] +
            self.ffmpeg_output_args + [
                '-metadata', 'artist=%s' % artist,
                '-metadata', 'album=%s' % album,
                '-metadata', 'title=%s' % title,
                '-metadata', 'tracknumber=%s' % track_number,
                '-'
            ])

        for index, arg in enumerate(args):
            if not isinstance(arg, bytes):
                arg = arg.encode('utf8')

            args[index] = arg.replace(b'EXT', ext)

        self.process = subprocess.Popen(args, shell = False, stdout = subprocess.PIPE,
                                        stderr = self.FNULL,
                                        stdin = None)

        cherrypy.request.ffmpegtranscoder_subprocess = self.process
        cherrypy.request.transcoding_track = self.track

        return self.transcode

    def __exit__(self, type, value, traceback):
        self.process.wait()

        if self.process.returncode != 0:
            raise FFMPEGError('ffmpeg returned non-zero status "%d".' % self.process.returncode)

        cherrypy.request.ffmpegtranscoder_subprocess = None

    def transcode(self):

        # total seconds transcoded
        seconds = 0

        # how many seconds to stay ahead of the client
        seconds_keep_ahead = 10

        start_time = time.time()
        one_second = .95

        bitrate = self.bitrate()

        while True:
            data = self.process.stdout.read(bitrate)

            if len(data) == 0:
                break

            yield data

            seconds += 1

            # we pace the streaming so we don't have to send more than
            # the client can chew. this is also good for start/end_transcoding
            # events that lastfm uses so it can more accurately know how much
            # the client has actually played.
            #
            # what we do is try to stay seconds_keep_ahead seconds ahead of the
            # client, no more, no less...

            seconds_ahead = seconds * one_second - (time.time() - start_time)

            if seconds_ahead < seconds_keep_ahead:
                wait = one_second - (seconds_keep_ahead - seconds_ahead)
            else:
                wait = one_second

            if wait > 0:
                time.sleep(wait)

    def bitrate(self):
        """
        Should return the approximate bitrate this transcoder produces, so we
        can approximately pace the streaming properly
        """
        raise NotImplementedError()

class CopyFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "copy", "-f", "EXT"]
    ffmpeg_input_args = []

    def outputs():
        return None

    def bitrate(self):
        if self.track.bitrate is not None and self.track.bitrate != 0:
            return int(self.track.bitrate / 8)
        else:
            # default to a bitrate of 128kbit/s
            return int(128000 / 8)

class Mp3FFMPEGTranscoder(FFMPEGTranscoder):
    # -aq 0 should be v0
    ffmpeg_output_args = ["-acodec", "libmp3lame", "-f", "mp3", "-aq", "0"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/mp3'

    def bitrate(self):
        # lame v0's target bitrate is 245kbit/s (but is of course VBR)
        return int(245000 / 8)

class OggFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "libvorbis", "-f", "ogg", "-aq", "6"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/ogg'

    def bitrate(self):
        # -aq 6 is about 192kbit/s
        return int(192000 / 8)

class Transcoding:
    # list of players that don't supply proper Accept headers, basically...
    # don't have audio/flac here, always transcode it seeing as it's lossless...
    players = [
        ['Music Player Daemon', ['audio/ogg', 'audio/mp3']],
        ['foobar2000', ['audio/ogg', 'audio/mp3']],
        ['Windows.*Chrome', ['audio/ogg', 'audio/mp3']],
        ['Linux.*Chrome', ['audio/ogg']],
        ['Windows-Media-Player', ['audio/mp3']],
        ['NSPlayer', ['audio/mp3']],
        ['WinampMPEG', ['audio/mp3']],
        ['iTunes.*Windows', ['audio/mp3']],
        ['iTunes.*Macintosh', ['audio/mp3']],
        ['VLC', ['audio/ogg', 'audio/mp3']],
        # This matches Nokia N9s default player
        ['GStreamer', ['audio/ogg', 'audio/mp3']],
        ['iPad', ['audio/mp3']],
    ]

    transcoders = [Mp3FFMPEGTranscoder, OggFFMPEGTranscoder]

    def determine_transcoder(self, track, user_agent, accepts):
        if not (len(accepts) == 0 or len(accepts) == 1 and accepts[0] == '*/*'):
            transcoder, format = self._determine_transcoder(track, accepts)
            if transcoder is not None:
                return transcoder, format

        for player, formats in self.players:
            if re.search(player, user_agent):
                transcoder, format = self._determine_transcoder(track, formats)
                if transcoder is not None:
                    return transcoder, format
                break

        if track.format == 'audio/ogg':
            return CopyFFMPEGTranscoder, 'audio/ogg'
        else:
            return OggFFMPEGTranscoder, 'audio/ogg'

    def _determine_transcoder(self, track, formats):
        if track.format in formats:
            return CopyFFMPEGTranscoder, track.format

        for format in formats:
            for transcoder in self.transcoders:
                if transcoder.outputs() == format:
                    return transcoder, format

        return None, None

    def transcode(self, tracks, transcoder = None):
        if transcoder is None:
            transcoder = CopyFFMPEGTranscoder

        for track in tracks:

            start_time = time.time()

            if isinstance(transcoder, Transcoder):
                raise Exception("transcoder must be an instance of Transcoder")

            dispatcher.send(signal='start_transcoding', sender=track)

            with transcoder(track) as transcode:
                for data in transcode():
                    yield data

            total_time = int(time.time() - start_time)

transcoding = Transcoding()

<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os
import sys
from os.path import join, isdir, isfile, abspath, expanduser
from shutil import copytree, copy2
from subprocess import check_call, Popen, PIPE, CalledProcessError
import locale

from conda.fetch import download
from conda.utils import hashsum_file

from conda_build import external
from conda_build.config import config
from conda_build.utils import rm_rf, tar_xf, unzip, safe_print_unicode

SRC_CACHE = join(config.croot, 'src_cache')
GIT_CACHE = join(config.croot, 'git_cache')
HG_CACHE = join(config.croot, 'hg_cache')
SVN_CACHE = join(config.croot, 'svn_cache')
WORK_DIR = join(config.croot, 'work')

def get_dir():
    if not isdir(WORK_DIR):
        os.makedirs(WORK_DIR)
    lst = [fn for fn in os.listdir(WORK_DIR) if not fn.startswith('.')]
    if len(lst) == 1:
        dir_path = join(WORK_DIR, lst[0])
        if isdir(dir_path):
            return dir_path
    return WORK_DIR

def download_to_cache(meta):
    ''' Download a source to the local cache. '''
    print('Source cache directory is: %s' % SRC_CACHE)
    if not isdir(SRC_CACHE):
        os.makedirs(SRC_CACHE)

    fn = meta['fn']
    path = join(SRC_CACHE, fn)

    if isfile(path):
        print('Found source in cache: %s' % fn)
    else:
        print('Downloading source to cache: %s' % fn)
        if not isinstance(meta['url'], list):
            meta['url'] = [meta['url']]

        for url in meta['url']:
            try:
                print("Downloading %s" % url)
                download(url, path)
            except RuntimeError as e:
                print("Error: %s" % str(e).strip(), file=sys.stderr)
            else:
                print("Success")
                break
        else: # no break
            raise RuntimeError("Could not download %s" % fn)

    for tp in 'md5', 'sha1', 'sha256':
        if meta.get(tp) and hashsum_file(path, tp) != meta[tp]:
            raise RuntimeError("%s mismatch: '%s' != '%s'" %
                               (tp.upper(), hashsum_file(path, tp), meta[tp]))

    return path

def unpack(meta):
    ''' Uncompress a downloaded source. '''
    src_path = download_to_cache(meta)

    os.makedirs(WORK_DIR)
    print("Extracting download")
    if src_path.lower().endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz',
        '.tar', 'tar.z')):
        tar_xf(src_path, WORK_DIR)
    elif src_path.lower().endswith('.zip'):
        unzip(src_path, WORK_DIR)
    else:
        # In this case, the build script will need to deal with unpacking the source
        print("Warning: Unrecognized source format. Source file will be copied to the SRC_DIR")
        copy2(src_path, WORK_DIR)

def git_source(meta, recipe_dir):
    ''' Download a source from Git repo. '''
    if not isdir(GIT_CACHE):
        os.makedirs(GIT_CACHE)

    git = external.find_executable('git')
    if not git:
        sys.exit("Error: git is not installed")
    git_url = meta['git_url']
    git_depth = int(meta.get('git_depth', -1))
    if git_url.startswith('.'):
        # It's a relative path from the conda recipe
        os.chdir(recipe_dir)
        git_dn = abspath(expanduser(git_url))
        git_dn = "_".join(git_dn.split(os.path.sep)[1:])
    else:
        git_dn = git_url.split(':')[-1].replace('/', '_')
    cache_repo = cache_repo_arg = join(GIT_CACHE, git_dn)
    if sys.platform == 'win32':
        is_cygwin = 'cygwin' in git.lower()
        cache_repo_arg = cache_repo_arg.replace('\\', '/')
        if is_cygwin:
            cache_repo_arg = '/cygdrive/c/' + cache_repo_arg[3:]

    # update (or create) the cache repo
    if isdir(cache_repo):
        check_call([git, 'fetch'], cwd=cache_repo)
    else:
        args = [git, 'clone', '--mirror']
        if git_depth > 0:
            args += ['--depth', git_depth]

        check_call(args + [git_url, cache_repo_arg],  cwd=recipe_dir)
        assert isdir(cache_repo)

    # now clone into the work directory
    checkout = meta.get('git_rev')
    # if rev is not specified, and the git_url is local,
    # assume the user wants the current HEAD
    if not checkout and git_url.startswith('.'):
        process = Popen(["git", "rev-parse", "HEAD"],
                    stdout=PIPE, stderr=PIPE,
                               cwd=git_url)
        output = process.communicate()[0].strip()
        checkout = output.decode('utf-8')
    if checkout:
        print('checkout: %r' % checkout)

    check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR])
    if checkout:
        check_call([git, 'checkout', checkout], cwd=WORK_DIR)

    git_info()
    return WORK_DIR

def git_info(fo=None):
    ''' Print info about a Git repo. '''
    assert isdir(WORK_DIR)

    # Ensure to explicitly set GIT_DIR as some Linux machines will not
    # properly execute without it.
    env = os.environ.copy()
    env['GIT_DIR'] = join(WORK_DIR, '.git')
    env = {str(key): str(value) for key, value in env.items()}
    for cmd, check_error in [
                ('git log -n1', True),
                ('git describe --tags --dirty', False),
                ('git status', True)]:
        p = Popen(cmd.split(), stdout=PIPE, stderr=PIPE, cwd=WORK_DIR, env=env)
        stdout, stderr = p.communicate()
        encoding = locale.getpreferredencoding()
        if not fo:
            encoding = sys.stdout.encoding
        encoding = encoding or 'utf-8'
        stdout = stdout.decode(encoding, 'ignore')
        stderr = stderr.decode(encoding, 'ignore')
        if check_error and stderr and stderr.strip():
            raise Exception("git error: %s" % stderr)
        if fo:
            fo.write(u'==> %s <==\n' % cmd)
            fo.write(stdout + u'\n')
        else:
            print(u'==> %s <==\n' % cmd)
            safe_print_unicode(stdout + u'\n')

def hg_source(meta):
    ''' Download a source from Mercurial repo. '''
    hg = external.find_executable('hg')
    if not hg:
        sys.exit('Error: hg not installed')
    hg_url = meta['hg_url']
    if not isdir(HG_CACHE):
        os.makedirs(HG_CACHE)
    hg_dn = hg_url.split(':')[-1].replace('/', '_')
    cache_repo = join(HG_CACHE, hg_dn)
    if isdir(cache_repo):
        check_call([hg, 'pull'], cwd=cache_repo)
    else:
        check_call([hg, 'clone', hg_url, cache_repo])
        assert isdir(cache_repo)

    # now clone in to work directory
    update = meta.get('hg_tag') or 'tip'
    print('checkout: %r' % update)

    check_call([hg, 'clone', cache_repo, WORK_DIR])
    check_call([hg, 'update', '-C', update], cwd=WORK_DIR)
    return WORK_DIR

def svn_source(meta):
    ''' Download a source from SVN repo. '''
    def parse_bool(s):
        return str(s).lower().strip() in ('yes', 'true', '1', 'on')

    svn = external.find_executable('svn')
    if not svn:
        sys.exit("Error: svn is not installed")
    svn_url = meta['svn_url']
    svn_revision = meta.get('svn_rev') or 'head'
    svn_ignore_externals = parse_bool(meta.get('svn_ignore_externals') or 'no')
    if not isdir(SVN_CACHE):
        os.makedirs(SVN_CACHE)
    svn_dn = svn_url.split(':', 1)[-1].replace('/', '_').replace(':', '_')
    cache_repo = join(SVN_CACHE, svn_dn)
    if svn_ignore_externals:
        extra_args = ['--ignore-externals']
    else:
        extra_args = []
    if isdir(cache_repo):
        check_call([svn, 'up', '-r', svn_revision] + extra_args, cwd=cache_repo)
    else:
        check_call([svn, 'co', '-r', svn_revision] + extra_args + [svn_url,
                                                                   cache_repo])
        assert isdir(cache_repo)

    # now copy into work directory
    copytree(cache_repo, WORK_DIR, symlinks=True)
    return WORK_DIR

def _ensure_unix_line_endings(path):
    """Replace windows line endings with Unix.  Return path to modified file."""
    out_path = path + "_unix"
    with open(path) as inputfile:
        with open(out_path, "w") as outputfile:
            for line in inputfile:
                outputfile.write(line.replace("\r\n", "\n"))
    return out_path

def apply_patch(src_dir, path):
    print('Applying patch: %r' % path)
    if not isfile(path):
        sys.exit('Error: no such patch: %s' % path)

    patch = external.find_executable('patch')
    if patch is None:
        sys.exit("""\
Error:
    Did not find 'patch' in: %s
    You can install 'patch' using apt-get, yum (Linux), Xcode (MacOSX),
    or conda, cygwin (Windows),
""" % (os.pathsep.join(external.dir_paths)))
    patch_args = ['-p0', '-i', path]
    if sys.platform == 'win32':
        patch_args[-1] =  _ensure_unix_line_endings(path)
    try:
        check_call([patch] + patch_args, cwd=src_dir)
    except CalledProcessError:
        sys.exit(1)
    if sys.platform == 'win32' and os.path.exists(patch_args[-1]):
        os.remove(patch_args[-1])  # clean up .patch_unix file

def provide(recipe_dir, meta, patch=True):
    """
    given a recipe_dir:
      - download (if necessary)
      - unpack
      - apply patches (if any)
    """
    print("Removing old work directory")
    rm_rf(WORK_DIR)
    if 'fn' in meta:
        unpack(meta)
    elif 'git_url' in meta:
        git_source(meta, recipe_dir)
    elif 'hg_url' in meta:
        hg_source(meta)
    elif 'svn_url' in meta:
        svn_source(meta)
    elif 'path' in meta:
        print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR))
        copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)
    else: # no source
        os.makedirs(WORK_DIR)

    if patch:
        src_dir = get_dir()
        for patch in meta.get('patches', []):
            apply_patch(src_dir, join(recipe_dir, patch))

if __name__ == '__main__':
    print(provide('.',
                  {'url': 'http://pypi.python.org/packages/source/b/bitarray/bitarray-0.8.0.tar.gz',
                   'git_url': 'git@github.com:ilanschnell/bitarray.git',
                   'git_tag': '0.5.2'}))
<commit_msg>source: Fix git_depth option (was failing on Linux)<commit_after>from __future__ import absolute_import, division, print_function

import os
import sys
from os.path import join, isdir, isfile, abspath, expanduser
from shutil import copytree, copy2
from subprocess import check_call, Popen, PIPE, CalledProcessError
import locale

from conda.fetch import download
from conda.utils import hashsum_file

from conda_build import external
from conda_build.config import config
from conda_build.utils import rm_rf, tar_xf, unzip, safe_print_unicode

SRC_CACHE = join(config.croot, 'src_cache')
GIT_CACHE = join(config.croot, 'git_cache')
HG_CACHE = join(config.croot, 'hg_cache')
SVN_CACHE = join(config.croot, 'svn_cache')
WORK_DIR = join(config.croot, 'work')

def get_dir():
    if not isdir(WORK_DIR):
        os.makedirs(WORK_DIR)
    lst = [fn for fn in os.listdir(WORK_DIR) if not fn.startswith('.')]
    if len(lst) == 1:
        dir_path = join(WORK_DIR, lst[0])
        if isdir(dir_path):
            return dir_path
    return WORK_DIR

def download_to_cache(meta):
    ''' Download a source to the local cache. '''
    print('Source cache directory is: %s' % SRC_CACHE)
    if not isdir(SRC_CACHE):
        os.makedirs(SRC_CACHE)

    fn = meta['fn']
    path = join(SRC_CACHE, fn)

    if isfile(path):
        print('Found source in cache: %s' % fn)
    else:
        print('Downloading source to cache: %s' % fn)
        if not isinstance(meta['url'], list):
            meta['url'] = [meta['url']]

        for url in meta['url']:
            try:
                print("Downloading %s" % url)
                download(url, path)
            except RuntimeError as e:
                print("Error: %s" % str(e).strip(), file=sys.stderr)
            else:
                print("Success")
                break
        else: # no break
            raise RuntimeError("Could not download %s" % fn)

    for tp in 'md5', 'sha1', 'sha256':
        if meta.get(tp) and hashsum_file(path, tp) != meta[tp]:
            raise RuntimeError("%s mismatch: '%s' != '%s'" %
                               (tp.upper(), hashsum_file(path, tp), meta[tp]))

    return path

def unpack(meta):
    ''' Uncompress a downloaded source. '''
    src_path = download_to_cache(meta)

    os.makedirs(WORK_DIR)
    print("Extracting download")
    if src_path.lower().endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz',
        '.tar', 'tar.z')):
        tar_xf(src_path, WORK_DIR)
    elif src_path.lower().endswith('.zip'):
        unzip(src_path, WORK_DIR)
    else:
        # In this case, the build script will need to deal with unpacking the source
        print("Warning: Unrecognized source format. Source file will be copied to the SRC_DIR")
        copy2(src_path, WORK_DIR)

def git_source(meta, recipe_dir):
    ''' Download a source from Git repo. '''
    if not isdir(GIT_CACHE):
        os.makedirs(GIT_CACHE)

    git = external.find_executable('git')
    if not git:
        sys.exit("Error: git is not installed")
    git_url = meta['git_url']
    git_depth = int(meta.get('git_depth', -1))
    if git_url.startswith('.'):
        # It's a relative path from the conda recipe
        os.chdir(recipe_dir)
        git_dn = abspath(expanduser(git_url))
        git_dn = "_".join(git_dn.split(os.path.sep)[1:])
    else:
        git_dn = git_url.split(':')[-1].replace('/', '_')
    cache_repo = cache_repo_arg = join(GIT_CACHE, git_dn)
    if sys.platform == 'win32':
        is_cygwin = 'cygwin' in git.lower()
        cache_repo_arg = cache_repo_arg.replace('\\', '/')
        if is_cygwin:
            cache_repo_arg = '/cygdrive/c/' + cache_repo_arg[3:]

    # update (or create) the cache repo
    if isdir(cache_repo):
        check_call([git, 'fetch'], cwd=cache_repo)
    else:
        args = [git, 'clone', '--mirror']
        if git_depth > 0:
            args += ['--depth', str(git_depth)]

        check_call(args + [git_url, cache_repo_arg],  cwd=recipe_dir)
        assert isdir(cache_repo)

    # now clone into the work directory
    checkout = meta.get('git_rev')
    # if rev is not specified, and the git_url is local,
    # assume the user wants the current HEAD
    if not checkout and git_url.startswith('.'):
        process = Popen(["git", "rev-parse", "HEAD"],
                    stdout=PIPE, stderr=PIPE,
                               cwd=git_url)
        output = process.communicate()[0].strip()
        checkout = output.decode('utf-8')
    if checkout:
        print('checkout: %r' % checkout)

    check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR])
    if checkout:
        check_call([git, 'checkout', checkout], cwd=WORK_DIR)

    git_info()
    return WORK_DIR

def git_info(fo=None):
    ''' Print info about a Git repo. '''
    assert isdir(WORK_DIR)

    # Ensure to explicitly set GIT_DIR as some Linux machines will not
    # properly execute without it.
    env = os.environ.copy()
    env['GIT_DIR'] = join(WORK_DIR, '.git')
    env = {str(key): str(value) for key, value in env.items()}
    for cmd, check_error in [
                ('git log -n1', True),
                ('git describe --tags --dirty', False),
                ('git status', True)]:
        p = Popen(cmd.split(), stdout=PIPE, stderr=PIPE, cwd=WORK_DIR, env=env)
        stdout, stderr = p.communicate()
        encoding = locale.getpreferredencoding()
        if not fo:
            encoding = sys.stdout.encoding
        encoding = encoding or 'utf-8'
        stdout = stdout.decode(encoding, 'ignore')
        stderr = stderr.decode(encoding, 'ignore')
        if check_error and stderr and stderr.strip():
            raise Exception("git error: %s" % stderr)
        if fo:
            fo.write(u'==> %s <==\n' % cmd)
            fo.write(stdout + u'\n')
        else:
            print(u'==> %s <==\n' % cmd)
            safe_print_unicode(stdout + u'\n')

def hg_source(meta):
    ''' Download a source from Mercurial repo. '''
    hg = external.find_executable('hg')
    if not hg:
        sys.exit('Error: hg not installed')
    hg_url = meta['hg_url']
    if not isdir(HG_CACHE):
        os.makedirs(HG_CACHE)
    hg_dn = hg_url.split(':')[-1].replace('/', '_')
    cache_repo = join(HG_CACHE, hg_dn)
    if isdir(cache_repo):
        check_call([hg, 'pull'], cwd=cache_repo)
    else:
        check_call([hg, 'clone', hg_url, cache_repo])
        assert isdir(cache_repo)

    # now clone in to work directory
    update = meta.get('hg_tag') or 'tip'
    print('checkout: %r' % update)

    check_call([hg, 'clone', cache_repo, WORK_DIR])
    check_call([hg, 'update', '-C', update], cwd=WORK_DIR)
    return WORK_DIR

def svn_source(meta):
    ''' Download a source from SVN repo. '''
    def parse_bool(s):
        return str(s).lower().strip() in ('yes', 'true', '1', 'on')

    svn = external.find_executable('svn')
    if not svn:
        sys.exit("Error: svn is not installed")
    svn_url = meta['svn_url']
    svn_revision = meta.get('svn_rev') or 'head'
    svn_ignore_externals = parse_bool(meta.get('svn_ignore_externals') or 'no')
    if not isdir(SVN_CACHE):
        os.makedirs(SVN_CACHE)
    svn_dn = svn_url.split(':', 1)[-1].replace('/', '_').replace(':', '_')
    cache_repo = join(SVN_CACHE, svn_dn)
    if svn_ignore_externals:
        extra_args = ['--ignore-externals']
    else:
        extra_args = []
    if isdir(cache_repo):
        check_call([svn, 'up', '-r', svn_revision] + extra_args, cwd=cache_repo)
    else:
        check_call([svn, 'co', '-r', svn_revision] + extra_args + [svn_url,
                                                                   cache_repo])
        assert isdir(cache_repo)

    # now copy into work directory
    copytree(cache_repo, WORK_DIR, symlinks=True)
    return WORK_DIR

def _ensure_unix_line_endings(path):
    """Replace windows line endings with Unix.  Return path to modified file."""
    out_path = path + "_unix"
    with open(path) as inputfile:
        with open(out_path, "w") as outputfile:
            for line in inputfile:
                outputfile.write(line.replace("\r\n", "\n"))
    return out_path

def apply_patch(src_dir, path):
    print('Applying patch: %r' % path)
    if not isfile(path):
        sys.exit('Error: no such patch: %s' % path)

    patch = external.find_executable('patch')
    if patch is None:
        sys.exit("""\
Error:
    Did not find 'patch' in: %s
    You can install 'patch' using apt-get, yum (Linux), Xcode (MacOSX),
    or conda, cygwin (Windows),
""" % (os.pathsep.join(external.dir_paths)))
    patch_args = ['-p0', '-i', path]
    if sys.platform == 'win32':
        patch_args[-1] =  _ensure_unix_line_endings(path)
    try:
        check_call([patch] + patch_args, cwd=src_dir)
    except CalledProcessError:
        sys.exit(1)
    if sys.platform == 'win32' and os.path.exists(patch_args[-1]):
        os.remove(patch_args[-1])  # clean up .patch_unix file

def provide(recipe_dir, meta, patch=True):
    """
    given a recipe_dir:
      - download (if necessary)
      - unpack
      - apply patches (if any)
    """
    print("Removing old work directory")
    rm_rf(WORK_DIR)
    if 'fn' in meta:
        unpack(meta)
    elif 'git_url' in meta:
        git_source(meta, recipe_dir)
    elif 'hg_url' in meta:
        hg_source(meta)
    elif 'svn_url' in meta:
        svn_source(meta)
    elif 'path' in meta:
        print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR))
        copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)
    else: # no source
        os.makedirs(WORK_DIR)

    if patch:
        src_dir = get_dir()
        for patch in meta.get('patches', []):
            apply_patch(src_dir, join(recipe_dir, patch))

if __name__ == '__main__':
    print(provide('.',
                  {'url': 'http://pypi.python.org/packages/source/b/bitarray/bitarray-0.8.0.tar.gz',
                   'git_url': 'git@github.com:ilanschnell/bitarray.git',
                   'git_tag': '0.5.2'}))
<|endoftext|>


<commit_before>"""This is IOS XR Classic driver implementation."""

from functools import partial
import re
import time
from condoor.drivers.generic import Driver as Generic
from condoor import pattern_manager, TIMEOUT, EOF, ConnectionAuthenticationError, ConnectionError, \
    CommandSyntaxError, ConfigurationErrors
from condoor.fsm import FSM
from condoor.actions import a_reload_na, a_send, a_send_boot, a_reconnect, a_send_username, a_send_password,\
    a_message_callback, a_return_and_reconnect, a_not_committed, a_send_line, a_capture_show_configuration_failed, \
    a_configuration_inconsistency
from condoor.config import CONF

_C = CONF['driver']['XR']

class Driver(Generic):
    """This is a Driver class implementation for IOS XR Classic."""

    platform = 'XR'
    inventory_cmd = 'admin show inventory chassis'
    users_cmd = 'show users'
    target_prompt_components = ['prompt_dynamic', 'prompt_default', 'rommon', 'xml', 'qnx']
    prepare_terminal_session = ['terminal exec prompt no-timestamp', 'terminal len 0', 'terminal width 0']
    reload_cmd = 'admin reload location all'
    commit_cmd = 'commit label {}'
    rollback_cmd = 'rollback configuration {}'
    families = {
        "ASR9K": "ASR9K",
        "ASR-9": "ASR9K",
        "CRS": "CRS",
        "12": "XR12K",
    }

    def __init__(self, device):
        """Initialize the IOS XR Classic driver object."""
        super(Driver, self).__init__(device)

    def update_driver(self, prompt):
        """Return driver name based on prompt analysis."""
        return pattern_manager.platform(prompt, ['XR', 'QNX'])

    def reload(self, reload_timeout, save_config):
        """Reload the device."""
        PROCEED = re.compile(re.escape("Proceed with reload? [confirm]"))
        DONE = re.compile(re.escape("[Done]"))
        CONFIGURATION_COMPLETED = re.compile("SYSTEM CONFIGURATION COMPLETED")
        CONFIGURATION_IN_PROCESS = re.compile("SYSTEM CONFIGURATION IN PROCESS")

        # CONSOLE = re.compile("ios con[0|1]/RS?P[0-1]/CPU0 is now available")
        CONSOLE = re.compile("ios con[0|1]/(?:RS?P)?[0-1]/CPU0 is now available")
        RECONFIGURE_USERNAME_PROMPT = "[Nn][Oo] root-system username is configured"
        ROOT_USERNAME_PROMPT = "Enter root-system username\: "
        ROOT_PASSWORD_PROMPT = "Enter secret( again)?\: "

        # BOOT=disk0:asr9k-os-mbi-6.1.1/0x100305/mbiasr9k-rsp3.vm,1; \
        # disk0:asr9k-os-mbi-5.3.4/0x100305/mbiasr9k-rsp3.vm,2;
        # Candidate Boot Image num 0 is disk0:asr9k-os-mbi-6.1.1/0x100305/mbiasr9k-rsp3.vm
        # Candidate Boot Image num 1 is disk0:asr9k-os-mbi-5.3.4/0x100305/mbiasr9k-rsp3.vm
        CANDIDATE_BOOT_IMAGE = "Candidate Boot Image num 0 is .*vm"
        NOT_COMMITTED = re.compile(re.escape("Some active software packages are not yet committed. Proceed?[confirm]"))
        RELOAD_NA = re.compile("Reload to the ROM monitor disallowed from a telnet line")
        #           0          1      2                3                   4                  5
        events = [RELOAD_NA, DONE, PROCEED, CONFIGURATION_IN_PROCESS, self.rommon_re, self.press_return_re,
                  #   6               7                   8                           9
                  CONSOLE, CONFIGURATION_COMPLETED, RECONFIGURE_USERNAME_PROMPT, ROOT_USERNAME_PROMPT,
                  #    10                    11              12     13          14           15
                  ROOT_PASSWORD_PROMPT, self.username_re, TIMEOUT, EOF, self.reload_cmd, CANDIDATE_BOOT_IMAGE,
                  #   16
                  NOT_COMMITTED]

        transitions = [
            (RELOAD_NA, [0], -1, a_reload_na, 0),
            (NOT_COMMITTED, [0], -1, a_not_committed, 0),
            (DONE, [0], 2, None, 120),
            (PROCEED, [2], 3, partial(a_send, "\r"), reload_timeout),
            # this needs to be verified
            (self.rommon_re, [0, 3], 3, partial(a_send_boot, "boot"), 600),
            (CANDIDATE_BOOT_IMAGE, [0, 3], 4, a_message_callback, 600),
            (CONSOLE, [0, 1, 3, 4], 5, None, 600),
            (self.press_return_re, [5], 6, partial(a_send, "\r"), 300),
            # configure root username and password the same as used for device connection.
            (RECONFIGURE_USERNAME_PROMPT, [6, 7, 10], 8, None, 10),
            (ROOT_USERNAME_PROMPT, [8], 9, partial(a_send_username, self.device.node_info.username), 1),
            (ROOT_PASSWORD_PROMPT, [9], 9, partial(a_send_password, self.device.node_info.password), 1),
            (CONFIGURATION_IN_PROCESS, [6, 9], 10, None, 600),
            (CONFIGURATION_COMPLETED, [10], -1, a_reconnect, 0),
            (self.username_re, [7, 9], -1, a_return_and_reconnect, 0),
            (TIMEOUT, [0, 1, 2], -1, ConnectionAuthenticationError("Unable to reload"), 0),
            (EOF, [0, 1, 2, 3, 4, 5], -1, ConnectionError("Device disconnected"), 0),
            (TIMEOUT, [6], 7, partial(a_send, "\r"), 180),
            (TIMEOUT, [7, 10], -1, ConnectionAuthenticationError("Unable to reconnect after reloading"), 0),
        ]

        fsm = FSM("RELOAD", self.device, events, transitions, timeout=600)
        return fsm.run()

    def config(self, config_text, plane):
        """Apply config."""
        NO_CONFIGURATION_CHANGE = re.compile("No configuration changes to commit")
        CONFIGURATION_FAILED = re.compile("show configuration failed")
        CONFIGURATION_INCONSITENCY = re.compile("No configuration commits for this SDR will be allowed until "
                                                "a 'clear configuration inconsistency' command is performed.")

        self.enter_plane(plane)

        nol = config_text.count('\n')
        config_lines = iter(config_text.splitlines())
        events = [self.prompt_re, self.syntax_error_re]
        transitions = [
            (self.prompt_re, [0], 0, partial(a_send_line, config_lines), 10),
            (self.syntax_error_re, [0], -1, CommandSyntaxError("Configuration syntax error."), 0)
        ]
        self.device.ctrl.send_command(self.config_cmd)
        fsm = FSM("CONFIG", self.device, events, transitions, timeout=10, max_transitions=nol + 5)
        fsm.run()

        events = [self.prompt_re, NO_CONFIGURATION_CHANGE, CONFIGURATION_FAILED]
        transitions = [
            (NO_CONFIGURATION_CHANGE, [0], -1, ConfigurationErrors("No configuration changes to commit."), 0),
            (CONFIGURATION_FAILED, [0], 2, a_capture_show_configuration_failed, 10),
            (CONFIGURATION_INCONSITENCY, [0], 2, a_configuration_inconsistency, 10),
            (self.prompt_re, [0], 1, partial(a_send_line, 'end'), 60),
            (self.prompt_re, [1], -1, None, 0)
        ]

        label = 'condoor-{}'.format(int(time.time()))
        self.device.ctrl.send_command(self.commit_cmd.format(label))
        fsm = FSM("COMMIT", self.device, events, transitions, timeout=120, max_transitions=5)
        fsm.run()
        self.exit_plane()
        return label

    def rollback(self, label, plane):
        """Rollback config."""
        cm_label = 'condoor-{}'.format(int(time.time()))
        self.device.send(self.rollback_cmd.format(label), timeout=120)
        return cm_label
<commit_msg>Move to authentication phase if CONFIGURATION_COMPLETED not received in 1200s<commit_after>"""This is IOS XR Classic driver implementation."""

from functools import partial
import re
import time
from condoor.drivers.generic import Driver as Generic
from condoor import pattern_manager, TIMEOUT, EOF, ConnectionAuthenticationError, ConnectionError, \
    CommandSyntaxError, ConfigurationErrors
from condoor.fsm import FSM
from condoor.actions import a_reload_na, a_send, a_send_boot, a_reconnect, a_send_username, a_send_password,\
    a_message_callback, a_return_and_reconnect, a_not_committed, a_send_line, a_capture_show_configuration_failed, \
    a_configuration_inconsistency
from condoor.config import CONF

_C = CONF['driver']['XR']

class Driver(Generic):
    """This is a Driver class implementation for IOS XR Classic."""

    platform = 'XR'
    inventory_cmd = 'admin show inventory chassis'
    users_cmd = 'show users'
    target_prompt_components = ['prompt_dynamic', 'prompt_default', 'rommon', 'xml', 'qnx']
    prepare_terminal_session = ['terminal exec prompt no-timestamp', 'terminal len 0', 'terminal width 0']
    reload_cmd = 'admin reload location all'
    commit_cmd = 'commit label {}'
    rollback_cmd = 'rollback configuration {}'
    families = {
        "ASR9K": "ASR9K",
        "ASR-9": "ASR9K",
        "CRS": "CRS",
        "12": "XR12K",
    }

    def __init__(self, device):
        """Initialize the IOS XR Classic driver object."""
        super(Driver, self).__init__(device)

    def update_driver(self, prompt):
        """Return driver name based on prompt analysis."""
        return pattern_manager.platform(prompt, ['XR', 'QNX'])

    def reload(self, reload_timeout, save_config):
        """Reload the device."""
        PROCEED = re.compile(re.escape("Proceed with reload? [confirm]"))
        DONE = re.compile(re.escape("[Done]"))
        CONFIGURATION_COMPLETED = re.compile("SYSTEM CONFIGURATION COMPLETED")
        CONFIGURATION_IN_PROCESS = re.compile("SYSTEM CONFIGURATION IN PROCESS")

        # CONSOLE = re.compile("ios con[0|1]/RS?P[0-1]/CPU0 is now available")
        CONSOLE = re.compile("ios con[0|1]/(?:RS?P)?[0-1]/CPU0 is now available")
        RECONFIGURE_USERNAME_PROMPT = "[Nn][Oo] root-system username is configured"
        ROOT_USERNAME_PROMPT = "Enter root-system username\: "
        ROOT_PASSWORD_PROMPT = "Enter secret( again)?\: "

        # BOOT=disk0:asr9k-os-mbi-6.1.1/0x100305/mbiasr9k-rsp3.vm,1; \
        # disk0:asr9k-os-mbi-5.3.4/0x100305/mbiasr9k-rsp3.vm,2;
        # Candidate Boot Image num 0 is disk0:asr9k-os-mbi-6.1.1/0x100305/mbiasr9k-rsp3.vm
        # Candidate Boot Image num 1 is disk0:asr9k-os-mbi-5.3.4/0x100305/mbiasr9k-rsp3.vm
        CANDIDATE_BOOT_IMAGE = "Candidate Boot Image num 0 is .*vm"
        NOT_COMMITTED = re.compile(re.escape("Some active software packages are not yet committed. Proceed?[confirm]"))
        RELOAD_NA = re.compile("Reload to the ROM monitor disallowed from a telnet line")
        #           0          1      2                3                   4                  5
        events = [RELOAD_NA, DONE, PROCEED, CONFIGURATION_IN_PROCESS, self.rommon_re, self.press_return_re,
                  #   6               7                   8                           9
                  CONSOLE, CONFIGURATION_COMPLETED, RECONFIGURE_USERNAME_PROMPT, ROOT_USERNAME_PROMPT,
                  #    10                    11              12     13          14           15
                  ROOT_PASSWORD_PROMPT, self.username_re, TIMEOUT, EOF, self.reload_cmd, CANDIDATE_BOOT_IMAGE,
                  #   16
                  NOT_COMMITTED]

        transitions = [
            (RELOAD_NA, [0], -1, a_reload_na, 0),
            # temp for testing
            (NOT_COMMITTED, [0], -1, a_not_committed, 10),
            (DONE, [0], 2, None, 120),
            (PROCEED, [2], 3, partial(a_send, "\r"), reload_timeout),
            # this needs to be verified
            (self.rommon_re, [0, 3], 3, partial(a_send_boot, "boot"), 600),
            (CANDIDATE_BOOT_IMAGE, [0, 3], 4, a_message_callback, 600),
            (CONSOLE, [0, 1, 3, 4], 5, None, 600),
            # This is required. Otherwise nothing more is displayed on the console
            (self.press_return_re, [5], 6, partial(a_send, "\r"), 300),
            # configure root username and password the same as used for device connection.
            (RECONFIGURE_USERNAME_PROMPT, [6, 7, 10], 8, None, 10),
            (ROOT_USERNAME_PROMPT, [8], 9, partial(a_send_username, self.device.node_info.username), 1),
            (ROOT_PASSWORD_PROMPT, [9], 9, partial(a_send_password, self.device.node_info.password), 1),
            (CONFIGURATION_IN_PROCESS, [6, 9], 10, None, 1200),
            (CONFIGURATION_COMPLETED, [10], -1, a_reconnect, 0),
            (self.username_re, [7, 9], -1, a_return_and_reconnect, 0),
            (TIMEOUT, [0, 1, 2], -1, ConnectionAuthenticationError("Unable to reload"), 0),
            (EOF, [0, 1, 2, 3, 4, 5], -1, ConnectionError("Device disconnected"), 0),
            (TIMEOUT, [6], 7, partial(a_send, "\r"), 180),
            (TIMEOUT, [7], -1, ConnectionAuthenticationError("Unable to reconnect after reloading"), 0),
            (TIMEOUT, [10], -1, a_reconnect, 0),

        ]

        fsm = FSM("RELOAD", self.device, events, transitions, timeout=600)
        return fsm.run()

    def config(self, config_text, plane):
        """Apply config."""
        NO_CONFIGURATION_CHANGE = re.compile("No configuration changes to commit")
        CONFIGURATION_FAILED = re.compile("show configuration failed")
        CONFIGURATION_INCONSITENCY = re.compile("No configuration commits for this SDR will be allowed until "
                                                "a 'clear configuration inconsistency' command is performed.")

        self.enter_plane(plane)

        nol = config_text.count('\n')
        config_lines = iter(config_text.splitlines())
        events = [self.prompt_re, self.syntax_error_re]
        transitions = [
            (self.prompt_re, [0], 0, partial(a_send_line, config_lines), 10),
            (self.syntax_error_re, [0], -1, CommandSyntaxError("Configuration syntax error."), 0)
        ]
        self.device.ctrl.send_command(self.config_cmd)
        fsm = FSM("CONFIG", self.device, events, transitions, timeout=10, max_transitions=nol + 5)
        fsm.run()

        events = [self.prompt_re, NO_CONFIGURATION_CHANGE, CONFIGURATION_FAILED]
        transitions = [
            (NO_CONFIGURATION_CHANGE, [0], -1, ConfigurationErrors("No configuration changes to commit."), 0),
            (CONFIGURATION_FAILED, [0], 2, a_capture_show_configuration_failed, 10),
            (CONFIGURATION_INCONSITENCY, [0], 2, a_configuration_inconsistency, 10),
            (self.prompt_re, [0], 1, partial(a_send_line, 'end'), 60),
            (self.prompt_re, [1], -1, None, 0)
        ]

        label = 'condoor-{}'.format(int(time.time()))
        self.device.ctrl.send_command(self.commit_cmd.format(label))
        fsm = FSM("COMMIT", self.device, events, transitions, timeout=120, max_transitions=5)
        fsm.run()
        self.exit_plane()
        return label

    def rollback(self, label, plane):
        """Rollback config."""
        cm_label = 'condoor-{}'.format(int(time.time()))
        self.device.send(self.rollback_cmd.format(label), timeout=120)
        return cm_label
<|endoftext|>


<commit_before># The current Orchestra version.
__version__ = '1.0.31'

default_app_config = 'orchestra.apps.OrchestraAppConfig'
<commit_msg>Version bump.<commit_after># The current Orchestra version.
__version__ = '1.0.32'

default_app_config = 'orchestra.apps.OrchestraAppConfig'
<|endoftext|>


<commit_before># The current Orchestra version.
__version__ = '0.2.59'

default_app_config = 'orchestra.apps.OrchestraAppConfig'
<commit_msg>Version bump.<commit_after># The current Orchestra version.
__version__ = '0.2.60'

default_app_config = 'orchestra.apps.OrchestraAppConfig'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2012-2015, Ben Lopatin and contributors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.  Redistributions in binary
# form must reproduce the above copyright notice, this list of conditions and the
# following disclaimer in the documentation and/or other materials provided with
# the distribution
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from django.conf.urls import patterns, url
from django.contrib.auth.decorators import login_required

from .views import (OrganizationList, OrganizationDetail,
        OrganizationUpdate, OrganizationDelete, OrganizationCreate,
        OrganizationUserList, OrganizationUserDetail, OrganizationUserUpdate,
        OrganizationUserCreate, OrganizationUserRemind, OrganizationUserDelete)

urlpatterns = patterns('',
    # Organization URLs
    url(r'^$', view=login_required(OrganizationList.as_view()),
        name="organization_list"),
    url(r'^add/$', view=login_required(OrganizationCreate.as_view()),
        name="organization_add"),
    url(r'^(?P<organization_pk>[\d]+)/$',
        view=login_required(OrganizationDetail.as_view()),
        name="organization_detail"),
    url(r'^(?P<organization_pk>[\d]+)/edit/$',
        view=login_required(OrganizationUpdate.as_view()),
        name="organization_edit"),
    url(r'^(?P<organization_pk>[\d]+)/delete/$',
        view=login_required(OrganizationDelete.as_view()),
        name="organization_delete"),

    # Organization user URLs
    url(r'^(?P<organization_pk>[\d]+)/people/$',
        view=login_required(OrganizationUserList.as_view()),
        name="organization_user_list"),
    url(r'^(?P<organization_pk>[\d]+)/people/add/$',
        view=login_required(OrganizationUserCreate.as_view()),
        name="organization_user_add"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/remind/$',
        view=login_required(OrganizationUserRemind.as_view()),
        name="organization_user_remind"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/$',
        view=login_required(OrganizationUserDetail.as_view()),
        name="organization_user_detail"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/edit/$',
        view=login_required(OrganizationUserUpdate.as_view()),
        name="organization_user_edit"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/delete/$',
        view=login_required(OrganizationUserDelete.as_view()),
        name="organization_user_delete"),
)
<commit_msg>Fix RemovedInDjango110Warning for urlpatterns<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2012-2015, Ben Lopatin and contributors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.  Redistributions in binary
# form must reproduce the above copyright notice, this list of conditions and the
# following disclaimer in the documentation and/or other materials provided with
# the distribution
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from django.conf.urls import url
from django.contrib.auth.decorators import login_required

from .views import (OrganizationList, OrganizationDetail,
        OrganizationUpdate, OrganizationDelete, OrganizationCreate,
        OrganizationUserList, OrganizationUserDetail, OrganizationUserUpdate,
        OrganizationUserCreate, OrganizationUserRemind, OrganizationUserDelete)

urlpatterns = [
    # Organization URLs
    url(r'^$', view=login_required(OrganizationList.as_view()),
        name="organization_list"),
    url(r'^add/$', view=login_required(OrganizationCreate.as_view()),
        name="organization_add"),
    url(r'^(?P<organization_pk>[\d]+)/$',
        view=login_required(OrganizationDetail.as_view()),
        name="organization_detail"),
    url(r'^(?P<organization_pk>[\d]+)/edit/$',
        view=login_required(OrganizationUpdate.as_view()),
        name="organization_edit"),
    url(r'^(?P<organization_pk>[\d]+)/delete/$',
        view=login_required(OrganizationDelete.as_view()),
        name="organization_delete"),

    # Organization user URLs
    url(r'^(?P<organization_pk>[\d]+)/people/$',
        view=login_required(OrganizationUserList.as_view()),
        name="organization_user_list"),
    url(r'^(?P<organization_pk>[\d]+)/people/add/$',
        view=login_required(OrganizationUserCreate.as_view()),
        name="organization_user_add"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/remind/$',
        view=login_required(OrganizationUserRemind.as_view()),
        name="organization_user_remind"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/$',
        view=login_required(OrganizationUserDetail.as_view()),
        name="organization_user_detail"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/edit/$',
        view=login_required(OrganizationUserUpdate.as_view()),
        name="organization_user_edit"),
    url(r'^(?P<organization_pk>[\d]+)/people/(?P<user_pk>[\d]+)/delete/$',
        view=login_required(OrganizationUserDelete.as_view()),
        name="organization_user_delete"),
]
<|endoftext|>


<commit_before>#!/bin/python

"""
ProcessQueue - runs workers through queues
"""

from __future__ import division
from __future__ import print_function
from multiprocessing import Process, Queue, cpu_count
from collections import namedtuple
import time
import random

from test.algorithms.saes import f

# (Semi Auto) Configuration

try:
    PROCESS_COUNT = cpu_count()  # number of parallel processes
except NotImplementedError:
    PROCESS_COUNT = 2  # dual cores are very common, now

def worker_delay(index, queue_tasks, queue_results):
    """
    Dummy worker that does not have any dependencies.

    Waits a random time for each message it gets from the queue.
    """
    # set payload weight once per lifetime
    delay = random.choice([1, 2, 5, 10])

    while True:
        msg = queue_tasks.get()
        if (msg == 'DONE'):
            queue_results.put(Result(index, 'DONE'))
            break

        # execute payload
        result = time.sleep(delay / 1000)

        # report each result back
        queue_results.put(Result(index, result))

# Datastructure for results generated by the workers
Result = namedtuple("Result", ["id", "performance"])

def worker_f(index, queue_tasks, queue_results):
    """Runs the imported f() with the last queue message as argument."""
    while True:
        args = queue_tasks.get()
        if (args == 'DONE'):
            queue_results.put(Result(index, 'DONE'))
            break

        # execute payload
        result = f(args)

        # report each result back
        queue_results.put(Result(index, result))

# data structure for a worker process and attached queues
Worker = namedtuple("Worker", ["id", "process", "queue_tasks", "queue_results"])

# data struture for a pool of workers
WorkerPool = namedtuple("WorkerPool", ["workers", "queue_results"])

def fill_worker_pool(process_count):
    """Returns a tuple consisting of workers and their common result queue."""
    # shared result queue
    queue_results = Queue()
    # list of all workers
    workers = []
    for worker_index in range(0, process_count):
        # queue for this worker's tasks
        queue_tasks = Queue()
        # construct a worker process with the queues
        worker_process = Process(target=worker_f,
                                 args=(worker_index, queue_tasks, queue_results))
        worker_process.daemon = True
        worker_process.start()
        workers.append(Worker(worker_index, worker_process, queue_tasks, \
                              queue_results))
    return (WorkerPool(workers, queue_results))

def send_tasks(worker_pool):
    """Sends a bunch of tasks to all workers in the given worker pool."""
    # Config for the ES
    #TODO Get this from the paramspec of f()
    args = {
      'mu' : 15,
      'lambd' : 100,
      'd' : 2,
      'tau0' : 0.5,
      'tau1' : 0.6,
      'epsilon' : 0.0001
    }

    # fill each worker's queue with task messages
    for worker in worker_pool.workers:
        worker.queue_tasks.put(args)
        worker.queue_tasks.put("DONE")

def listen_to_workers(worker_pool):
    """Listens to the result queue for all workers and dispatches handling."""
    # handle feedback from the workers
    worker_done_count = 0
    while len(worker_pool.workers) != worker_done_count:
        # get a message from the queue
        result = worker_pool.queue_results.get()

        # handle finished workers
        if result.performance == "DONE":
            worker = worker_pool.workers[result.id]
            worker.process.join()
            print("Worker %i finished after %f seconds." % \
                  (result.id, time.time() - _start))
            worker_pool.workers[result.id] = None  # delete reference
            worker_done_count += 1
            continue

        # handle all other results
        handle_result(result)

def handle_result(result):
    """Handles a result"""
    #TODO do actually handle the results
    pass

if __name__ == '__main__':
    _start = time.time()

    # create pool of workers
    worker_pool = fill_worker_pool(PROCESS_COUNT)

    # send tasks to workers
    send_tasks(worker_pool)

    # get results
    listen_to_workers(worker_pool)

    print("Sending a task to %i process workers via a Queue() took %s seconds" % \
          (PROCESS_COUNT, (time.time() - _start)))
<commit_msg>Reduce indent to two to comply with our convention<commit_after>#!/bin/python

"""
ProcessQueue - runs workers through queues
"""

from __future__ import division
from __future__ import print_function
from multiprocessing import Process, Queue, cpu_count
from collections import namedtuple
import time
import random

from test.algorithms.saes import f

# (Semi Auto) Configuration

try:
  PROCESS_COUNT = cpu_count()  # number of parallel processes
except NotImplementedError:
  PROCESS_COUNT = 2  # dual cores are very common, now

def worker_delay(index, queue_tasks, queue_results):
  """
  Dummy worker that does not have any dependencies.

  Waits a random time for each message it gets from the queue.
  """
  # set payload weight once per lifetime
  delay = random.choice([1, 2, 5, 10])

  while True:
    msg = queue_tasks.get()
    if (msg == 'DONE'):
      queue_results.put(Result(index, 'DONE'))
      break

    # execute payload
    result = time.sleep(delay / 1000)

    # report each result back
    queue_results.put(Result(index, result))

# Datastructure for results generated by the workers
Result = namedtuple("Result", ["id", "performance"])

def worker_f(index, queue_tasks, queue_results):
  """Runs the imported f() with the last queue message as argument."""
  while True:
    args = queue_tasks.get()
    if (args == 'DONE'):
      queue_results.put(Result(index, 'DONE'))
      break

    # execute payload
    result = f(args)

    # report each result back
    queue_results.put(Result(index, result))

# data structure for a worker process and attached queues
Worker = namedtuple("Worker", ["id", "process", "queue_tasks", "queue_results"])

# data struture for a pool of workers
WorkerPool = namedtuple("WorkerPool", ["workers", "queue_results"])

def fill_worker_pool(process_count):
  """Returns a tuple consisting of workers and their common result queue."""
  # shared result queue
  queue_results = Queue()
  # list of all workers
  workers = []
  for worker_index in range(0, process_count):
    # queue for this worker's tasks
    queue_tasks = Queue()
    # construct a worker process with the queues
    worker_process = Process(target=worker_f,
                 args=(worker_index, queue_tasks, queue_results))
    worker_process.daemon = True
    worker_process.start()
    workers.append(Worker(worker_index, worker_process, queue_tasks, \
                queue_results))
  return (WorkerPool(workers, queue_results))

def send_tasks(worker_pool):
  """Sends a bunch of tasks to all workers in the given worker pool."""
  # Config for the ES
  #TODO Get this from the paramspec of f()
  args = {
    'mu' : 15,
    'lambd' : 100,
    'd' : 2,
    'tau0' : 0.5,
    'tau1' : 0.6,
    'epsilon' : 0.0001
  }

  # fill each worker's queue with task messages
  for worker in worker_pool.workers:
    worker.queue_tasks.put(args)
    worker.queue_tasks.put("DONE")

def listen_to_workers(worker_pool):
  """Listens to the result queue for all workers and dispatches handling."""
  # handle feedback from the workers
  worker_done_count = 0
  while len(worker_pool.workers) != worker_done_count:
    # get a message from the queue
    result = worker_pool.queue_results.get()

    # handle finished workers
    if result.performance == "DONE":
      worker = worker_pool.workers[result.id]
      worker.process.join()
      print("Worker %i finished after %f seconds." % \
          (result.id, time.time() - _start))
      worker_pool.workers[result.id] = None  # delete reference
      worker_done_count += 1
      continue

    # handle all other results
    handle_result(result)

def handle_result(result):
  """Handles a result"""
  #TODO Actually handle the results
  pass

if __name__ == '__main__':
  _start = time.time()

  # create pool of workers
  worker_pool = fill_worker_pool(PROCESS_COUNT)

  # send tasks to workers
  send_tasks(worker_pool)

  # get results
  listen_to_workers(worker_pool)

  print("Sending a task to %i process workers via a Queue() took %s seconds" % \
      (PROCESS_COUNT, (time.time() - _start)))
<|endoftext|>