<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function

import os
import pprint
import re
import subprocess
import time

from heatclient.common import event_utils
from openstackclient import shell

from tripleoclient import exceptions
from tripleoclient import utils

from tripleoclient.workflows import base

def deploy(clients, **workflow_input):

    workflow_client = clients.workflow_engine
    tripleoclients = clients.tripleoclient
    wf_name = 'tripleo.deployment.v1.deploy_plan'

    with tripleoclients.messaging_websocket() as ws:
        execution = base.start_workflow(
            workflow_client,
            wf_name,
            workflow_input=workflow_input
        )

        # The deploy workflow ends once the Heat create/update starts. This
        # means that is shouldn't take very long. Wait for six minutes for
        # messages from the workflow.
        for payload in base.wait_for_messages(workflow_client, ws, execution,
                                              360):
            status = payload.get('status', 'RUNNING')
            if 'message' in payload and status == "RUNNING":
                print(payload['message'])

        if payload['status'] != "SUCCESS":
            pprint.pformat(payload)
            raise ValueError("Unexpected status %s for %s"
                             % (payload['status'], wf_name))

def deploy_and_wait(log, clients, stack, plan_name, verbose_level,
                    timeout=None, run_validations=False,
                    skip_deploy_identifier=False):
    """Start the deploy and wait for it to finish"""

    workflow_input = {
        "container": plan_name,
        "run_validations": run_validations,
        "skip_deploy_identifier": skip_deploy_identifier
    }

    if timeout is not None:
        workflow_input['timeout'] = timeout

    deploy(clients, **workflow_input)

    orchestration_client = clients.orchestration

    if stack is None:
        log.info("Performing Heat stack create")
        action = 'CREATE'
        marker = None
    else:
        log.info("Performing Heat stack update")
        # Make sure existing parameters for stack are reused
        # Find the last top-level event to use for the first marker
        events = event_utils.get_events(orchestration_client,
                                        stack_id=plan_name,
                                        event_args={'sort_dir': 'desc',
                                                    'limit': 1})
        marker = events[0].id if events else None
        action = 'UPDATE'

    time.sleep(10)
    verbose_events = verbose_level > 1
    create_result = utils.wait_for_stack_ready(
        orchestration_client, plan_name, marker, action, verbose_events)
    if not create_result:
        shell.OpenStackShell().run(["stack", "failures", "list", plan_name])
        if stack is None:
            raise exceptions.DeploymentError("Heat Stack create failed.")
        else:
            raise exceptions.DeploymentError("Heat Stack update failed.")

def overcloudrc(workflow_client, **input_):
    return base.call_action(workflow_client, 'tripleo.deployment.overcloudrc',
                            **input_)

def config_download(log, clients, stack, templates, deployed_server,
                    ssh_user, ssh_key, output_dir, verbosity=1):
    role_net_hostname_map = utils.get_role_net_hostname_map(stack)
    hostnames = []
    for role in role_net_hostname_map:
        hostnames.extend(role_net_hostname_map[role].get('ctlplane', []))

    ips = []
    hosts_entry = utils.get_hosts_entry(stack)
    for hostname in hostnames:
        for line in hosts_entry.split('\n'):
            match = re.search('\s*%s\s*' % hostname, line)
            if match:
                ips.append(line.split(' ')[0])

    script_path = os.path.join(templates,
                               'deployed-server',
                               'scripts',
                               'enable-ssh-admin.sh')

    env = os.environ.copy()
    env.update(dict(OVERCLOUD_HOSTS=' '.join(ips),
                    OVERCLOUD_SSH_USER=ssh_user))

    if ssh_key:
        env['OVERCLOUD_SSH_KEY'] = ssh_key

    proc = subprocess.Popen([script_path], env=env, shell=True,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)

    while True:
        line = proc.stdout.readline().decode('utf-8')
        if line:
            log.info(line.rstrip())
        if line == '' and proc.poll() is not None:
            break
    if proc.returncode != 0:
        raise RuntimeError('%s failed.' % script_path)

    workflow_client = clients.workflow_engine
    tripleoclients = clients.tripleoclient

    workflow_input = {
        'verbosity': verbosity or 1,
        'plan_name': stack.stack_name
    }
    if output_dir:
        workflow_input.update(dict(work_dir=output_dir))

    with tripleoclients.messaging_websocket() as ws:
        execution = base.start_workflow(
            workflow_client,
            'tripleo.deployment.v1.config_download_deploy',
            workflow_input=workflow_input
        )

        for payload in base.wait_for_messages(workflow_client, ws, execution,
                                              3600):
            print(payload['message'])

    if payload['status'] == 'SUCCESS':
        print("Overcloud configuration completed.")
    else:
        raise exceptions.DeploymentError("Overcloud configuration failed.")

def get_horizon_url(clients, **workflow_input):
    workflow_client = clients.workflow_engine
    tripleoclients = clients.tripleoclient

    with tripleoclients.messaging_websocket() as ws:
        execution = base.start_workflow(
            workflow_client,
            'tripleo.deployment.v1.get_horizon_url',
            workflow_input=workflow_input
        )

        for payload in base.wait_for_messages(workflow_client, ws, execution,
                                              360):
            assert payload['status'] == "SUCCESS"

            return payload['horizon_url']
<commit_msg>Actually print the error during deployment fail<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function

import os
import pprint
import re
import subprocess
import time

from heatclient.common import event_utils
from openstackclient import shell

from tripleoclient import exceptions
from tripleoclient import utils

from tripleoclient.workflows import base

def deploy(log, clients, **workflow_input):

    workflow_client = clients.workflow_engine
    tripleoclients = clients.tripleoclient
    wf_name = 'tripleo.deployment.v1.deploy_plan'

    with tripleoclients.messaging_websocket() as ws:
        execution = base.start_workflow(
            workflow_client,
            wf_name,
            workflow_input=workflow_input
        )

        # The deploy workflow ends once the Heat create/update starts. This
        # means that is shouldn't take very long. Wait for six minutes for
        # messages from the workflow.
        for payload in base.wait_for_messages(workflow_client, ws, execution,
                                              360):
            status = payload.get('status', 'RUNNING')
            if 'message' in payload and status == "RUNNING":
                print(payload['message'])

        if payload['status'] != "SUCCESS":
            log.info(pprint.pformat(payload))
            print(payload['message'])
            raise ValueError("Unexpected status %s for %s"
                             % (payload['status'], wf_name))

def deploy_and_wait(log, clients, stack, plan_name, verbose_level,
                    timeout=None, run_validations=False,
                    skip_deploy_identifier=False):
    """Start the deploy and wait for it to finish"""

    workflow_input = {
        "container": plan_name,
        "run_validations": run_validations,
        "skip_deploy_identifier": skip_deploy_identifier
    }

    if timeout is not None:
        workflow_input['timeout'] = timeout

    deploy(log, clients, **workflow_input)

    orchestration_client = clients.orchestration

    if stack is None:
        log.info("Performing Heat stack create")
        action = 'CREATE'
        marker = None
    else:
        log.info("Performing Heat stack update")
        # Make sure existing parameters for stack are reused
        # Find the last top-level event to use for the first marker
        events = event_utils.get_events(orchestration_client,
                                        stack_id=plan_name,
                                        event_args={'sort_dir': 'desc',
                                                    'limit': 1})
        marker = events[0].id if events else None
        action = 'UPDATE'

    time.sleep(10)
    verbose_events = verbose_level > 1
    create_result = utils.wait_for_stack_ready(
        orchestration_client, plan_name, marker, action, verbose_events)
    if not create_result:
        shell.OpenStackShell().run(["stack", "failures", "list", plan_name])
        if stack is None:
            raise exceptions.DeploymentError("Heat Stack create failed.")
        else:
            raise exceptions.DeploymentError("Heat Stack update failed.")

def overcloudrc(workflow_client, **input_):
    return base.call_action(workflow_client, 'tripleo.deployment.overcloudrc',
                            **input_)

def config_download(log, clients, stack, templates, deployed_server,
                    ssh_user, ssh_key, output_dir, verbosity=1):
    role_net_hostname_map = utils.get_role_net_hostname_map(stack)
    hostnames = []
    for role in role_net_hostname_map:
        hostnames.extend(role_net_hostname_map[role].get('ctlplane', []))

    ips = []
    hosts_entry = utils.get_hosts_entry(stack)
    for hostname in hostnames:
        for line in hosts_entry.split('\n'):
            match = re.search('\s*%s\s*' % hostname, line)
            if match:
                ips.append(line.split(' ')[0])

    script_path = os.path.join(templates,
                               'deployed-server',
                               'scripts',
                               'enable-ssh-admin.sh')

    env = os.environ.copy()
    env.update(dict(OVERCLOUD_HOSTS=' '.join(ips),
                    OVERCLOUD_SSH_USER=ssh_user))

    if ssh_key:
        env['OVERCLOUD_SSH_KEY'] = ssh_key

    proc = subprocess.Popen([script_path], env=env, shell=True,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)

    while True:
        line = proc.stdout.readline().decode('utf-8')
        if line:
            log.info(line.rstrip())
        if line == '' and proc.poll() is not None:
            break
    if proc.returncode != 0:
        raise RuntimeError('%s failed.' % script_path)

    workflow_client = clients.workflow_engine
    tripleoclients = clients.tripleoclient

    workflow_input = {
        'verbosity': verbosity or 1,
        'plan_name': stack.stack_name
    }
    if output_dir:
        workflow_input.update(dict(work_dir=output_dir))

    with tripleoclients.messaging_websocket() as ws:
        execution = base.start_workflow(
            workflow_client,
            'tripleo.deployment.v1.config_download_deploy',
            workflow_input=workflow_input
        )

        for payload in base.wait_for_messages(workflow_client, ws, execution,
                                              3600):
            print(payload['message'])

    if payload['status'] == 'SUCCESS':
        print("Overcloud configuration completed.")
    else:
        raise exceptions.DeploymentError("Overcloud configuration failed.")

def get_horizon_url(clients, **workflow_input):
    workflow_client = clients.workflow_engine
    tripleoclients = clients.tripleoclient

    with tripleoclients.messaging_websocket() as ws:
        execution = base.start_workflow(
            workflow_client,
            'tripleo.deployment.v1.get_horizon_url',
            workflow_input=workflow_input
        )

        for payload in base.wait_for_messages(workflow_client, ws, execution,
                                              360):
            assert payload['status'] == "SUCCESS"

            return payload['horizon_url']
<|endoftext|>


<commit_before>"""
Internal shared-state variables such as config settings and host lists.
"""

import os
import sys
from optparse import make_option

from fabric.network import HostConnectionCache
from fabric.version import get_version

#
# Win32 flag
#

# Impacts a handful of platform specific behaviors. Note that Cygwin's Python
# is actually close enough to "real" UNIXes that it doesn't need (or want!) to
# use PyWin32 -- so we only test for literal Win32 setups (vanilla Python,
# ActiveState etc) here.
win32 = (sys.platform == 'win32')

#
# Environment dictionary - support structures
#

class _AttributeDict(dict):
    """
    Dictionary subclass enabling attribute lookup/assignment of keys/values.

    For example::

        >>> m = _AttributeDict({'foo': 'bar'})
        >>> m.foo
        'bar'
        >>> m.foo = 'not bar'
        >>> m['foo']
        'not bar'

    ``_AttributeDict`` objects also provide ``.first()`` which acts like
    ``.get()`` but accepts multiple keys as arguments, and returns the value of
    the first hit, e.g.::

        >>> m = _AttributeDict({'foo': 'bar', 'biz': 'baz'})
        >>> m.first('wrong', 'incorrect', 'foo', 'biz')
        'bar'

    """
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            # to conform with __getattr__ spec
            raise AttributeError(key)

    def __setattr__(self, key, value):
        self[key] = value

    def first(self, *names):
        for name in names:
            value = self.get(name)
            if value:
                return value

# By default, if the user (including code using Fabric as a library) doesn't
# set the username, we obtain the currently running username and use that.
def _get_system_username():
    """
    Obtain name of current system user, which will be default connection user.
    """
    if not win32:
        import pwd
        return pwd.getpwuid(os.getuid())[0]
    else:
        import win32api
        import win32security
        import win32profile
        return win32api.GetUserName()

def _rc_path():
    """
    Return platform-specific default file path for $HOME/.fabricrc.
    """
    rc_file = '.fabricrc'
    if not win32:
        return os.path.expanduser("~/" + rc_file)
    else:
        from win32com.shell.shell import SHGetSpecialFolderPath
        from win32com.shell.shellcon import CSIDL_PROFILE
        return "%s/%s" % (
            SHGetSpecialFolderPath(0,CSIDL_PROFILE),
            rc_file
        )

# Options/settings which exist both as environment keys and which can be set on
# the command line, are defined here. When used via `fab` they will be added to
# the optparse parser, and either way they are added to `env` below (i.e.  the
# 'dest' value becomes the environment key and the value, the env value).
#
# Keep in mind that optparse changes hyphens to underscores when automatically
# deriving the `dest` name, e.g. `--reject-unknown-hosts` becomes
# `reject_unknown_hosts`.
#
# Furthermore, *always* specify some sort of default to avoid ending up with
# optparse.NO_DEFAULT (currently a two-tuple)! In general, None is a better
# default than ''.
#
# User-facing documentation for these are kept in docs/env.rst.
env_options = [

    make_option('-r', '--reject-unknown-hosts',
        action='store_true',
        default=False,
        help="reject unknown hosts"
    ),

    make_option('-D', '--disable-known-hosts',
        action='store_true',
        default=False,
        help="do not load user known_hosts file"
    ),

    make_option('-u', '--user',
        default=_get_system_username(),
        help="username to use when connecting to remote hosts"
    ),

    make_option('-p', '--password',
        default=None,
        help="password for use with authentication and/or sudo"
    ),

    make_option('-H', '--hosts',
        default=[],
        help="comma-separated list of hosts to operate on"
    ),

    make_option('-R', '--roles',
        default=[],
        help="comma-separated list of roles to operate on"
    ),

    make_option('-i',
        action='append',
        dest='key_filename',
        default=None,
        help="path to SSH private key file. May be repeated."
    ),

    # Use -a here to mirror ssh(1) options.
    make_option('-a', '--no_agent',
        action='store_true',
        default=False,
        help="don't use the running SSH agent"
    ),

    # No matching option for ssh(1) so just picked something appropriate.
    make_option('-k', '--no-keys',
        action='store_true',
        default=False,
        help="don't load private key files from ~/.ssh/"
    ),

    make_option('-f', '--fabfile',
        default='fabfile',
        help="Python module file to import, e.g. '../other.py'"
    ),

    make_option('-w', '--warn-only',
        action='store_true',
        default=False,
        help="warn, instead of abort, when commands fail"
    ),

    make_option('-s', '--shell',
        default='/bin/bash -l -c',
        help="specify a new shell, defaults to '/bin/bash -l -c'"
    ),

    make_option('-c', '--config',
        dest='rcfile',
        default=_rc_path(),
        help="specify location of config file to use"
    ),

    # Verbosity controls, analogous to context_managers.(hide|show)
    make_option('--hide',
        metavar='LEVELS',
        help="comma-separated list of output levels to hide"
    ),
    make_option('--show',
        metavar='LEVELS',
        help="comma-separated list of output levels to show"
    ),

    # Global PTY flag for run/sudo
    make_option('--pty',
        dest='always_use_pty',
        action='store_true',
        default=False,
        help="force use of pseudo-terminal in run/sudo"
    )

]

#
# Environment dictionary - actual dictionary object
#

# Global environment dict. Currently a catchall for everything: config settings
# such as global deep/broad mode, host lists, username etc.
# Most default values are specified in `env_options` above, in the interests of
# preserving DRY: anything in here is generally not settable via the command
# line.
env = _AttributeDict({
    'all_hosts': None,
    'command': None,
    'cwd': '', # Must be empty string, not None, for concatenation purposes
    'host': None,
    'host_string': None,
    'port': None,
    'real_fabfile': None,
    'roledefs': {},
    'sudo_prompt': 'sudo password:',
    # -S so sudo accepts passwd via stdin, -p with our known-value prompt for
    # later detection (thus %s -- gets filled with env.sudo_prompt at runtime)
    'sudo_prefix': "sudo -S -p '%s' ",
    'again_prompt': 'Sorry, try again.\n',
    'use_shell': True,
    'roledefs': {},
    'path': '',
    'path_behavior': 'append',
    'user': None,
    'version': get_version('short'),
    'command_prefixes': [],
})

# Add in option defaults
for option in env_options:
    env[option.dest] = option.default

#
# Command dictionary
#

# Keys are the command/function names, values are the callables themselves.
# This is filled in when main() runs.
commands = {}

#
# Host connection dict/cache
#

connections = HostConnectionCache()

#
# Output controls
#

class _AliasDict(_AttributeDict):
    """
    `_AttributeDict` subclass that allows for "aliasing" of keys to other keys.

    Upon creation, takes an ``aliases`` mapping, which should map alias names
    to lists of key names. Aliases do not store their own value, but instead
    set (override) all mapped keys' values. For example, in the following
    `_AliasDict`, calling ``mydict['foo'] = True`` will set the values of
    ``mydict['bar']``, ``mydict['biz']`` and ``mydict['baz']`` all to True::

        mydict = _AliasDict(
            {'biz': True, 'baz': False},
            aliases={'foo': ['bar', 'biz', 'baz']}
        )

    Because it is possible for the aliased values to be in a heterogenous
    state, reading aliases is not supported -- only writing to them is allowed.
    This also means they will not show up in e.g. ``dict.keys()``.

    ..note::

        Aliases are recursive, so you may refer to an alias within the key list
        of another alias. Naturally, this means that you can end up with
        infinite loops if you're not careful.

    `_AliasDict` provides a special function, `expand_aliases`, which will take
    a list of keys as an argument and will return that list of keys with any
    aliases expanded. This function will **not** dedupe, so any aliases which
    overlap will result in duplicate keys in the resulting list.
    """
    def __init__(self, arg=None, aliases=None):
        init = super(_AliasDict, self).__init__
        if arg is not None:
            init(arg)
        else:
            init()
        # Can't use super() here because of _AttributeDict's setattr override
        dict.__setattr__(self, 'aliases', aliases)

    def __setitem__(self, key, value):
        if key in self.aliases:
            for aliased in self.aliases[key]:
                self[aliased] = value
        else:
            return super(_AliasDict, self).__setitem__(key, value)

    def expand_aliases(self, keys):
        ret = []
        for key in keys:
            if key in self.aliases:
                ret.extend(self.expand_aliases(self.aliases[key]))
            else:
                ret.append(key)
        return ret

# Keys are "levels" or "groups" of output, values are always boolean,
# determining whether output falling into the given group is printed or not
# printed.
#
# By default, everything except 'debug' is printed, as this is what the average
# user, and new users, are most likely to expect.
#
# See docs/usage.rst for details on what these levels mean.
output = _AliasDict({
    'status': True,
    'aborts': True,
    'warnings': True,
    'running': True,
    'stdout': True,
    'stderr': True,
    'debug': False,
    'user': True
}, aliases={
    'everything': ['warnings', 'running', 'user', 'output'],
    'output': ['stdout', 'stderr']
})
<commit_msg>Sort initial env dict<commit_after>"""
Internal shared-state variables such as config settings and host lists.
"""

import os
import sys
from optparse import make_option

from fabric.network import HostConnectionCache
from fabric.version import get_version

#
# Win32 flag
#

# Impacts a handful of platform specific behaviors. Note that Cygwin's Python
# is actually close enough to "real" UNIXes that it doesn't need (or want!) to
# use PyWin32 -- so we only test for literal Win32 setups (vanilla Python,
# ActiveState etc) here.
win32 = (sys.platform == 'win32')

#
# Environment dictionary - support structures
#

class _AttributeDict(dict):
    """
    Dictionary subclass enabling attribute lookup/assignment of keys/values.

    For example::

        >>> m = _AttributeDict({'foo': 'bar'})
        >>> m.foo
        'bar'
        >>> m.foo = 'not bar'
        >>> m['foo']
        'not bar'

    ``_AttributeDict`` objects also provide ``.first()`` which acts like
    ``.get()`` but accepts multiple keys as arguments, and returns the value of
    the first hit, e.g.::

        >>> m = _AttributeDict({'foo': 'bar', 'biz': 'baz'})
        >>> m.first('wrong', 'incorrect', 'foo', 'biz')
        'bar'

    """
    def __getattr__(self, key):
        try:
            return self[key]
        except KeyError:
            # to conform with __getattr__ spec
            raise AttributeError(key)

    def __setattr__(self, key, value):
        self[key] = value

    def first(self, *names):
        for name in names:
            value = self.get(name)
            if value:
                return value

# By default, if the user (including code using Fabric as a library) doesn't
# set the username, we obtain the currently running username and use that.
def _get_system_username():
    """
    Obtain name of current system user, which will be default connection user.
    """
    if not win32:
        import pwd
        return pwd.getpwuid(os.getuid())[0]
    else:
        import win32api
        import win32security
        import win32profile
        return win32api.GetUserName()

def _rc_path():
    """
    Return platform-specific default file path for $HOME/.fabricrc.
    """
    rc_file = '.fabricrc'
    if not win32:
        return os.path.expanduser("~/" + rc_file)
    else:
        from win32com.shell.shell import SHGetSpecialFolderPath
        from win32com.shell.shellcon import CSIDL_PROFILE
        return "%s/%s" % (
            SHGetSpecialFolderPath(0,CSIDL_PROFILE),
            rc_file
        )

# Options/settings which exist both as environment keys and which can be set on
# the command line, are defined here. When used via `fab` they will be added to
# the optparse parser, and either way they are added to `env` below (i.e.  the
# 'dest' value becomes the environment key and the value, the env value).
#
# Keep in mind that optparse changes hyphens to underscores when automatically
# deriving the `dest` name, e.g. `--reject-unknown-hosts` becomes
# `reject_unknown_hosts`.
#
# Furthermore, *always* specify some sort of default to avoid ending up with
# optparse.NO_DEFAULT (currently a two-tuple)! In general, None is a better
# default than ''.
#
# User-facing documentation for these are kept in docs/env.rst.
env_options = [

    make_option('-r', '--reject-unknown-hosts',
        action='store_true',
        default=False,
        help="reject unknown hosts"
    ),

    make_option('-D', '--disable-known-hosts',
        action='store_true',
        default=False,
        help="do not load user known_hosts file"
    ),

    make_option('-u', '--user',
        default=_get_system_username(),
        help="username to use when connecting to remote hosts"
    ),

    make_option('-p', '--password',
        default=None,
        help="password for use with authentication and/or sudo"
    ),

    make_option('-H', '--hosts',
        default=[],
        help="comma-separated list of hosts to operate on"
    ),

    make_option('-R', '--roles',
        default=[],
        help="comma-separated list of roles to operate on"
    ),

    make_option('-i',
        action='append',
        dest='key_filename',
        default=None,
        help="path to SSH private key file. May be repeated."
    ),

    # Use -a here to mirror ssh(1) options.
    make_option('-a', '--no_agent',
        action='store_true',
        default=False,
        help="don't use the running SSH agent"
    ),

    # No matching option for ssh(1) so just picked something appropriate.
    make_option('-k', '--no-keys',
        action='store_true',
        default=False,
        help="don't load private key files from ~/.ssh/"
    ),

    make_option('-f', '--fabfile',
        default='fabfile',
        help="Python module file to import, e.g. '../other.py'"
    ),

    make_option('-w', '--warn-only',
        action='store_true',
        default=False,
        help="warn, instead of abort, when commands fail"
    ),

    make_option('-s', '--shell',
        default='/bin/bash -l -c',
        help="specify a new shell, defaults to '/bin/bash -l -c'"
    ),

    make_option('-c', '--config',
        dest='rcfile',
        default=_rc_path(),
        help="specify location of config file to use"
    ),

    # Verbosity controls, analogous to context_managers.(hide|show)
    make_option('--hide',
        metavar='LEVELS',
        help="comma-separated list of output levels to hide"
    ),
    make_option('--show',
        metavar='LEVELS',
        help="comma-separated list of output levels to show"
    ),

    # Global PTY flag for run/sudo
    make_option('--pty',
        dest='always_use_pty',
        action='store_true',
        default=False,
        help="force use of pseudo-terminal in run/sudo"
    )

]

#
# Environment dictionary - actual dictionary object
#

# Global environment dict. Currently a catchall for everything: config settings
# such as global deep/broad mode, host lists, username etc.
# Most default values are specified in `env_options` above, in the interests of
# preserving DRY: anything in here is generally not settable via the command
# line.
env = _AttributeDict({
    'again_prompt': 'Sorry, try again.\n',
    'all_hosts': None,
    'command': None,
    'command_prefixes': [],
    'cwd': '', # Must be empty string, not None, for concatenation purposes
    'host': None,
    'host_string': None,
    'path': '',
    'path_behavior': 'append',
    'port': None,
    'real_fabfile': None,
    'roledefs': {},
    'roledefs': {},
    # -S so sudo accepts passwd via stdin, -p with our known-value prompt for
    # later detection (thus %s -- gets filled with env.sudo_prompt at runtime)
    'sudo_prefix': "sudo -S -p '%s' ",
    'sudo_prompt': 'sudo password:',
    'use_shell': True,
    'user': None,
    'version': get_version('short'),
})

# Add in option defaults
for option in env_options:
    env[option.dest] = option.default

#
# Command dictionary
#

# Keys are the command/function names, values are the callables themselves.
# This is filled in when main() runs.
commands = {}

#
# Host connection dict/cache
#

connections = HostConnectionCache()

#
# Output controls
#

class _AliasDict(_AttributeDict):
    """
    `_AttributeDict` subclass that allows for "aliasing" of keys to other keys.

    Upon creation, takes an ``aliases`` mapping, which should map alias names
    to lists of key names. Aliases do not store their own value, but instead
    set (override) all mapped keys' values. For example, in the following
    `_AliasDict`, calling ``mydict['foo'] = True`` will set the values of
    ``mydict['bar']``, ``mydict['biz']`` and ``mydict['baz']`` all to True::

        mydict = _AliasDict(
            {'biz': True, 'baz': False},
            aliases={'foo': ['bar', 'biz', 'baz']}
        )

    Because it is possible for the aliased values to be in a heterogenous
    state, reading aliases is not supported -- only writing to them is allowed.
    This also means they will not show up in e.g. ``dict.keys()``.

    ..note::

        Aliases are recursive, so you may refer to an alias within the key list
        of another alias. Naturally, this means that you can end up with
        infinite loops if you're not careful.

    `_AliasDict` provides a special function, `expand_aliases`, which will take
    a list of keys as an argument and will return that list of keys with any
    aliases expanded. This function will **not** dedupe, so any aliases which
    overlap will result in duplicate keys in the resulting list.
    """
    def __init__(self, arg=None, aliases=None):
        init = super(_AliasDict, self).__init__
        if arg is not None:
            init(arg)
        else:
            init()
        # Can't use super() here because of _AttributeDict's setattr override
        dict.__setattr__(self, 'aliases', aliases)

    def __setitem__(self, key, value):
        if key in self.aliases:
            for aliased in self.aliases[key]:
                self[aliased] = value
        else:
            return super(_AliasDict, self).__setitem__(key, value)

    def expand_aliases(self, keys):
        ret = []
        for key in keys:
            if key in self.aliases:
                ret.extend(self.expand_aliases(self.aliases[key]))
            else:
                ret.append(key)
        return ret

# Keys are "levels" or "groups" of output, values are always boolean,
# determining whether output falling into the given group is printed or not
# printed.
#
# By default, everything except 'debug' is printed, as this is what the average
# user, and new users, are most likely to expect.
#
# See docs/usage.rst for details on what these levels mean.
output = _AliasDict({
    'status': True,
    'aborts': True,
    'warnings': True,
    'running': True,
    'stdout': True,
    'stderr': True,
    'debug': False,
    'user': True
}, aliases={
    'everything': ['warnings', 'running', 'user', 'output'],
    'output': ['stdout', 'stderr']
})
<|endoftext|>


<commit_before>import json
import re
import os
import math

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from collections import Counter

EXTRACT_RE = r"[\w]+\(?|:{2}|<{2}|:\n|<-|[.,@!?;&{}\[\]#\\*?/|%$]"
STRING_RE = r"([\"\'])(?:(?=(\\?))\2.)*?\1"
COMMENTS = ["//", "-", "#", "*"]

def get_parts(src, is_file=False, filtered=[]):
    """Attempt to extract all non-string and non-comment content from `src`.

    Args:
        `src` (str): Either a string or a file path.
        `is_file` (bool): `True` if `src` is a file.
        `filtered` (list): A list of words to keep (others will be ignored).

    Returns:
     (list, int): The list of extracted content and the total number of lines
        examined.
    """
    lines = 0.0
    parts = []

    if is_file:
        text = open(src)
    else:
        text = StringIO(src)

    for line in text:
        line = line.lstrip()
        if not line or any(line.startswith(c) for c in COMMENTS):
            continue
        line = re.sub(STRING_RE, '', line)
        extr = re.findall(EXTRACT_RE, line)
        parts.extend([s for s in extr if s in filtered or not filtered])
        lines += 1

    text.close()
    return parts, lines

def compare_signatures(unknown, known):
    """Compare two signatures using only the keys in `known`.

    Args:
        `unknown` (dict): A signature for an unknown language.
        `known` (dict): A signature for a known language.

    Returns:
        float: A score indicating how closely `unknown` resembled `known`
    """
    total = 1.0
    found = 0.0
    for k, v in known.items():
        test_value = unknown.get(k)
        if test_value:
            total += math.fabs(v - test_value)
            found += 1
        elif v > 0.10:
            total += 1
            found -= 1
    return found / total

def compute_signature(src, is_file=False, lang=None, ext=[]):
    """Compute a signature for `src`.

    Args:
        `src` (str): Either a string or a file path.
        `is_file` (bool): `True` if `src` is a file.
        `lang` (str): The name of the language `src` is written in.
        `ext` (list): A list of file extensions associated with `lang`.

    Returns:
     dict: A dictionary with words as keys and their frequency per line as
        values in `src`.
    """
    lines = 0
    parts = []
    words = get_lang_data(lang)
    if not os.path.isdir(src):
        sparts, slines = get_parts(src, is_file=is_file, filtered=words)
        parts.extend(sparts)
        lines += slines
    else:
        for subdir, dirs, files in os.walk(src):
            for f in files:
                if ext and not any(f.endswith(e) for e in ext):
                    continue
                sparts, slines = get_parts(
                    os.path.join(subdir, f),
                    is_file=is_file,
                    filtered=words
                )
                parts.extend(sparts)
                lines += slines
    signature = Counter(parts)
    for key in signature:
        signature[key] /= lines
    return signature

def read_signature(lang):
    """Load an existing signature.

    Args:
        `lang`: The name of the existing signature.
    """
    with open(os.path.join("signatures", lang + ".json")) as sig:
        return json.load(sig)

def get_lang_data(lang):
    """Load existing data on `lang`.

    Args:
        `lang`: The name of the language.

    Returns:
        list: A list of all keywords associated with `lang`.
    """
    d = []
    if lang is None:
        return d
    with open(os.path.join("data", lang + ".json")) as jdata:
        d = json.load(jdata)
    return sum([d[s] for s in d.keys()], [])

def write_signature(src, lang, ext, is_file=True):
    """Write a signature for `src`.

    Args:
        `src` (str): Either a string or a file path.
        `lang`: The name of the language.
        `ext`: A list of file extensions associated with `lang`.
        `is_file` (bool): `True` if `src` is a file.
    """
    data = compute_signature(src, lang, ext, is_file=is_file)
    with open(os.path.join("signatures", lang + ".json"), "w+") as sig:
        json.dump(data, sig, indent=4, sort_keys=True)
<commit_msg>Use StringIO from io for Python 3<commit_after>import json
import re
import os
import math

try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

from collections import Counter

EXTRACT_RE = r"[\w]+\(?|:{2}|<{2}|:\n|<-|[.,@!?;&{}\[\]#\\*?/|%$]"
STRING_RE = r"([\"\'])(?:(?=(\\?))\2.)*?\1"
COMMENTS = ["//", "-", "#", "*"]

def get_parts(src, is_file=False, filtered=[]):
    """Attempt to extract all non-string and non-comment content from `src`.

    Args:
        `src` (str): Either a string or a file path.
        `is_file` (bool): `True` if `src` is a file.
        `filtered` (list): A list of words to keep (others will be ignored).

    Returns:
     (list, int): The list of extracted content and the total number of lines
        examined.
    """
    lines = 0.0
    parts = []

    if is_file:
        text = open(src)
    else:
        text = StringIO(src)

    for line in text:
        line = line.lstrip()
        if not line or any(line.startswith(c) for c in COMMENTS):
            continue
        line = re.sub(STRING_RE, '', line)
        extr = re.findall(EXTRACT_RE, line)
        parts.extend([s for s in extr if s in filtered or not filtered])
        lines += 1

    text.close()
    return parts, lines

def compare_signatures(unknown, known):
    """Compare two signatures using only the keys in `known`.

    Args:
        `unknown` (dict): A signature for an unknown language.
        `known` (dict): A signature for a known language.

    Returns:
        float: A score indicating how closely `unknown` resembled `known`
    """
    total = 1.0
    found = 0.0
    for k, v in known.items():
        test_value = unknown.get(k)
        if test_value:
            total += math.fabs(v - test_value)
            found += 1
        elif v > 0.10:
            total += 1
            found -= 1
    return found / total

def compute_signature(src, is_file=False, lang=None, ext=[]):
    """Compute a signature for `src`.

    Args:
        `src` (str): Either a string or a file path.
        `is_file` (bool): `True` if `src` is a file.
        `lang` (str): The name of the language `src` is written in.
        `ext` (list): A list of file extensions associated with `lang`.

    Returns:
     dict: A dictionary with words as keys and their frequency per line as
        values in `src`.
    """
    lines = 0
    parts = []
    words = get_lang_data(lang)
    if not os.path.isdir(src):
        sparts, slines = get_parts(src, is_file=is_file, filtered=words)
        parts.extend(sparts)
        lines += slines
    else:
        for subdir, dirs, files in os.walk(src):
            for f in files:
                if ext and not any(f.endswith(e) for e in ext):
                    continue
                sparts, slines = get_parts(
                    os.path.join(subdir, f),
                    is_file=is_file,
                    filtered=words
                )
                parts.extend(sparts)
                lines += slines
    signature = Counter(parts)
    for key in signature:
        signature[key] /= lines
    return signature

def read_signature(lang):
    """Load an existing signature.

    Args:
        `lang`: The name of the existing signature.
    """
    with open(os.path.join("signatures", lang + ".json")) as sig:
        return json.load(sig)

def get_lang_data(lang):
    """Load existing data on `lang`.

    Args:
        `lang`: The name of the language.

    Returns:
        list: A list of all keywords associated with `lang`.
    """
    d = []
    if lang is None:
        return d
    with open(os.path.join("data", lang + ".json")) as jdata:
        d = json.load(jdata)
    return sum([d[s] for s in d.keys()], [])

def write_signature(src, lang, ext, is_file=True):
    """Write a signature for `src`.

    Args:
        `src` (str): Either a string or a file path.
        `lang`: The name of the language.
        `ext`: A list of file extensions associated with `lang`.
        `is_file` (bool): `True` if `src` is a file.
    """
    data = compute_signature(src, lang, ext, is_file=is_file)
    with open(os.path.join("signatures", lang + ".json"), "w+") as sig:
        json.dump(data, sig, indent=4, sort_keys=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from astropy import units
import unittest
from ibei.physicalproperty import PhysicalProperty

# Classes used in tests
# =====================
class MockClassEmpty(object):
    """
    Class containing nothing.
    """
    pass

class MockClassDefaultInstantiation(object):
    """
    Default instantiated `PhysicalProperty` public data attribute.
    """
    attrib = PhysicalProperty()

class MockClassUserDefinedUnitAmp(object):
    """
    `PhysicalProperty` public data attribute w/ user-defined unit as amp.
    """
    attrib = PhysicalProperty(unit = "A")

    def __init__(self, current = 1):
        """
        Initialize the `PhysicalProperty` public data attribute.
        """
        self.attrib = current

# Tests
# =====
class StandaloneInstantiation(unittest.TestCase):
    """
    PhysicalProperty instantiated as a standalone object.
    """
    def test_instantiate_lo_bnd_greater_than_up_bnd(self):
        """
        ValueError raised if instantiated with up_bnd < lo_bnd.
        """
        self.assertRaises(ValueError, PhysicalProperty, "A", -1, 1)

    # __set__ method tests
    # ====================
    def test_set_non_numeric(self):
        """
        Set with non-numeric, non-Quantity input raises TypeError
        """
        pp = PhysicalProperty()
        self.assertRaises(TypeError, pp.__set__, MockClassEmpty, "not a number")

    def test_set_Quantity_incompatible_units(self):
        """
        Raise UnitsException when setting with Quantity of incompatible units.
        """
        pp = PhysicalProperty()
        qty = units.Quantity(100., "km")

        self.assertRaises(units.UnitsException, pp.__set__, MockClassEmpty, qty)

    def test_set_numeric_above_up_bnd(self):
        """
        Raises ValueError when attempting to set with numeric above up_bnd.
        """
        ubnd = 10.

        pp = PhysicalProperty(up_bnd = ubnd)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, 12.)

    def test_set_numeric_below_lo_bnd(self):
        """
        Raises ValueError when attempting to set with numeric below lo_bnd.
        """
        lbnd = -10.

        pp = PhysicalProperty(lo_bnd = lbnd)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, -12.)

    def test_set_Quantity_above_up_bnd_same_units(self):
        """
        Raises ValueError when attempting to set with Quantity in same units above up_bnd.
        """
        ubnd = 10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, up_bnd = ubnd)

        qty = units.Quantity(12., unit)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

    def test_set_Quantity_above_up_bnd_compatible_units(self):
        """
        Raises ValueError when attempting to set with Quantity in compatible units above up_bnd.
        """
        ubnd = 10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, up_bnd = ubnd)

        # Note 1e3mm = 1m. Thus, 1m > 10mm.
        qty = units.Quantity(1., "m")
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

    def test_set_Quantity_below_lo_bnd_same_units(self):
        """
        Raises ValueError when attempting to set with Quantity in same units below lo_bnd.
        """
        lbnd = -10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, lo_bnd = lbnd)

        qty = units.Quantity(-12., unit)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

    def test_set_Quantity_below_lo_bnd_compatible_units(self):
        """
        Raises ValueError when attempting to set with Quantity in compatible units below lo_bnd.
        """
        lbnd = -10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, lo_bnd = lbnd)

        # Note 1e3mm = 1m. Thus, 1m > 10mm.
        qty = units.Quantity(-1., "m")
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

class SimpleInstantiation(unittest.TestCase):
    """
    Instantiation with no arguments from within an owner class.

    These tests require some simple manipulation of the owner class's `PhysicalProperty` data attribute via the `PhysicalProperty.__set__` and `PhysicalProperty.__get__` methods.
    """
    def setUp(self):
        """
        A MockClassDefaultInstantiation instance for the tests.
        """
        self.mcdi = MockClassDefaultInstantiation()

    def tearDown(self):
        """
        I'm pretty sure I have to destroy self.mcdi.
        """
        del self.mcdi

    def test_default_value(self):
        """
        Default value should be `None`.
        """
        self.assertEqual(self.mcdi.attrib, None)

    def test_modifying_value_returns_Quantity(self):
        """
        Valid `__set__` yields `astropy.units.Quantity` on later `__get__` call
        """
        self.mcdi.attrib = 7.
        self.assertIsInstance(self.mcdi.attrib, units.Quantity)

    def test_default_unit(self):
        """
        Default unit is `astropy.units.dimensionless_unscaled`.
        """
        self.mcdi.attrib = 7.
        self.assertEqual(self.mcdi.attrib.unit, units.dimensionless_unscaled)

class InstantiationWithUnitArg(unittest.TestCase):
    """
    Instantiation with unit arguments from within an owner class.
    """
    def setUp(self):
        """
        A MockClassUserDefinedUnitAmp instance for the tests.
        """
        self.mcudua = MockClassUserDefinedUnitAmp()

    def tearDown(self):
        """
        Destroy the mock class used in the test.
        """
        del self.mcudua

    def test_unit(self):
        """
        Units are in amps.
        """
        self.assertEqual(self.mcudua.attrib.unit, "A")

    def test_set_with_number_roundtrip(self):
        """
        Set `attrib` with numeric type yields same numeric value on later `__get__` call.
        """
        number = 100.
        self.mcudua.attrib = number
        self.assertEqual(self.mcudua.attrib.value, number)

    def test_set_with_Quantity_stores_a_copy(self):
        """
        When setting the object with a Quantity, the object should store a new copy of the Quantity and not simply point to the original.

        This test tests two things:
        1. `__set__`ting with an `astropy.units.Quantity` works.
        2. The `PhysicalProperty` object stores a Quantity object that's different from the one passed to its `__set__` method.
        """
        qty = units.Quantity(100., "A")
        self.mcudua.attrib = qty

        self.assertIsNot(self.mcudua.attrib, qty)

    def test_set_with_Quantity_units_dont_change(self):
        """
        Set with Quantity with compatible units doesn't change `attrib` units.
        """
        qty = units.Quantity(100., "C/s")
        self.mcudua.attrib = qty

        self.assertEqual(self.mcudua.attrib.unit, "A")

class MultipleOwnerInstances(unittest.TestCase):
    """
    Multiple owner instances shouldn't interfere with one another.
    """
    def test_two_owners_individually_set_attribute(self):
        """
        Changing a `PhysicalProperty` attribute value of one owner instance shouldn't alter another owner instance.
        """
        val1 = 10.
        val2 = 3.

        mock1 = MockClassDefaultInstantiation()
        mock2 = MockClassDefaultInstantiation()

        mock1.attrib = val1
        mock2.attrib = val2

        self.assertEqual(mock1.attrib.value, val1)
<commit_msg>Change units.UnitsException to units.UnitsError<commit_after># -*- coding: utf-8 -*-
from astropy import units
import unittest
from ibei.physicalproperty import PhysicalProperty

# Classes used in tests
# =====================
class MockClassEmpty(object):
    """
    Class containing nothing.
    """
    pass

class MockClassDefaultInstantiation(object):
    """
    Default instantiated `PhysicalProperty` public data attribute.
    """
    attrib = PhysicalProperty()

class MockClassUserDefinedUnitAmp(object):
    """
    `PhysicalProperty` public data attribute w/ user-defined unit as amp.
    """
    attrib = PhysicalProperty(unit = "A")

    def __init__(self, current = 1):
        """
        Initialize the `PhysicalProperty` public data attribute.
        """
        self.attrib = current

# Tests
# =====
class StandaloneInstantiation(unittest.TestCase):
    """
    PhysicalProperty instantiated as a standalone object.
    """
    def test_instantiate_lo_bnd_greater_than_up_bnd(self):
        """
        ValueError raised if instantiated with up_bnd < lo_bnd.
        """
        self.assertRaises(ValueError, PhysicalProperty, "A", -1, 1)

    # __set__ method tests
    # ====================
    def test_set_non_numeric(self):
        """
        Set with non-numeric, non-Quantity input raises TypeError
        """
        pp = PhysicalProperty()
        self.assertRaises(TypeError, pp.__set__, MockClassEmpty, "not a number")

    def test_set_Quantity_incompatible_units(self):
        """
        Raise UnitsException when setting with Quantity of incompatible units.
        """
        pp = PhysicalProperty()
        qty = units.Quantity(100., "km")

        self.assertRaises(units.UnitsError, pp.__set__, MockClassEmpty, qty)

    def test_set_numeric_above_up_bnd(self):
        """
        Raises ValueError when attempting to set with numeric above up_bnd.
        """
        ubnd = 10.

        pp = PhysicalProperty(up_bnd = ubnd)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, 12.)

    def test_set_numeric_below_lo_bnd(self):
        """
        Raises ValueError when attempting to set with numeric below lo_bnd.
        """
        lbnd = -10.

        pp = PhysicalProperty(lo_bnd = lbnd)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, -12.)

    def test_set_Quantity_above_up_bnd_same_units(self):
        """
        Raises ValueError when attempting to set with Quantity in same units above up_bnd.
        """
        ubnd = 10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, up_bnd = ubnd)

        qty = units.Quantity(12., unit)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

    def test_set_Quantity_above_up_bnd_compatible_units(self):
        """
        Raises ValueError when attempting to set with Quantity in compatible units above up_bnd.
        """
        ubnd = 10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, up_bnd = ubnd)

        # Note 1e3mm = 1m. Thus, 1m > 10mm.
        qty = units.Quantity(1., "m")
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

    def test_set_Quantity_below_lo_bnd_same_units(self):
        """
        Raises ValueError when attempting to set with Quantity in same units below lo_bnd.
        """
        lbnd = -10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, lo_bnd = lbnd)

        qty = units.Quantity(-12., unit)
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

    def test_set_Quantity_below_lo_bnd_compatible_units(self):
        """
        Raises ValueError when attempting to set with Quantity in compatible units below lo_bnd.
        """
        lbnd = -10.
        unit = "mm"
        pp = PhysicalProperty(unit = unit, lo_bnd = lbnd)

        # Note 1e3mm = 1m. Thus, 1m > 10mm.
        qty = units.Quantity(-1., "m")
        self.assertRaises(ValueError, pp.__set__, MockClassEmpty, qty)

class SimpleInstantiation(unittest.TestCase):
    """
    Instantiation with no arguments from within an owner class.

    These tests require some simple manipulation of the owner class's `PhysicalProperty` data attribute via the `PhysicalProperty.__set__` and `PhysicalProperty.__get__` methods.
    """
    def setUp(self):
        """
        A MockClassDefaultInstantiation instance for the tests.
        """
        self.mcdi = MockClassDefaultInstantiation()

    def tearDown(self):
        """
        I'm pretty sure I have to destroy self.mcdi.
        """
        del self.mcdi

    def test_default_value(self):
        """
        Default value should be `None`.
        """
        self.assertEqual(self.mcdi.attrib, None)

    def test_modifying_value_returns_Quantity(self):
        """
        Valid `__set__` yields `astropy.units.Quantity` on later `__get__` call
        """
        self.mcdi.attrib = 7.
        self.assertIsInstance(self.mcdi.attrib, units.Quantity)

    def test_default_unit(self):
        """
        Default unit is `astropy.units.dimensionless_unscaled`.
        """
        self.mcdi.attrib = 7.
        self.assertEqual(self.mcdi.attrib.unit, units.dimensionless_unscaled)

class InstantiationWithUnitArg(unittest.TestCase):
    """
    Instantiation with unit arguments from within an owner class.
    """
    def setUp(self):
        """
        A MockClassUserDefinedUnitAmp instance for the tests.
        """
        self.mcudua = MockClassUserDefinedUnitAmp()

    def tearDown(self):
        """
        Destroy the mock class used in the test.
        """
        del self.mcudua

    def test_unit(self):
        """
        Units are in amps.
        """
        self.assertEqual(self.mcudua.attrib.unit, "A")

    def test_set_with_number_roundtrip(self):
        """
        Set `attrib` with numeric type yields same numeric value on later `__get__` call.
        """
        number = 100.
        self.mcudua.attrib = number
        self.assertEqual(self.mcudua.attrib.value, number)

    def test_set_with_Quantity_stores_a_copy(self):
        """
        When setting the object with a Quantity, the object should store a new copy of the Quantity and not simply point to the original.

        This test tests two things:
        1. `__set__`ting with an `astropy.units.Quantity` works.
        2. The `PhysicalProperty` object stores a Quantity object that's different from the one passed to its `__set__` method.
        """
        qty = units.Quantity(100., "A")
        self.mcudua.attrib = qty

        self.assertIsNot(self.mcudua.attrib, qty)

    def test_set_with_Quantity_units_dont_change(self):
        """
        Set with Quantity with compatible units doesn't change `attrib` units.
        """
        qty = units.Quantity(100., "C/s")
        self.mcudua.attrib = qty

        self.assertEqual(self.mcudua.attrib.unit, "A")

class MultipleOwnerInstances(unittest.TestCase):
    """
    Multiple owner instances shouldn't interfere with one another.
    """
    def test_two_owners_individually_set_attribute(self):
        """
        Changing a `PhysicalProperty` attribute value of one owner instance shouldn't alter another owner instance.
        """
        val1 = 10.
        val2 = 3.

        mock1 = MockClassDefaultInstantiation()
        mock2 = MockClassDefaultInstantiation()

        mock1.attrib = val1
        mock2.attrib = val2

        self.assertEqual(mock1.attrib.value, val1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2012, Clément MATHIEU
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import errno
import os
import re
import shutil
import subprocess
import tempfile

from infoqscraper import client
from infoqscraper import ConversionError

from six.moves import http_cookiejar
from six.moves import urllib

class Converter(object):

    def __init__(self, presentation, output, **kwargs):
        self.presentation = presentation
        self.output = output

        self.ffmpeg = kwargs['ffmpeg']
        self.rtmpdump = kwargs['rtmpdump']
        self.swfrender = kwargs['swfrender']
        self.overwrite = kwargs['overwrite']
        self.type = kwargs['type']

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        shutil.rmtree(self.tmp_dir)

    @property
    def tmp_dir(self):
        if not hasattr(self, "_tmp_dir"):
            self._tmp_dir = tempfile.mkdtemp(prefix="infoq")

        return self._tmp_dir

    @property
    def _audio_path(self):
        return os.path.join(self.tmp_dir, "audio.ogg")

    @property
    def _video_path(self):
        return os.path.join(self.tmp_dir, 'video.avi')

    def create_presentation(self):
        """ Create the presentation.

        The audio track is mixed with the slides. The resulting file is saved as self.output

        DownloadError is raised if some resources cannot be fetched.
        ConversionError is raised if the final video cannot be created.
        """
        # Avoid wasting time and bandwidth if we known that conversion will fail.
        if not self.overwrite and os.path.exists(self.output):
            raise ConversionError("File %s already exist and --overwrite not specified" % self.output)

        video = self.download_video()
        raw_slides = self.download_slides()

        # ffmpeg does not support SWF
        png_slides = self._convert_slides(raw_slides)
        # Create one frame per second using the time code information
        frame_pattern = self._prepare_frames(png_slides)

        return self._assemble(video, frame_pattern)

    def download_video(self):
        """Downloads the video.

        If self.client.cache_enabled is True, then the disk cache is used.

        Returns:
            The path where the video has been saved.

        Raises:
            DownloadError: If the video cannot be downloaded.
        """
        rvideo_path = self.presentation.metadata['video_path']

        if self.presentation.client.cache:
            video_path = self.presentation.client.cache.get_path(rvideo_path)
            if not video_path:
                video_path = self.download_video_no_cache()
                self.presentation.client.cache.put_path(rvideo_path, video_path)
        else:
            video_path = self.download_video_no_cache()

        return video_path

    def download_video_no_cache(self):
        """Downloads the video.

        Returns:
            The path where the video has been saved.

        Raises:
            DownloadError: If the video cannot be downloaded.
        """
        video_url = self.presentation.metadata['video_url']
        video_path = self.presentation.metadata['video_path']

        # After a while, when downloading a long video (> 1h), the RTMP server seems to reset the connection (rtmpdump
        # returns exit code 2). The only way to get the full stream is to resume the download.
        resume_download = True
        while resume_download:
            try:
                cmd = [self.rtmpdump, '-q', '-e', '-r', video_url, '-y', video_path, "-o", self._video_path]
                subprocess.check_output(cmd, stderr=subprocess.STDOUT)
                resume_download = False
            except subprocess.CalledProcessError as e:
                if e.returncode != 2:
                    try:
                        os.unlink(self._video_path)
                    except OSError:
                        pass

                    raise client.DownloadError("Failed to download video at %s: rtmpdump exited with %s.\n\tOutput:\n%s"
                                           % (video_url, e.returncode, e.output))

        return self._video_path

    def download_slides(self):
        """ Download all SWF slides.

        The location of the slides files are returned.

        A DownloadError is raised if at least one of the slides cannot be download..
        """
        return self.presentation.client.download_all(self.presentation.metadata['slides'], self.tmp_dir)

    def _ffmpeg_legacy(self, audio, frame_pattern):
        # Try to be compatible as much as possible with old ffmpeg releases (>= 0.7)
        #   - Do not use new syntax options
        #   - Do not use libx264, not available on old Ubuntu/Debian
        #   - Do not use -threads auto, not available on 0.8.*
        #   - Old releases are very picky regarding arguments position
        #   - -n is not supported on 0.8
        #
        # 0.5 (Debian Squeeze & Ubuntu 10.4) is not supported because of
        # scaling issues with image2.
        cmd = [
            self.ffmpeg, "-v", "0",
            "-i", audio,
            "-f", "image2", "-r", "1", "-s", "hd720", "-i", frame_pattern,
            "-map", "1:0", "-acodec", "libmp3lame", "-ab", "128k",
            "-map", "0:1", "-vcodec", "mpeg4", "-vb", "2M", "-y", self.output
        ]

        if not self.overwrite and os.path.exists(self.output):
            # Handle already existing file manually since nor -n nor -nostdin is available on 0.8
            raise Exception("File %s already exist and --overwrite not specified" % self.output)

        return cmd

    def _ffmpeg_h264(self, audio, frame_pattern):
        return [
            self.ffmpeg, "-v", "error",
            "-i", audio,
            "-r", "1", "-i", frame_pattern,
            "-c:a", "copy",
            "-c:v", "libx264", "-profile:v", "baseline", "-preset", "ultrafast", "-level", "3.0",
            "-crf", "28", "-pix_fmt", "yuv420p",
            "-s", "1280x720",
            "-y" if self.overwrite else "-n",
            self.output
        ]

    def _ffmpeg_h264_overlay(self, video, frame_pattern):
        cmd = [self.ffmpeg, "-i", video]
        video_details = ""
        try:
            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            video_details = e.output

        fps_match = re.search("\S+(?=\s+tbr)", video_details)
        fps = float(fps_match.group(0))

        timings = self.presentation.metadata['demo_timings'][:]

        if len(timings) == 0 or timings[0] != 0:
            slides_first = True
            timings.insert(0, 0)
        else:
            slides_first = False

        timings.append(float('inf'))

        inputs = []
        filter_complex = []
        concat = []

        for i, right_range in enumerate(timings[1:]):
            left_range = timings[i]
            duration = right_range - left_range

            if left_range > 0:
                inputs += ["-ss", str(left_range)]
            if right_range != float('inf'):
                inputs += ["-t", str(duration)]
            inputs += ["-i", video]

            if (i % 2 == 0) == slides_first:
                inputs += [
                    "-f", "image2", "-r", "1", "-s", "hd720", "-start_number", str(left_range), "-i", frame_pattern
                ]
                stream_id = i / 2 * 3
                if not slides_first:
                    stream_id += 1

                filter_complex += [
                    "[{0:d}:v] setpts=PTS-STARTPTS, scale=w=320:h=-1 [sp-{1:d}];".format(stream_id, i),
                    "[{0:d}:v] setpts=PTS-STARTPTS, scale=w=1280-320:h=-1[sl-{1:d}];".format(stream_id + 1, i),
                    "color=size=1280x720:c=Black [b-{0:d}];".format(i),
                    "[b-{0:d}][sl-{0:d}] overlay=shortest=1:x=0:y=0 [bsl-{0:d}];".format(i),
                    "[bsl-{0:d}][sp-{0:d}] overlay=shortest=1:x=main_w-320:y=main_h-overlay_h [c-{0:d}];".format(i)
                ]
            else:
                stream_id = i / 2 * 3
                if slides_first:
                    stream_id += 2
                filter_complex += [
                    "[{0:d}:v] scale='if(gt(a,16/9),1280,-1)':'if(gt(a,16/9),-1,720)' [c-{1:d}];".format(stream_id, i)
                ]

            concat += ["[c-{0:d}] [{1:d}:a:0]".format(i, stream_id)]

        concat += ["concat=n={0:d}:v=1:a=1 [v] [a]".format(len(timings) - 1)]

        filter_script_path = os.path.join(self.tmp_dir, "filter")
        with open(filter_script_path, 'w') as filter_script_file:
            filter_script_file.write("\n".join(filter_complex))
            filter_script_file.write("\n")
            filter_script_file.write(" ".join(concat))

        cmd = [self.ffmpeg, "-v", "error"]

        cmd += inputs

        cmd += [
            "-filter_complex_script", filter_script_path,
            "-map", "[v]", "-map", "[a]",
            "-r", str(fps),
            "-acodec", "libmp3lame", "-ab", "92k",
            "-vcodec", "libx264", "-profile:v", "baseline", "-preset", "fast", "-level", "3.0", "-crf", "28",
            "-y" if self.overwrite else "-n",
            self.output
        ]

        return cmd

    def _assemble(self, audio, frame_pattern):
        if self.type == "legacy":
            cmd = self._ffmpeg_legacy(audio, frame_pattern)
        elif self.type == "h264":
            cmd = self._ffmpeg_h264(audio, frame_pattern)
        elif self.type == "h264_overlay":
            cmd = self._ffmpeg_h264_overlay(audio, frame_pattern)
        else:
            raise Exception("Unknown output type %s" % self.type)

        self._run_command(cmd)

    def _run_command(self, cmd):
        try:
            return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            msg = "Failed to create final movie as %s.\n" \
                  "\tCommand: %s\n" \
                  "\tExit code: %s\n" \
                  "\tOutput:\n%s" % (self.output, " ".join(cmd), e.returncode, e.output)

            if self.type != "legacy":
                msg += "\n Please note that %s output format requires a recent version of ffmpeg and libx264." \
                       " Perhaps you should check your setup." \
                       % self.type

            raise ConversionError(msg)

    def _convert_slides(self, slides):

        def convert(slide):
            if slide.endswith("swf"):
                png_slide = slide.replace(".swf", ".png")
                swf2png(slide, png_slide, swfrender_path=self.swfrender)
                return png_slide
            elif slide.endswith("jpg"):
                return slide
            else:
                raise Exception("Unsupported slide type: %s" % slide)

        return [convert(s) for s in slides]

    def _prepare_frames(self, slides):
        timecodes = self.presentation.metadata['timecodes']
        ext = os.path.splitext(slides[0])[1]

        frame = 0
        for slide_index, src in enumerate(slides):
            for remaining in range(timecodes[slide_index], timecodes[slide_index+1]):
                dst = os.path.join(self.tmp_dir, "frame-{0:04d}." + ext).format(frame)
                try:
                    os.link(src, dst)
                except OSError as e:
                    if e.errno == errno.EMLINK:
                        # Create a new reference file when the upper limit is reached
                        # (previous to Linux 3.7, btrfs had a very low limit)
                        shutil.copyfile(src, dst)
                        src = dst
                    else:
                        raise e

                frame += 1

        return os.path.join(self.tmp_dir, "frame-%04d." + ext)

def swf2png(swf_path, png_path, swfrender_path="swfrender"):
    """Convert SWF slides into a PNG image

    Raises:
        OSError is raised if swfrender is not available.
        ConversionError is raised if image cannot be created.
    """
    # Currently rely on swftools
    #
    # Would be great to have a native python dependency to convert swf into png or jpg.
    # However it seems that pyswf  isn't flawless. Some graphical elements (like the text!) are lost during
    # the export.
    try:
        cmd = [swfrender_path, swf_path, '-o', png_path]
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        raise ConversionError("Failed to convert SWF file %s.\n"
                              "\tCommand: %s\n"
                              "\tExit status: %s.\n"
                              "\tOutput:\n%s"
                              % (swf_path, " ".join(cmd), e.returncode, e.output))

<commit_msg>Restore Python 3 compatibility after h264_overlay changes<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2012, Clément MATHIEU
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
#  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import errno
import os
import re
import shutil
import six
import subprocess
import tempfile

from infoqscraper import client
from infoqscraper import ConversionError

class Converter(object):

    def __init__(self, presentation, output, **kwargs):
        self.presentation = presentation
        self.output = output

        self.ffmpeg = kwargs['ffmpeg']
        self.rtmpdump = kwargs['rtmpdump']
        self.swfrender = kwargs['swfrender']
        self.overwrite = kwargs['overwrite']
        self.type = kwargs['type']

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        shutil.rmtree(self.tmp_dir)

    @property
    def tmp_dir(self):
        if not hasattr(self, "_tmp_dir"):
            self._tmp_dir = tempfile.mkdtemp(prefix="infoq")

        return self._tmp_dir

    @property
    def _audio_path(self):
        return os.path.join(self.tmp_dir, "audio.ogg")

    @property
    def _video_path(self):
        return os.path.join(self.tmp_dir, 'video.avi')

    def create_presentation(self):
        """ Create the presentation.

        The audio track is mixed with the slides. The resulting file is saved as self.output

        DownloadError is raised if some resources cannot be fetched.
        ConversionError is raised if the final video cannot be created.
        """
        # Avoid wasting time and bandwidth if we known that conversion will fail.
        if not self.overwrite and os.path.exists(self.output):
            raise ConversionError("File %s already exist and --overwrite not specified" % self.output)

        video = self.download_video()
        raw_slides = self.download_slides()

        # ffmpeg does not support SWF
        png_slides = self._convert_slides(raw_slides)
        # Create one frame per second using the time code information
        frame_pattern = self._prepare_frames(png_slides)

        return self._assemble(video, frame_pattern)

    def download_video(self):
        """Downloads the video.

        If self.client.cache_enabled is True, then the disk cache is used.

        Returns:
            The path where the video has been saved.

        Raises:
            DownloadError: If the video cannot be downloaded.
        """
        rvideo_path = self.presentation.metadata['video_path']

        if self.presentation.client.cache:
            video_path = self.presentation.client.cache.get_path(rvideo_path)
            if not video_path:
                video_path = self.download_video_no_cache()
                self.presentation.client.cache.put_path(rvideo_path, video_path)
        else:
            video_path = self.download_video_no_cache()

        return video_path

    def download_video_no_cache(self):
        """Downloads the video.

        Returns:
            The path where the video has been saved.

        Raises:
            DownloadError: If the video cannot be downloaded.
        """
        video_url = self.presentation.metadata['video_url']
        video_path = self.presentation.metadata['video_path']

        # After a while, when downloading a long video (> 1h), the RTMP server seems to reset the connection (rtmpdump
        # returns exit code 2). The only way to get the full stream is to resume the download.
        resume_download = True
        while resume_download:
            try:
                cmd = [self.rtmpdump, '-q', '-e', '-r', video_url, '-y', video_path, "-o", self._video_path]
                subprocess.check_output(cmd, stderr=subprocess.STDOUT)
                resume_download = False
            except subprocess.CalledProcessError as e:
                if e.returncode != 2:
                    try:
                        os.unlink(self._video_path)
                    except OSError:
                        pass

                    raise client.DownloadError("Failed to download video at %s: rtmpdump exited with %s.\n\tOutput:\n%s"
                                           % (video_url, e.returncode, e.output))

        return self._video_path

    def download_slides(self):
        """ Download all SWF slides.

        The location of the slides files are returned.

        A DownloadError is raised if at least one of the slides cannot be download..
        """
        return self.presentation.client.download_all(self.presentation.metadata['slides'], self.tmp_dir)

    def _ffmpeg_legacy(self, audio, frame_pattern):
        # Try to be compatible as much as possible with old ffmpeg releases (>= 0.7)
        #   - Do not use new syntax options
        #   - Do not use libx264, not available on old Ubuntu/Debian
        #   - Do not use -threads auto, not available on 0.8.*
        #   - Old releases are very picky regarding arguments position
        #   - -n is not supported on 0.8
        #
        # 0.5 (Debian Squeeze & Ubuntu 10.4) is not supported because of
        # scaling issues with image2.
        cmd = [
            self.ffmpeg, "-v", "0",
            "-i", audio,
            "-f", "image2", "-r", "1", "-s", "hd720", "-i", frame_pattern,
            "-map", "1:0", "-acodec", "libmp3lame", "-ab", "128k",
            "-map", "0:1", "-vcodec", "mpeg4", "-vb", "2M", "-y", self.output
        ]

        if not self.overwrite and os.path.exists(self.output):
            # Handle already existing file manually since nor -n nor -nostdin is available on 0.8
            raise Exception("File %s already exist and --overwrite not specified" % self.output)

        return cmd

    def _ffmpeg_h264(self, audio, frame_pattern):
        return [
            self.ffmpeg, "-v", "error",
            "-i", audio,
            "-r", "1", "-i", frame_pattern,
            "-c:a", "copy",
            "-c:v", "libx264", "-profile:v", "baseline", "-preset", "ultrafast", "-level", "3.0",
            "-crf", "28", "-pix_fmt", "yuv420p",
            "-s", "1280x720",
            "-y" if self.overwrite else "-n",
            self.output
        ]

    def _ffmpeg_h264_overlay(self, video, frame_pattern):
        cmd = [self.ffmpeg, "-i", video]
        video_details = ""
        try:
            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            video_details = e.output

        fps_match = re.search(six.b('\S+(?=\s+tbr)'), video_details)
        fps = float(fps_match.group(0))
        timings = self.presentation.metadata['demo_timings'][:]

        if len(timings) == 0 or timings[0] != 0:
            slides_first = True
            timings.insert(0, 0)
        else:
            slides_first = False

        timings.append(float('inf'))

        inputs = []
        filter_complex = []
        concat = []

        for i, right_range in enumerate(timings[1:]):
            left_range = timings[i]
            duration = right_range - left_range

            if left_range > 0:
                inputs += ["-ss", str(left_range)]
            if right_range != float('inf'):
                inputs += ["-t", str(duration)]
            inputs += ["-i", video]

            if (i % 2 == 0) == slides_first:
                inputs += [
                    "-f", "image2", "-r", "1", "-s", "hd720", "-start_number", str(left_range), "-i", frame_pattern
                ]
                stream_id = i // 2 * 3
                if not slides_first:
                    stream_id += 1

                filter_complex += [
                    "[{0:d}:v] setpts=PTS-STARTPTS, scale=w=320:h=-1 [sp-{1:d}];".format(stream_id, i),
                    "[{0:d}:v] setpts=PTS-STARTPTS, scale=w=1280-320:h=-1[sl-{1:d}];".format(stream_id + 1, i),
                    "color=size=1280x720:c=Black [b-{0:d}];".format(i),
                    "[b-{0:d}][sl-{0:d}] overlay=shortest=1:x=0:y=0 [bsl-{0:d}];".format(i),
                    "[bsl-{0:d}][sp-{0:d}] overlay=shortest=1:x=main_w-320:y=main_h-overlay_h [c-{0:d}];".format(i)
                ]
            else:
                stream_id = i / 2 * 3
                if slides_first:
                    stream_id += 2
                filter_complex += [
                    "[{0:d}:v] scale='if(gt(a,16/9),1280,-1)':'if(gt(a,16/9),-1,720)' [c-{1:d}];".format(stream_id, i)
                ]

            concat += ["[c-{0:d}] [{1:d}:a:0]".format(i, stream_id)]

        concat += ["concat=n={0:d}:v=1:a=1 [v] [a]".format(len(timings) - 1)]

        filter_script_path = os.path.join(self.tmp_dir, "filter")
        with open(filter_script_path, 'w') as filter_script_file:
            filter_script_file.write("\n".join(filter_complex))
            filter_script_file.write("\n")
            filter_script_file.write(" ".join(concat))

        cmd = [self.ffmpeg, "-v", "error"]
        cmd += inputs
        cmd += [
            "-filter_complex_script", filter_script_path,
            "-map", "[v]", "-map", "[a]",
            "-r", str(fps),
            "-acodec", "libmp3lame", "-ab", "92k",
            "-vcodec", "libx264", "-profile:v", "baseline", "-preset", "fast", "-level", "3.0", "-crf", "28",
            "-y" if self.overwrite else "-n",
            self.output
        ]

        return cmd

    def _assemble(self, audio, frame_pattern):
        if self.type == "legacy":
            cmd = self._ffmpeg_legacy(audio, frame_pattern)
        elif self.type == "h264":
            cmd = self._ffmpeg_h264(audio, frame_pattern)
        elif self.type == "h264_overlay":
            cmd = self._ffmpeg_h264_overlay(audio, frame_pattern)
        else:
            raise Exception("Unknown output type %s" % self.type)

        self._run_command(cmd)

    def _run_command(self, cmd):
        try:
            return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            msg = "Failed to create final movie as %s.\n" \
                  "\tCommand: %s\n" \
                  "\tExit code: %s\n" \
                  "\tOutput:\n%s" % (self.output, " ".join(cmd), e.returncode, e.output)

            if self.type != "legacy":
                msg += "\n Please note that %s output format requires a recent version of ffmpeg and libx264." \
                       " Perhaps you should check your setup." \
                       % self.type

            raise ConversionError(msg)

    def _convert_slides(self, slides):

        def convert(slide):
            if slide.endswith("swf"):
                png_slide = slide.replace(".swf", ".png")
                swf2png(slide, png_slide, swfrender_path=self.swfrender)
                return png_slide
            elif slide.endswith("jpg"):
                return slide
            else:
                raise Exception("Unsupported slide type: %s" % slide)

        return [convert(s) for s in slides]

    def _prepare_frames(self, slides):
        timecodes = self.presentation.metadata['timecodes']
        ext = os.path.splitext(slides[0])[1]

        frame = 0
        for slide_index, src in enumerate(slides):
            for remaining in range(timecodes[slide_index], timecodes[slide_index+1]):
                dst = os.path.join(self.tmp_dir, "frame-{0:04d}." + ext).format(frame)
                try:
                    os.link(src, dst)
                except OSError as e:
                    if e.errno == errno.EMLINK:
                        # Create a new reference file when the upper limit is reached
                        # (previous to Linux 3.7, btrfs had a very low limit)
                        shutil.copyfile(src, dst)
                        src = dst
                    else:
                        raise e

                frame += 1

        return os.path.join(self.tmp_dir, "frame-%04d." + ext)

def swf2png(swf_path, png_path, swfrender_path="swfrender"):
    """Convert SWF slides into a PNG image

    Raises:
        OSError is raised if swfrender is not available.
        ConversionError is raised if image cannot be created.
    """
    # Currently rely on swftools
    #
    # Would be great to have a native python dependency to convert swf into png or jpg.
    # However it seems that pyswf  isn't flawless. Some graphical elements (like the text!) are lost during
    # the export.
    try:
        cmd = [swfrender_path, swf_path, '-o', png_path]
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        raise ConversionError("Failed to convert SWF file %s.\n"
                              "\tCommand: %s\n"
                              "\tExit status: %s.\n"
                              "\tOutput:\n%s"
                              % (swf_path, " ".join(cmd), e.returncode, e.output))

<|endoftext|>


<commit_before>'''
Created on Sep 22, 2010

@author: sjoshi42
'''

import tables as pt
import time

h5f = pt.openFile("C:\\generated data files\\timestamp files\\timestamps.h5" , mode = "a")
fileIter=h5f.getNode('/timestamps','timestamps')

for row in fileIter:
    print str(time.strftime("%Y%m%d", time.gmtime(row['timestamp'])))

h5f.close()

<commit_msg>defunct.<commit_after><|endoftext|>