<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2007 Edgewall Software
# Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.

from ConfigParser import ConfigParser
import os

from trac.core import ExtensionPoint, TracError
from trac.util.compat import set, sorted
from trac.util.text import to_unicode, CRLF

__all__ = ['Configuration', 'Option', 'BoolOption', 'IntOption', 'ListOption',
           'PathOption', 'ExtensionOption', 'OrderedExtensionsOption',
           'ConfigurationError']

_TRUE_VALUES = ('yes', 'true', 'enabled', 'on', 'aye', '1', 1, True)

class ConfigurationError(TracError):
    """Exception raised when a value in the configuration file is not valid."""

class Configuration(object):
    """Thin layer over `ConfigParser` from the Python standard library.

    In addition to providing some convenience methods, the class remembers
    the last modification time of the configuration file, and reparses it
    when the file has changed.
    """
    def __init__(self, filename):
        self.filename = filename
        self.parser = ConfigParser()
        self.parent = None
        self._lastmtime = 0
        self._sections = {}
        self.parse_if_needed()

    def __contains__(self, name):
        """Return whether the configuration contains a section of the given
        name.
        """
        return name in self.sections()

    def __getitem__(self, name):
        """Return the configuration section with the specified name."""
        if name not in self._sections:
            self._sections[name] = Section(self, name)
        return self._sections[name]

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self.filename)

    def get(self, section, name, default=''):
        """Return the value of the specified option.

        Valid default input is a string. Returns a string.
        """
        return self[section].get(name, default)

    def getbool(self, section, name, default=''):
        """Return the specified option as boolean value.

        If the value of the option is one of "yes", "true", "enabled", "on",
        or "1", this method wll return `True`, otherwise `False`.

        Valid default input is a string or a bool. Returns a bool.

        (since Trac 0.9.3, "enabled" added in 0.11)
        """
        return self[section].getbool(name, default)

    def getint(self, section, name, default=''):
        """Return the value of the specified option as integer.

        If the specified option can not be converted to an integer, a
        `ConfigurationError` exception is raised.

        Valid default input is a string or an int. Returns an int.

        (since Trac 0.10)
        """
        return self[section].getint(name, default)

    def getlist(self, section, name, default='', sep=',', keep_empty=False):
        """Return a list of values that have been specified as a single
        comma-separated option.

        A different separator can be specified using the `sep` parameter. If
        the `keep_empty` parameter is set to `True`, empty elements are
        included in the list.

        Valid default input is a string or a list. Returns a string.

        (since Trac 0.10)
        """
        return self[section].getlist(name, default, sep, keep_empty)

    def set(self, section, name, value):
        """Change a configuration value.

        These changes are not persistent unless saved with `save()`.
        """
        self[section].set(name, value)

    def defaults(self):
        """Returns a dictionary of the default configuration values.

        (since Trac 0.10)
        """
        defaults = {}
        for (section, name), option in Option.registry.items():
            defaults.setdefault(section, {})[name] = option.default
        return defaults

    def options(self, section):
        """Return a list of `(name, value)` tuples for every option in the
        specified section.

        This includes options that have default values that haven't been
        overridden.
        """
        return self[section].options()

    def remove(self, section, name):
        """Remove the specified option."""
        if self.parser.has_section(section):
            self.parser.remove_option(section, name)

    def sections(self):
        """Return a list of section names."""
        sections = set(self.parser.sections())
        parent = self.parent
        while parent:
            sections |= set(parent.parser.sections())
            parent = parent.parent
        return sorted(sections)

    def has_option(self, section, option):
        """Returns True if option exists in section in either project or
        parent trac.ini, or available through the Option registry.

        (since Trac 0.11)
        """
        # Check project trac.ini
        for file_option, val in self.options(section):
            if file_option == option:
                return True
        # Check parent trac.ini
        if self.parent:
            for parent_option, val in self.parent.options(section):
                if parent_option == option:
                    return True
        # Check the registry
        if (section, option) in Option.registry:
            return True
        # Not found
        return False

    def save(self):
        """Write the configuration options to the primary file."""
        if not self.filename:
            return

        # Only save options that differ from the defaults
        sections = []
        for section in self.sections():
            options = []
            for option in self[section]:
                default = None
                if self.parent:
                    default = self.parent.get(section, option)
                current = self.parser.has_option(section, option) and \
                          to_unicode(self.parser.get(section, option))
                if current is not False and current != default:
                    options.append((option, current))
            if options:
                sections.append((section, sorted(options)))

        fileobj = open(self.filename, 'w')
        try:
            fileobj.write('# -*- coding: utf-8 -*-\n\n')
            for section, options in sections:
                fileobj.write('[%s]\n' % section)
                for key, val in options:
                    if key in self[section].overridden:
                        fileobj.write('# %s = <inherited>\n' % key)
                    else:
                        val = val.replace(CRLF, '\n').replace('\n', '\n ')
                        fileobj.write('%s = %s\n' % (key, val.encode('utf-8')))
                fileobj.write('\n')
        finally:
            fileobj.close()

    def parse_if_needed(self):
        if not self.filename or not os.path.isfile(self.filename):
            return False

        changed = False
        modtime = os.path.getmtime(self.filename)
        if modtime > self._lastmtime:
            self.parser._sections = {}
            self.parser.read(self.filename)
            self._lastmtime = modtime
            changed = True

        if self.parser.has_option('inherit', 'file'):
            filename = self.parser.get('inherit', 'file')
            if not os.path.isabs(filename):
                filename = os.path.join(os.path.dirname(self.filename),
                                        filename)
            if not self.parent or self.parent.filename != filename:
                self.parent = Configuration(filename)
                changed = True
            else:
                changed |= self.parent.parse_if_needed()
        elif self.parent:
            changed = True
            self.parent = None

        return changed

    def touch(self):
        os.utime(self.filename, None)

class Section(object):
    """Proxy for a specific configuration section.

    Objects of this class should not be instantiated directly.
    """
    __slots__ = ['config', 'name', 'overridden']

    def __init__(self, config, name):
        self.config = config
        self.name = name
        self.overridden = {}

    def __contains__(self, name):
        if self.config.parser.has_option(self.name, name):
            return True
        if self.config.parent:
            return name in self.config.parent[self.name]
        return False

    def __iter__(self):
        options = set()
        if self.config.parser.has_section(self.name):
            for option in self.config.parser.options(self.name):
                options.add(option.lower())
                yield option
        if self.config.parent:
            for option in self.config.parent[self.name]:
                if option.lower() not in options:
                    yield option

    def __repr__(self):
        return '<Section [%s]>' % (self.name)

    def get(self, name, default=''):
        """Return the value of the specified option.

        Valid default input is a string. Returns a string.
        """
        if self.config.parser.has_option(self.name, name):
            value = self.config.parser.get(self.name, name)
        elif self.config.parent:
            value = self.config.parent[self.name].get(name, default)
        else:
            option = Option.registry.get((self.name, name))
            if option:
                value = option.default or default
            else:
                value = default
        if not value:
            return u''
        elif isinstance(value, basestring):
            return to_unicode(value)
        else:
            return value

    def getbool(self, name, default=''):
        """Return the value of the specified option as boolean.

        This method returns `True` if the option value is one of "yes", "true",
        "enabled", "on", or "1", ignoring case. Otherwise `False` is returned.

        Valid default input is a string or a bool. Returns a bool.
        """
        value = self.get(name, default)
        if isinstance(value, basestring):
            value = value.lower() in _TRUE_VALUES
        return bool(value)

    def getint(self, name, default=''):
        """Return the value of the specified option as integer.

        If the specified option can not be converted to an integer, a
        `ConfigurationError` exception is raised.

        Valid default input is a string or an int. Returns an int.
        """
        value = self.get(name, default)
        if not value:
            return 0
        try:
            return int(value)
        except ValueError:
            raise ConfigurationError('expected integer, got %s' % repr(value))

    def getlist(self, name, default='', sep=',', keep_empty=True):
        """Return a list of values that have been specified as a single
        comma-separated option.

        A different separator can be specified using the `sep` parameter. If
        the `keep_empty` parameter is set to `False`, empty elements are omitted
        from the list.

        Valid default input is a string or a list. Returns a list.
        """
        value = self.get(name, default)
        if not value:
            return []
        if isinstance(value, basestring):
            items = [item.strip() for item in value.split(sep)]
        else:
            items = list(value)
        if not keep_empty:
            items = filter(None, items)
        return items

    def getpath(self, name, default=''):
        """Return the value of the specified option as a path name, relative to
        the location of the configuration file the option is defined in.

        Valid default input is a string. Returns a string with normalised path.
        """
        if self.config.parser.has_option(self.name, name):
            path = self.config.parser.get(self.name, name)
            if not path:
                return default
            if not os.path.isabs(path):
                path = os.path.join(os.path.dirname(self.config.filename),
                                    path)
            return os.path.normcase(os.path.realpath(path))
        elif self.config.parent:
            return self.config.parent[self.name].getpath(name, default)
        else:
            return default

    def options(self):
        """Return `(name, value)` tuples for every option in the section."""
        for name in self:
            yield name, self.get(name)

    def set(self, name, value):
        """Change a configuration value.

        These changes are not persistent unless saved with `save()`.
        """
        if not self.config.parser.has_section(self.name):
            self.config.parser.add_section(self.name)
        if value is None:
            self.overridden[name] = True
            value = ''
        else:
            value = to_unicode(value).encode('utf-8')
        return self.config.parser.set(self.name, name, value)

class Option(object):
    """Descriptor for configuration options on `Configurable` subclasses."""

    registry = {}
    accessor = Section.get

    def __init__(self, section, name, default=None, doc=''):
        """Create the extension point.

        @param section: the name of the configuration section this option
            belongs to
        @param name: the name of the option
        @param default: the default value for the option
        @param doc: documentation of the option
        """
        self.section = section
        self.name = name
        self.default = default
        self.registry[(self.section, self.name)] = self
        self.__doc__ = doc

    def __get__(self, instance, owner):
        if instance is None:
            return self
        config = getattr(instance, 'config', None)
        if config and isinstance(config, Configuration):
            section = config[self.section]
            value = self.accessor(section, self.name, self.default)
            return value
        return None

    def __set__(self, instance, value):
        raise AttributeError, 'can\'t set attribute'

    def __repr__(self):
        return '<%s [%s] "%s">' % (self.__class__.__name__, self.section,
                                   self.name)

class BoolOption(Option):
    """Descriptor for boolean configuration options."""
    accessor = Section.getbool

class IntOption(Option):
    """Descriptor for integer configuration options."""
    accessor = Section.getint

class ListOption(Option):
    """Descriptor for configuration options that contain multiple values
    separated by a specific character."""

    def __init__(self, section, name, default=None, sep=',', keep_empty=False,
                 doc=''):
        Option.__init__(self, section, name, default, doc)
        self.sep = sep
        self.keep_empty = keep_empty

    def accessor(self, section, name, default):
        return section.getlist(name, default, self.sep, self.keep_empty)

class PathOption(Option):
    """Descriptor for file system path configuration options."""
    accessor = Section.getpath

class ExtensionOption(Option):

    def __init__(self, section, name, interface, default=None, doc=''):
        Option.__init__(self, section, name, default, doc)
        self.xtnpt = ExtensionPoint(interface)

    def __get__(self, instance, owner):
        if instance is None:
            return self
        value = Option.__get__(self, instance, owner)
        for impl in self.xtnpt.extensions(instance):
            if impl.__class__.__name__ == value:
                return impl
        raise AttributeError('Cannot find an implementation of the "%s" '
                             'interface named "%s".  Please update the option '
                             '%s.%s in trac.ini.'
                             % (self.xtnpt.interface.__name__, value,
                                self.section, self.name))

class OrderedExtensionsOption(ListOption):
    """A comma separated, ordered, list of components implementing `interface`.
    Can be empty.

    If `include_missing` is true (the default) all components implementing the
    interface are returned, with those specified by the option ordered first."""

    def __init__(self, section, name, interface, default=None,
                 include_missing=True, doc=''):
        ListOption.__init__(self, section, name, default, doc=doc)
        self.xtnpt = ExtensionPoint(interface)
        self.include_missing = include_missing

    def __get__(self, instance, owner):
        if instance is None:
            return self
        order = ListOption.__get__(self, instance, owner)
        components = []
        for impl in self.xtnpt.extensions(instance):
            if self.include_missing or impl.__class__.__name__ in order:
                components.append(impl)

        def compare(x, y):
            x, y = x.__class__.__name__, y.__class__.__name__
            if x not in order:
                return int(y in order)
            if y not in order:
                return -int(x in order)
            return cmp(order.index(x), order.index(y))
        components.sort(compare)
        return components
<commit_msg>Followup to r6837, don't try to touch a file that doesn't exist, or we don't have rights to.  There was a test failure introduced in the wiki formatter tests because its EnvironmentStub doesn't have a filename.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2007 Edgewall Software
# Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.

from ConfigParser import ConfigParser
import os

from trac.core import ExtensionPoint, TracError
from trac.util.compat import set, sorted
from trac.util.text import to_unicode, CRLF

__all__ = ['Configuration', 'Option', 'BoolOption', 'IntOption', 'ListOption',
           'PathOption', 'ExtensionOption', 'OrderedExtensionsOption',
           'ConfigurationError']

_TRUE_VALUES = ('yes', 'true', 'enabled', 'on', 'aye', '1', 1, True)

class ConfigurationError(TracError):
    """Exception raised when a value in the configuration file is not valid."""

class Configuration(object):
    """Thin layer over `ConfigParser` from the Python standard library.

    In addition to providing some convenience methods, the class remembers
    the last modification time of the configuration file, and reparses it
    when the file has changed.
    """
    def __init__(self, filename):
        self.filename = filename
        self.parser = ConfigParser()
        self.parent = None
        self._lastmtime = 0
        self._sections = {}
        self.parse_if_needed()

    def __contains__(self, name):
        """Return whether the configuration contains a section of the given
        name.
        """
        return name in self.sections()

    def __getitem__(self, name):
        """Return the configuration section with the specified name."""
        if name not in self._sections:
            self._sections[name] = Section(self, name)
        return self._sections[name]

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self.filename)

    def get(self, section, name, default=''):
        """Return the value of the specified option.

        Valid default input is a string. Returns a string.
        """
        return self[section].get(name, default)

    def getbool(self, section, name, default=''):
        """Return the specified option as boolean value.

        If the value of the option is one of "yes", "true", "enabled", "on",
        or "1", this method wll return `True`, otherwise `False`.

        Valid default input is a string or a bool. Returns a bool.

        (since Trac 0.9.3, "enabled" added in 0.11)
        """
        return self[section].getbool(name, default)

    def getint(self, section, name, default=''):
        """Return the value of the specified option as integer.

        If the specified option can not be converted to an integer, a
        `ConfigurationError` exception is raised.

        Valid default input is a string or an int. Returns an int.

        (since Trac 0.10)
        """
        return self[section].getint(name, default)

    def getlist(self, section, name, default='', sep=',', keep_empty=False):
        """Return a list of values that have been specified as a single
        comma-separated option.

        A different separator can be specified using the `sep` parameter. If
        the `keep_empty` parameter is set to `True`, empty elements are
        included in the list.

        Valid default input is a string or a list. Returns a string.

        (since Trac 0.10)
        """
        return self[section].getlist(name, default, sep, keep_empty)

    def set(self, section, name, value):
        """Change a configuration value.

        These changes are not persistent unless saved with `save()`.
        """
        self[section].set(name, value)

    def defaults(self):
        """Returns a dictionary of the default configuration values.

        (since Trac 0.10)
        """
        defaults = {}
        for (section, name), option in Option.registry.items():
            defaults.setdefault(section, {})[name] = option.default
        return defaults

    def options(self, section):
        """Return a list of `(name, value)` tuples for every option in the
        specified section.

        This includes options that have default values that haven't been
        overridden.
        """
        return self[section].options()

    def remove(self, section, name):
        """Remove the specified option."""
        if self.parser.has_section(section):
            self.parser.remove_option(section, name)

    def sections(self):
        """Return a list of section names."""
        sections = set(self.parser.sections())
        parent = self.parent
        while parent:
            sections |= set(parent.parser.sections())
            parent = parent.parent
        return sorted(sections)

    def has_option(self, section, option):
        """Returns True if option exists in section in either project or
        parent trac.ini, or available through the Option registry.

        (since Trac 0.11)
        """
        # Check project trac.ini
        for file_option, val in self.options(section):
            if file_option == option:
                return True
        # Check parent trac.ini
        if self.parent:
            for parent_option, val in self.parent.options(section):
                if parent_option == option:
                    return True
        # Check the registry
        if (section, option) in Option.registry:
            return True
        # Not found
        return False

    def save(self):
        """Write the configuration options to the primary file."""
        if not self.filename:
            return

        # Only save options that differ from the defaults
        sections = []
        for section in self.sections():
            options = []
            for option in self[section]:
                default = None
                if self.parent:
                    default = self.parent.get(section, option)
                current = self.parser.has_option(section, option) and \
                          to_unicode(self.parser.get(section, option))
                if current is not False and current != default:
                    options.append((option, current))
            if options:
                sections.append((section, sorted(options)))

        fileobj = open(self.filename, 'w')
        try:
            fileobj.write('# -*- coding: utf-8 -*-\n\n')
            for section, options in sections:
                fileobj.write('[%s]\n' % section)
                for key, val in options:
                    if key in self[section].overridden:
                        fileobj.write('# %s = <inherited>\n' % key)
                    else:
                        val = val.replace(CRLF, '\n').replace('\n', '\n ')
                        fileobj.write('%s = %s\n' % (key, val.encode('utf-8')))
                fileobj.write('\n')
        finally:
            fileobj.close()

    def parse_if_needed(self):
        if not self.filename or not os.path.isfile(self.filename):
            return False

        changed = False
        modtime = os.path.getmtime(self.filename)
        if modtime > self._lastmtime:
            self.parser._sections = {}
            self.parser.read(self.filename)
            self._lastmtime = modtime
            changed = True

        if self.parser.has_option('inherit', 'file'):
            filename = self.parser.get('inherit', 'file')
            if not os.path.isabs(filename):
                filename = os.path.join(os.path.dirname(self.filename),
                                        filename)
            if not self.parent or self.parent.filename != filename:
                self.parent = Configuration(filename)
                changed = True
            else:
                changed |= self.parent.parse_if_needed()
        elif self.parent:
            changed = True
            self.parent = None

        return changed

    def touch(self):
        if self.filename and os.path.isfile(self.filename) \
           and os.access(self.filename, W_OK):
            os.utime(self.filename, None)

class Section(object):
    """Proxy for a specific configuration section.

    Objects of this class should not be instantiated directly.
    """
    __slots__ = ['config', 'name', 'overridden']

    def __init__(self, config, name):
        self.config = config
        self.name = name
        self.overridden = {}

    def __contains__(self, name):
        if self.config.parser.has_option(self.name, name):
            return True
        if self.config.parent:
            return name in self.config.parent[self.name]
        return False

    def __iter__(self):
        options = set()
        if self.config.parser.has_section(self.name):
            for option in self.config.parser.options(self.name):
                options.add(option.lower())
                yield option
        if self.config.parent:
            for option in self.config.parent[self.name]:
                if option.lower() not in options:
                    yield option

    def __repr__(self):
        return '<Section [%s]>' % (self.name)

    def get(self, name, default=''):
        """Return the value of the specified option.

        Valid default input is a string. Returns a string.
        """
        if self.config.parser.has_option(self.name, name):
            value = self.config.parser.get(self.name, name)
        elif self.config.parent:
            value = self.config.parent[self.name].get(name, default)
        else:
            option = Option.registry.get((self.name, name))
            if option:
                value = option.default or default
            else:
                value = default
        if not value:
            return u''
        elif isinstance(value, basestring):
            return to_unicode(value)
        else:
            return value

    def getbool(self, name, default=''):
        """Return the value of the specified option as boolean.

        This method returns `True` if the option value is one of "yes", "true",
        "enabled", "on", or "1", ignoring case. Otherwise `False` is returned.

        Valid default input is a string or a bool. Returns a bool.
        """
        value = self.get(name, default)
        if isinstance(value, basestring):
            value = value.lower() in _TRUE_VALUES
        return bool(value)

    def getint(self, name, default=''):
        """Return the value of the specified option as integer.

        If the specified option can not be converted to an integer, a
        `ConfigurationError` exception is raised.

        Valid default input is a string or an int. Returns an int.
        """
        value = self.get(name, default)
        if not value:
            return 0
        try:
            return int(value)
        except ValueError:
            raise ConfigurationError('expected integer, got %s' % repr(value))

    def getlist(self, name, default='', sep=',', keep_empty=True):
        """Return a list of values that have been specified as a single
        comma-separated option.

        A different separator can be specified using the `sep` parameter. If
        the `keep_empty` parameter is set to `False`, empty elements are omitted
        from the list.

        Valid default input is a string or a list. Returns a list.
        """
        value = self.get(name, default)
        if not value:
            return []
        if isinstance(value, basestring):
            items = [item.strip() for item in value.split(sep)]
        else:
            items = list(value)
        if not keep_empty:
            items = filter(None, items)
        return items

    def getpath(self, name, default=''):
        """Return the value of the specified option as a path name, relative to
        the location of the configuration file the option is defined in.

        Valid default input is a string. Returns a string with normalised path.
        """
        if self.config.parser.has_option(self.name, name):
            path = self.config.parser.get(self.name, name)
            if not path:
                return default
            if not os.path.isabs(path):
                path = os.path.join(os.path.dirname(self.config.filename),
                                    path)
            return os.path.normcase(os.path.realpath(path))
        elif self.config.parent:
            return self.config.parent[self.name].getpath(name, default)
        else:
            return default

    def options(self):
        """Return `(name, value)` tuples for every option in the section."""
        for name in self:
            yield name, self.get(name)

    def set(self, name, value):
        """Change a configuration value.

        These changes are not persistent unless saved with `save()`.
        """
        if not self.config.parser.has_section(self.name):
            self.config.parser.add_section(self.name)
        if value is None:
            self.overridden[name] = True
            value = ''
        else:
            value = to_unicode(value).encode('utf-8')
        return self.config.parser.set(self.name, name, value)

class Option(object):
    """Descriptor for configuration options on `Configurable` subclasses."""

    registry = {}
    accessor = Section.get

    def __init__(self, section, name, default=None, doc=''):
        """Create the extension point.

        @param section: the name of the configuration section this option
            belongs to
        @param name: the name of the option
        @param default: the default value for the option
        @param doc: documentation of the option
        """
        self.section = section
        self.name = name
        self.default = default
        self.registry[(self.section, self.name)] = self
        self.__doc__ = doc

    def __get__(self, instance, owner):
        if instance is None:
            return self
        config = getattr(instance, 'config', None)
        if config and isinstance(config, Configuration):
            section = config[self.section]
            value = self.accessor(section, self.name, self.default)
            return value
        return None

    def __set__(self, instance, value):
        raise AttributeError, 'can\'t set attribute'

    def __repr__(self):
        return '<%s [%s] "%s">' % (self.__class__.__name__, self.section,
                                   self.name)

class BoolOption(Option):
    """Descriptor for boolean configuration options."""
    accessor = Section.getbool

class IntOption(Option):
    """Descriptor for integer configuration options."""
    accessor = Section.getint

class ListOption(Option):
    """Descriptor for configuration options that contain multiple values
    separated by a specific character."""

    def __init__(self, section, name, default=None, sep=',', keep_empty=False,
                 doc=''):
        Option.__init__(self, section, name, default, doc)
        self.sep = sep
        self.keep_empty = keep_empty

    def accessor(self, section, name, default):
        return section.getlist(name, default, self.sep, self.keep_empty)

class PathOption(Option):
    """Descriptor for file system path configuration options."""
    accessor = Section.getpath

class ExtensionOption(Option):

    def __init__(self, section, name, interface, default=None, doc=''):
        Option.__init__(self, section, name, default, doc)
        self.xtnpt = ExtensionPoint(interface)

    def __get__(self, instance, owner):
        if instance is None:
            return self
        value = Option.__get__(self, instance, owner)
        for impl in self.xtnpt.extensions(instance):
            if impl.__class__.__name__ == value:
                return impl
        raise AttributeError('Cannot find an implementation of the "%s" '
                             'interface named "%s".  Please update the option '
                             '%s.%s in trac.ini.'
                             % (self.xtnpt.interface.__name__, value,
                                self.section, self.name))

class OrderedExtensionsOption(ListOption):
    """A comma separated, ordered, list of components implementing `interface`.
    Can be empty.

    If `include_missing` is true (the default) all components implementing the
    interface are returned, with those specified by the option ordered first."""

    def __init__(self, section, name, interface, default=None,
                 include_missing=True, doc=''):
        ListOption.__init__(self, section, name, default, doc=doc)
        self.xtnpt = ExtensionPoint(interface)
        self.include_missing = include_missing

    def __get__(self, instance, owner):
        if instance is None:
            return self
        order = ListOption.__get__(self, instance, owner)
        components = []
        for impl in self.xtnpt.extensions(instance):
            if self.include_missing or impl.__class__.__name__ in order:
                components.append(impl)

        def compare(x, y):
            x, y = x.__class__.__name__, y.__class__.__name__
            if x not in order:
                return int(y in order)
            if y not in order:
                return -int(x in order)
            return cmp(order.index(x), order.index(y))
        components.sort(compare)
        return components
<|endoftext|>


<commit_before>from core import Widget
from level import *
from widgets import *
from textwrap import TextWrapper
import timing

class GameWidget ( Widget ):
    def __init__(self, level, context, *args, **kwargs):
        Widget.__init__( self, *args, **kwargs )
        self.name = None
        self.level = level
        self.player = context.player
        self.sim = context.sim
        context.game = self
        self.movementKeys = {
            'h': (-1, 0),
            'l': (1, 0),
            'j': (0, 1),
            'k': (0, -1),
            'y': (-1, -1),
            'u': (1, -1),
            'b': (-1, 1),
            'n': (1, 1),
            'west': (-1, 0),
            'east': (1, 0),
            'south': (0, 1),
            'north': (0, -1),
            'northwest': (-1, -1),
            'northeast': (1, -1),
            'southwest': (-1, 1),
            'southeast': (1, 1),
        }
        self.textfieldheight = 4
        self.playervision = None
        screenw, screenh = self.ui.dimensions()
        self.turnlogWrapper = TextWrapper( screenw, self.textfieldheight )
        self.turnlogLines = []
    def log(self, s):
        self.turnlogWrapper.feed( s + " " )
        while self.turnlogWrapper.pages:
            self.turnlogLines = self.turnlogWrapper.popPage()
            self.main.query( HitEnterWidget )
        self.turnlogLines = []
        for i in range(self.textfieldheight):
            self.turnlogLines.append( self.turnlogWrapper.line(i) )
    def resized(self, screenw, screenh):
        # quirk: will lose data in text field when resizing
        self.turnlogWrapper = TextWrapper( screenw, self.textfieldheight )
    def clearlog(self):
        screenw, screenh = self.ui.dimensions()
        self.turnlogWrapper = TextWrapper( screenw, self.textfieldheight )
        self.turnlogLines = []
    def draw(self):
        import time
        screenw, screenh = self.ui.dimensions()
        if not self.playervision:
            from vision import VisionField
            self.playervision = VisionField( self.player.tile, lambda tile: tile.impassable)
        vp = Viewport( level = self.level, window = Subwindow( self.ui, 0, self.textfieldheight, screenw, screenh - self.textfieldheight ), visibility = lambda tile : (tile.x, tile.y) in self.playervision.visible )
        vp.paint( self.player.tile.x, self.player.tile.y )
        for i in range( self.textfieldheight ):
            try:
                self.ui.putString( 0, i, self.turnlogLines[i] )
            except IndexError:
                pass
    def advanceTime(self, dt):
        self.sim.advance( dt )
        while True:
            ev = self.sim.next()
            if not ev:
                break
            ev.trigger()
    def tookAction(self, weight):
        self.advanceTime( self.player.speed * weight )
    def wait(self):
        self.log( "You wait." )
        self.tookAction( 1 )
    def tryMoveAttack(self, dx, dy):
        tile = self.player.tile.getRelative( dx, dy )
        if not tile.cannotEnterBecause( self.player ):
            self.playervision = None
            self.player.moveto( tile )
            self.tookAction( 1 )
        else:
            self.log( "You can't move there because %s." % tile.cannotEnterBecause( self.player ) )
    def pickup(self):
        if self.player.tile.items:
            item = self.player.tile.items.pop()
            self.player.inventory.append( item )
            for i in range(40):
                self.log( "You pick up the %s (%d)." % (item.name,i) )
            self.tookAction( 1 )
    def drop(self):
        if self.player.inventory:
            item = self.player.inventory.pop()
            self.player.tile.items.append( item )
            self.log( "You drop the %s." % item.name )
            self.tookAction( 1 )
    def keyboard(self, key):
        try:
            dx, dy = self.movementKeys[ key ]
            self.clearlog()
            self.tryMoveAttack( dx, dy )
            return
        except KeyError:
            pass
        try:
            standardAction = None
            standardAction = {
                '.': self.wait,
                ',': self.pickup,
                'd': self.drop,
            }[key]
        except KeyError:
            pass
        if standardAction:
            self.clearlog()
            standardAction()
        elif key == 'q':
            self.done = True
        elif key == 'N':
            from widgets import TextInputWidget
            import string
            self.name = self.main.query( TextInputWidget, 32, okay = string.letters, query = "Please enter your name: " )
        elif key == 'P':
            from pathfind import Pathfinder, infinity
            import math
            pf = Pathfinder(cost = lambda tile : infinity if tile.cannotEnterBecause( self.player ) else 1,
                            goal = lambda tile : tile.x == 5 and tile.y == 5,
                            heuristic = lambda tile : max( abs( tile.x - 5 ), abs( tile.y - 5 ) )
            )
            pf.addOrigin( self.player.tile )
            path = pf.seek()
            if path:
                for tile in path:
                    tile.fgColour = "blue"
        elif key == 'M':
            self.main.query( SelectionMenuWidget, [
                (1, "Team Cake"),
                (2, "Team Pie"),
                (3, "Team Edward"),
                (4, "Team Jacob"),
                (5, "Team Bella"),
                (6, "Team Buffy"),
            ], title = "Choose your team", padding = 5)
<commit_msg>Inventory uses menus<commit_after>from core import Widget
from level import *
from widgets import *
from textwrap import TextWrapper
import timing

class GameWidget ( Widget ):
    def __init__(self, level, context, *args, **kwargs):
        Widget.__init__( self, *args, **kwargs )
        self.name = None
        self.level = level
        self.player = context.player
        self.sim = context.sim
        context.game = self
        self.movementKeys = {
            'h': (-1, 0),
            'l': (1, 0),
            'j': (0, 1),
            'k': (0, -1),
            'y': (-1, -1),
            'u': (1, -1),
            'b': (-1, 1),
            'n': (1, 1),
            'west': (-1, 0),
            'east': (1, 0),
            'south': (0, 1),
            'north': (0, -1),
            'northwest': (-1, -1),
            'northeast': (1, -1),
            'southwest': (-1, 1),
            'southeast': (1, 1),
        }
        self.textfieldheight = 4
        self.playervision = None
        screenw, screenh = self.ui.dimensions()
        self.turnlogWrapper = TextWrapper( screenw, self.textfieldheight )
        self.turnlogLines = []
    def log(self, s):
        self.turnlogWrapper.feed( s + " " )
        while self.turnlogWrapper.pages:
            self.turnlogLines = self.turnlogWrapper.popPage()
            self.main.query( HitEnterWidget )
        self.turnlogLines = []
        for i in range(self.textfieldheight):
            self.turnlogLines.append( self.turnlogWrapper.line(i) )
    def resized(self, screenw, screenh):
        # quirk: will lose data in text field when resizing
        self.turnlogWrapper = TextWrapper( screenw, self.textfieldheight )
    def clearlog(self):
        screenw, screenh = self.ui.dimensions()
        self.turnlogWrapper = TextWrapper( screenw, self.textfieldheight )
        self.turnlogLines = []
    def draw(self):
        import time
        screenw, screenh = self.ui.dimensions()
        if not self.playervision:
            from vision import VisionField
            self.playervision = VisionField( self.player.tile, lambda tile: tile.impassable)
        vp = Viewport( level = self.level, window = Subwindow( self.ui, 0, self.textfieldheight, screenw, screenh - self.textfieldheight ), visibility = lambda tile : (tile.x, tile.y) in self.playervision.visible )
        vp.paint( self.player.tile.x, self.player.tile.y )
        for i in range( self.textfieldheight ):
            try:
                self.ui.putString( 0, i, self.turnlogLines[i] )
            except IndexError:
                pass
    def advanceTime(self, dt):
        self.sim.advance( dt )
        while True:
            ev = self.sim.next()
            if not ev:
                break
            ev.trigger()
    def tookAction(self, weight):
        self.advanceTime( self.player.speed * weight )
    def wait(self):
        self.log( "You wait." )
        self.tookAction( 1 )
    def tryMoveAttack(self, dx, dy):
        tile = self.player.tile.getRelative( dx, dy )
        if not tile.cannotEnterBecause( self.player ):
            self.playervision = None
            self.player.moveto( tile )
            self.tookAction( 1 )
        else:
            self.log( "You can't move there because %s." % tile.cannotEnterBecause( self.player ) )
    def pickup(self):
        if self.player.tile.items:
            if len( self.player.tile.items ) == 1:
                item = self.player.tile.items.pop()
            else:
                item = self.main.query( SelectionMenuWidget, choices = [
                    (item, item.name) for item in self.player.tile.items
                ] + [ (None, "nothing") ], title = "Pick up which item?", padding = 5 )
                if not item:
                    return
                self.player.tile.items.remove( item )
            self.player.inventory.append( item )
            self.log( "You pick up the %s." % (item.name) )
            self.tookAction( 1 )
    def drop(self):
        if self.player.inventory:
            item = self.main.query( SelectionMenuWidget, choices = [
                (item, item.name) for item in self.player.inventory
            ] + [ (None, "nothing") ], title = "Drop which item?", padding = 5 )
            if not item:
                return
            item = self.player.inventory.pop()
            self.player.tile.items.append( item )
            self.log( "You drop the %s." % item.name )
            self.tookAction( 1 )
    def keyboard(self, key):
        try:
            dx, dy = self.movementKeys[ key ]
            self.clearlog()
            self.tryMoveAttack( dx, dy )
            return
        except KeyError:
            pass
        try:
            standardAction = None
            standardAction = {
                '.': self.wait,
                ',': self.pickup,
                'd': self.drop,
            }[key]
        except KeyError:
            pass
        if standardAction:
            self.clearlog()
            standardAction()
        elif key == 'q':
            self.done = True
        elif key == 'N':
            from widgets import TextInputWidget
            import string
            self.name = self.main.query( TextInputWidget, 32, okay = string.letters, query = "Please enter your name: " )
        elif key == 'P':
            from pathfind import Pathfinder, infinity
            import math
            pf = Pathfinder(cost = lambda tile : infinity if tile.cannotEnterBecause( self.player ) else 1,
                            goal = lambda tile : tile.x == 5 and tile.y == 5,
                            heuristic = lambda tile : max( abs( tile.x - 5 ), abs( tile.y - 5 ) )
            )
            pf.addOrigin( self.player.tile )
            path = pf.seek()
            if path:
                for tile in path:
                    tile.fgColour = "blue"
        elif key == 'M':
            self.main.query( SelectionMenuWidget, [
                (1, "Team Cake"),
                (2, "Team Pie"),
                (3, "Team Edward"),
                (4, "Team Jacob"),
                (5, "Team Bella"),
                (6, "Team Buffy"),
            ], title = "Choose your team", padding = 5)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Author: Peter Withers
# Date: 2014/02/26

import csv
from base64 import encodestring
import json
import sys
import time
import urllib
import urllib2

githubUsername = sys.argv[1]
githubProject = sys.argv[2]
githubRepository = sys.argv[3]
print 'GitHub username: ', githubUsername
print 'GitHub project:', githubProject
print 'GitHub repository: ', githubRepository
tracCsvUrl = 'https://trac.mpi.nl/query?component=KinOath-desktop&component=KinOath-web&max=10000&col=id&col=summary&col=component&col=owner&col=type&col=status&col=priority&col=milestone&col=version&col=resolution&col=time&col=changetime&col=reporter&col=keywords&col=cc&report=24&order=milestone&col=description&format=csv'
githubApiUrl = 'https://api.github.com'

milestoneUrl = githubApiUrl + '/repos/%s/%s/milestones' % (githubProject, githubRepository)
issuesUrl = githubApiUrl + '/repos/%s/%s/issues' % (githubProject, githubRepository)
print(milestoneUrl)
print(issuesUrl)

knownMilestones = []

def getAllMileStones():
    response = urllib.urlopen(milestoneUrl)
    content = response.read()
    #print(content)
    milestones = json.loads(content)
    #print(milestones)
    del knownMilestones[:]
    for entry in milestones:
        print entry['number'], ' : ', entry['title']
        knownMilestones.insert(entry['number'], entry['title'])

def getMileStoneId(milestoneTitle):
    if len(milestoneTitle.strip()) == 0:
        return ""
    if not milestoneTitle in knownMilestones:
        data=json.dumps({'title': milestoneTitle, 'state': 'open'})
        #data=json.dumps({'title': 'milestone', 'state': 'open', 'desription': '.', 'due_on': '2012-10-09T23:39:01Z'})
        print(data)
        datalength = len(data)
        request = urllib2.Request(milestoneUrl, data, {'Content-Type': 'application/json', 'Content-Length': datalength})
        #request = urllib2.Request(milestoneUrl, data)
        #, 'Content-Length': datalength}
        request.add_header('Authorization', 'Basic %s' % base64string)
        response = urllib2.urlopen(request)
        print(response)
        knownMilestones.append(milestoneTitle)
        time.sleep(1)
    return knownMilestones.index(milestoneTitle)

def createRequiredEmptyTickets(tracTicketCount):
    print 'total ticket count: ', tracTicketCount
    response = urllib.urlopen(issuesUrl)
    content = response.read()
    tickets = json.loads(content)
    #print(tickets)
    currentIssueCount = tickets[0]['number']
    while tracTicketCount > currentIssueCount:
        currentIssueCount = currentIssueCount + 1
        makeEmptyIssueRequest()

def makeEmptyIssueRequest():
    data=json.dumps({'title': 'dummy ticket to allow imports with the correct ticket id'})
    datalength = len(data)
    request = urllib2.Request(issuesUrl, data, {'Content-Type': 'application/json', 'Content-Length': datalength})
    request.add_header('Authorization', 'Basic %s' % base64string)
    response = urllib2.urlopen(request)
    print(response)
    time.sleep(1)

def makeIssueRequest(ticketId, data):
    print (ticketId)
    print(data)
    datalength = len(data)
    request = urllib2.Request(issuesUrl + '/' + ticketId, data, {'Content-Type': 'application/json', 'Content-Length': datalength})
    request.add_header('Authorization', 'Basic %s' % base64string)
    response = urllib2.urlopen(request)
    print(response)
    time.sleep(1)

getAllMileStones()

githubToken = raw_input("GitHub Password:")
base64string = encodestring('%s:%s' % (githubUsername, githubToken)).replace('\n', '')

tracCSV = urllib.urlopen(tracCsvUrl)
tracTickets = csv.DictReader(tracCSV)

# print(tracTickets)
# create any missing milestones
print 'adding milestones'
tracTicketCount = 0
for ticket in tracTickets:
    tracTicketCount = tracTicketCount + 1
    milestoneId = getMileStoneId(ticket['milestone'])
# make sure the milestone list is up to date
getAllMileStones()
print 'adding tickets'
createRequiredEmptyTickets(tracTicketCount)
# insert the tickets as issues
tracCSV = urllib.urlopen(tracCsvUrl)
tracTickets = csv.DictReader(tracCSV)
currentTicket = 0
for ticket in tracTickets:
    #print (ticket)
    currentTicket = currentTicket + 1
    milestoneId = getMileStoneId(ticket['milestone'])
    print 'milestone: ', milestoneId
    if ticket['status'] == 'closed':
        status = 'closed'
    else:
        status = 'open'
    data=json.dumps({'title': ticket['summary'], 'body': ticket['description'], 'milestone': milestoneId, "state": status,
    'labels': [ticket['component'], ticket['type'], ticket['priority'], ticket['resolution']]})
    # so far unused fields: col=id& &col=time &col=changetime &col=reporter &col=keywords &col=cc 'assignee': ticket['owner'], , ticket['version']
    makeIssueRequest(str(currentTicket), data) # ticket['id']
exit(0)
<commit_msg>Updated the python script to import the tickets and milestones from trac to github.<commit_after>#!/usr/bin/python
# Author: Peter Withers
# Date: 2014/02/26

import csv
from base64 import encodestring
import json
import sys
import time
import urllib
import urllib2

githubUsername = sys.argv[1]
githubProject = sys.argv[2]
githubRepository = sys.argv[3]
print 'GitHub username: ', githubUsername
print 'GitHub project:', githubProject
print 'GitHub repository: ', githubRepository
tracCsvUrl = 'https://trac.mpi.nl/query?component=KinOath-desktop&component=KinOath-web&max=10000&col=id&col=summary&col=component&col=owner&col=type&col=status&col=priority&col=milestone&col=version&col=resolution&col=time&col=changetime&col=reporter&col=keywords&col=cc&report=24&order=milestone&col=description&format=csv'
githubApiUrl = 'https://api.github.com'

milestoneUrl = githubApiUrl + '/repos/%s/%s/milestones' % (githubProject, githubRepository)
issuesUrl = githubApiUrl + '/repos/%s/%s/issues' % (githubProject, githubRepository)
print(milestoneUrl)
print(issuesUrl)

knownMilestones = []

def getAllMileStones():
    response = urllib.urlopen(milestoneUrl)
    content = response.read()
    #print(content)
    milestones = json.loads(content)
    #print(milestones)
    del knownMilestones[:]
    for entry in milestones:
        print entry['number'], ' : ', entry['title']
        knownMilestones.insert(entry['number'], entry['title'])

def getMileStoneId(milestoneTitle):
    if len(milestoneTitle.strip()) == 0:
        return ""
    if not milestoneTitle in knownMilestones:
        data=json.dumps({'title': milestoneTitle, 'state': 'open'})
        #data=json.dumps({'title': 'milestone', 'state': 'open', 'desription': '.', 'due_on': '2012-10-09T23:39:01Z'})
        print(data)
        datalength = len(data)
        request = urllib2.Request(milestoneUrl, data, {'Content-Type': 'application/json', 'Content-Length': datalength})
        #request = urllib2.Request(milestoneUrl, data)
        #, 'Content-Length': datalength}
        request.add_header('Authorization', 'Basic %s' % base64string)
        response = urllib2.urlopen(request)
        print(response)
        knownMilestones.append(milestoneTitle)
        time.sleep(1)
    return knownMilestones.index(milestoneTitle)

def createRequiredEmptyTickets(tracTicketCount):
    print 'total ticket count: ', tracTicketCount
    response = urllib.urlopen(issuesUrl)
    content = response.read()
    tickets = json.loads(content)
    #print(tickets)
    currentIssueCount = tickets[0]['number']
    while tracTicketCount > currentIssueCount:
        currentIssueCount = currentIssueCount + 1
        makeEmptyIssueRequest()

def makeEmptyIssueRequest():
    data=json.dumps({'title': 'dummy ticket to allow imports with the correct ticket id'})
    datalength = len(data)
    request = urllib2.Request(issuesUrl, data, {'Content-Type': 'application/json', 'Content-Length': datalength})
    request.add_header('Authorization', 'Basic %s' % base64string)
    response = urllib2.urlopen(request)
    print(response)
    time.sleep(1)

def makeIssueRequest(ticketId, data):
    print (ticketId)
    print(data)
    datalength = len(data)
    request = urllib2.Request(issuesUrl + '/' + ticketId, data, {'Content-Type': 'application/json', 'Content-Length': datalength})
    request.add_header('Authorization', 'Basic %s' % base64string)
    response = urllib2.urlopen(request)
    print(response)
    time.sleep(1)

getAllMileStones()

githubToken = raw_input("GitHub Password:")
base64string = encodestring('%s:%s' % (githubUsername, githubToken)).replace('\n', '')

tracCSV = urllib.urlopen(tracCsvUrl)
tracTickets = csv.DictReader(tracCSV)

# print(tracTickets)
# create any missing milestones
print 'adding milestones'
tracTicketCount = 0
for ticket in tracTickets:
    tracTicketCount = tracTicketCount + 1
    milestoneId = getMileStoneId(ticket['milestone'])
# make sure the milestone list is up to date
getAllMileStones()
print 'adding tickets'
createRequiredEmptyTickets(tracTicketCount)
# insert the tickets as issues
tracCSV = urllib.urlopen(tracCsvUrl)
tracTickets = csv.DictReader(tracCSV)
currentTicket = 0
for ticket in tracTickets:
    #print (ticket)
    currentTicket = currentTicket + 1
    milestoneId = getMileStoneId(ticket['milestone'])
    #print 'milestone: ', milestoneId
    if ticket['status'] == 'closed':
        status = 'closed'
    else:
        status = 'open'
    if milestoneId == 0:
        # if there is no milestone then we must not pass it as a parameter
        data=json.dumps({'title': ticket['summary'], 'body': ticket['description'].strip(), "state": status, 'labels': [ticket['component'], ticket['type'], ticket['priority'], ticket['resolution']]})
    else:
        data=json.dumps({'title': ticket['summary'], 'body': ticket['description'].strip(), 'milestone': milestoneId, "state": status, 'labels': [ticket['component'], ticket['type'], ticket['priority'], ticket['resolution']]})
    # so far unused fields: col=id& &col=time &col=changetime &col=reporter &col=keywords &col=cc 'assignee': ticket['owner'], , ticket['version']
    #if currentTicket > 223: # ticket 224 has issues that are as yet un identified, ticket 147 is the fist one to not have a milestone
    makeIssueRequest(str(currentTicket), data) # ticket['id']
exit(0)
<|endoftext|>


<commit_before>import serial
import math

def calculate_temp(value):
    voltage = float(value) / 1024 * 5
    resistance = 10000 / (5 / voltage - 1)
    temp = 1 / (1/298.15 + math.log(resistance / 10000) / 3977)
    print "Temperature is: %f K, (%f degC)" % (temp, temp-273.15)
    return

ser = serial.Serial('/dev/tty.usbserial-A800etDk', 9600)
while 1:
    r = ser.readline()
    split = r.split(": ")
    if split[0] == "sensorValue":
        value = split[1].strip()
        calculate_temp(value)

<commit_msg>Updates conair.py to push data to TempoDB<commit_after>import serial
import math
import datetime
import random
from tempodb import Client, DataPoint

client = Client('cc0c654d01774b128c1e0495de51784b', 'a280c43f6b27400998a4aba0b1eb4545')

def calculate_temp(value):
    voltage = float(value) / 1024 * 5
    resistance = 10000 / (5 / voltage - 1)
    temp = 1 / (1/298.15 + math.log(resistance / 10000) / 3977)
    print "Temperature is: %f K, (%f degC)" % (temp, temp-273.15)
    return temp-273.15

ser = serial.Serial('/dev/tty.usbserial-A800etDk', 9600)
while 1:
    r = ser.readline()
    split = r.split(": ")
    if split[0] == "sensorValue":
        value = split[1].strip()
        temp = calculate_temp(value)
        client.write_key('temperature', DataPoint(datetime.datetime.now(), temp))

<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
import sys
import os
import sqlite3
from datetime import datetime
import subprocess
import re

GIT = "/opt/local/bin/git"

def format_user(entry):
    """ git requires users to conform to "A B <ab@c.de>"

    >>> format_user(['', '', '', u'user', u'127.0.0.1'])
    u'user <127.0.0.1>'
    >>> format_user(['', '', '', u'user@home.local', u'127.0.0.1'])
    u'user <user@home.local>'
    >>> format_user(['', '', '', u'user <user@home.local>', u'127.0.0.1'])
    u'user <user@home.local>'
    """
    user = entry[3]
    if u"<" in user and "@" in user:
        return user
    if u"@" in user:
        u, d = user.split(u"@")
        return u"%s <%s>" % (u, user)
    ip = entry[4]
    return u"%s <%s>" % (user, ip)

def format_comment(entry):
    comment = entry[6]
    if not comment:
        return u'Page "%s" updated.' % (entry[0])
    return comment

#
#   I hope you don't need to change anything below this line
#

def getargs():
    """ get database file to read from and git repository to write to from
        commandline
    """
    try:
        read = sys.argv[1]
        write = sys.argv[2]
        print u'Reading "%s", writing "%s".' % (sys.argv[1], sys.argv[2])
        if os.path.isfile(read) and os.path.isdir(os.path.join(write, ".git")):
            r = sqlite3.connect(read)
            return (r, write)
        else:
            print u'ERROR: Either file "%s" or git repository "%s" does not exist.' % (read, write)
            sys.exit(1)
    except IndexError:
        print u'Try: "%s trac.db git-repo' % (sys.argv[0])
        sys.exit(1)

def format_time(timestamp):
    """ return a git compatible timestamp

    >>> format_time(1229442008.852975)
    u'1229442008 0000'
    """
    return str(int(timestamp)) + " 0000".decode("UTF-8")

def convert_code(text):
    """ replace code blocks (very primitive)

    >>> convert_code(u"\\nTest\\n\\n{{{\\n#!sh\\nCode paragraph\\n}}}\\n\\nTest\\n")
    u'\\nTest\\n\\n```sh\\nCode paragraph\\n```\\n\\nTest\\n'
    >>> convert_code(u"\\nTest\\n\\n{{{\\nCode paragraph\\n}}}\\n\\nTest\\n")
    u'\\nTest\\n\\n```\\nCode paragraph\\n```\\n\\nTest\\n'

    """
    result = u""
    start = False
    running = False
    original = text
    indent = u""
    for line in text.splitlines():
        if line.strip() == u"{{{":
            start = True
            running = True
        elif start:
            start = False
            if line.startswith("#!"):
                result += u"```" + line.replace("#!", "") + os.linesep
            else:
                indent = u"    "
                result += os.linesep + indent + line + os.linesep
        elif line.strip() == u"}}}" and running:
            running = False
            if indent:
                indent = u""
                result += os.linesep
            else:
                result += u"```" + os.linesep
        else:
            result += indent + line + os.linesep
    if running:
        # something went wrong; don't touch the text.
        return original
    return result

re_inlinecode = re.compile(r'\{\{\{([^\n]+?)\}\}\}')
re_h4 = re.compile(r'====\s(.+?)\s====')
re_h3 = re.compile(r'===\s(.+?)\s===')
re_h2 = re.compile(r'==\s(.+?)\s==')
re_h1 = re.compile(r'=\s(.+?)\s=')
re_uri = re.compile(r'\[(?:wiki:)?([^\s]+)\s(.+)\]')
re_CamelCaseUri = re.compile(r'([^"\/\!])(([A-Z][a-z0-9]+){2,})')
re_NoUri = re.compile(r'\!(([A-Z][a-z0-9]+){2,})')
re_strong = re.compile(r"'''(.+)'''")
re_italic = re.compile(r"''(.+)''")
re_ul = re.compile(r'(^\s\*)', re.MULTILINE)
re_ol = re.compile(r'^\s(\d+\.)', re.MULTILINE)

def format_text(text):
    """ converts trac wiki to gollum markdown syntax

    >>> format_text(u"= One =\\n== Two ==\\n=== Three ===\\n==== Four ====")
    u'# One\\n## Two\\n### Three\\n#### Four\\n'
    >>> format_text(u"Paragraph with ''italic'' and '''bold'''.")
    u'Paragraph with *italic* and **bold**.\\n'
    >>> format_text(u"Example with [wiki:a/b one link].")
    u'Example with [[one link|a/b]].\\n'
    >>> format_text(u"Beispiel mit [http://blog.fefe.de Fefes Blog] Link.")
    u'Beispiel mit [[Fefes Blog|http://blog.fefe.de]] Link.\\n'
    >>> format_text(u"Beispiel mit CamelCase Link.")
    u'Beispiel mit [[CamelCase]] Link.\\n'
    >>> format_text(u"Beispiel ohne !CamelCase Link.")
    u'Beispiel ohne CamelCase Link.\\n'
    >>> format_text(u"Test {{{inline code}}}\\n\\nand more {{{inline code}}}.")
    u'Test `inline code`\\n\\nand more `inline code`.\\n'
    >>> format_text(u"\\n * one\\n * two\\n")
    u'\\n* one\\n* two\\n'
    >>> format_text(u"\\n 1. first\\n 2. second\\n")
    u'\\n1. first\\n2. second\\n'
    """
    # TODO: ticket: and source: links are not yet handled
    text = convert_code(text)
    text = re_inlinecode.sub(r'`\1`', text)
    text = re_h4.sub(r'#### \1', text)
    text = re_h3.sub(r'### \1', text)
    text = re_h2.sub(r'## \1', text)
    text = re_h1.sub(r'# \1', text)
    text = re_uri.sub(r'[[\2|' + r'\1]]', text)
    text = re_CamelCaseUri.sub(r'\1[[\2]]', text)
    text = re_NoUri.sub(r'\1', text)
    text = re_strong.sub(r'**\1**', text)
    text = re_italic.sub(r'*\1*', text)
    text = re_ul.sub(r'*', text)
    text = re_ol.sub(r'\1', text)
    return text

def format_page(page):
    """ rename WikiStart to Home

    >>> format_page(u'test')
    u'test'
    >>> format_page(u'WikiStart')
    u'Home'
    """
    if page == u"WikiStart":
        return u"Home"
    # Gollum wiki replaces slash and space with dash:
    return page.replace(u"/", u"-").replace(u" ", u"-")

def read_database(db):
    # get all pages except those generated by the trac system itself (help etc.)
    pages = [x[0] for x in db.execute('select name from wiki where ipnr != "127.0.0.1" group by name', []).fetchall()]
    for page in pages:
        for revision in db.execute('select * from wiki where name is ? order by version', [page]).fetchall():
            result = {}
            result["page"] = format_page(revision[0])
            result["version"] = revision[1]
            result["time"]  = format_time(revision[2])
            result["user"] = format_user(revision)
            result["ip"] = revision[4]
            result["text"] = format_text(revision[5])
            result["comment"] = format_comment(revision)
            yield result

def main():
    db, target = getargs()
    source = read_database(db)
    for entry in source:
        # make paths conform to local filesystem
        page = os.path.normpath(entry["page"] + u".md")
        if not os.path.supports_unicode_filenames:
            page = page.encode("utf-8")
        try:
            open(os.path.join(target, page), "wb").write(entry["text"].encode("utf-8"))
            subprocess.check_call([GIT, "add", page], cwd=target)
            try:
                subprocess.check_call([GIT, "commit", "--author", entry["user"],
                                       "--date", entry["time"], "-m", entry["comment"]], cwd=target)
            # trying to circumvent strange unicode-encoded file name problems:
            except subprocess.CalledProcessError:
                [subprocess.check_call([GIT, "add", x], cwd=target) for x in os.listdir(target)]
                subprocess.check_call([GIT, "commit", "--author", entry["user"],
                                       "--date", entry["time"], "-m", entry["comment"]], cwd=target)

        except Exception, e:
            print "\n\n\nXXX Problem: ", e
            sys.exit(23)
    # finally garbage collect git repository
    subprocess.check_call([GIT, "gc"], cwd=target)

if __name__ == "__main__":
    main()<commit_msg>Convert only the most recent revision and commit as a new revision.<commit_after>#!/usr/bin/env python2.7
import sys
import os
import sqlite3
from datetime import datetime
import subprocess
import re

GIT = "/opt/local/bin/git"

def format_user(entry):
    """ git requires users to conform to "A B <ab@c.de>"

    >>> format_user(['', '', '', u'user', u'127.0.0.1'])
    u'user <127.0.0.1>'
    >>> format_user(['', '', '', u'user@home.local', u'127.0.0.1'])
    u'user <user@home.local>'
    >>> format_user(['', '', '', u'user <user@home.local>', u'127.0.0.1'])
    u'user <user@home.local>'
    """
    user = entry[3]
    if u"<" in user and "@" in user:
        return user
    if u"@" in user:
        u, d = user.split(u"@")
        return u"%s <%s>" % (u, user)
    ip = entry[4]
    return u"%s <%s>" % (user, ip)

def format_comment(entry, final):
    """ creates / formats commit comment.
        "final" is true when content is converted from Trac markup to Markdown.
    """
    comment = entry[6] or (u'Page "%s" updated.' % (entry[0]))
    if final:
        return u'%s (automatically converted to Markdown)' % comment
    return comment

#
#   I hope you don't need to change anything below this line
#

def getargs():
    """ get database file to read from and git repository to write to from
        commandline
    """
    try:
        read = sys.argv[1]
        write = sys.argv[2]
        print u'Reading "%s", writing "%s".' % (sys.argv[1], sys.argv[2])
        if os.path.isfile(read) and os.path.isdir(os.path.join(write, ".git")):
            r = sqlite3.connect(read)
            return (r, write)
        else:
            print u'ERROR: Either file "%s" or git repository "%s" does not exist.' % (read, write)
            sys.exit(1)
    except IndexError:
        print u'Try: "%s trac.db git-repo' % (sys.argv[0])
        sys.exit(1)

def format_time(timestamp):
    """ return a git compatible timestamp

    >>> format_time(1229442008.852975)
    u'1229442008 0000'
    """
    return str(int(timestamp)) + " 0000".decode("UTF-8")

def convert_code(text):
    """ replace code blocks (very primitive)

    >>> convert_code(u"\\nTest\\n\\n{{{\\n#!sh\\nCode paragraph\\n}}}\\n\\nTest\\n")
    u'\\nTest\\n\\n```sh\\nCode paragraph\\n```\\n\\nTest\\n'
    >>> convert_code(u"\\nTest\\n\\n{{{\\nCode paragraph\\n}}}\\n\\nTest\\n")
    u'\\nTest\\n\\n\\n    Code paragraph\\n\\n\\nTest\\n'

    """
    result = u""
    start = False
    running = False
    original = text
    indent = u""
    for line in text.splitlines():
        if line.strip() == u"{{{":
            start = True
            running = True
        elif start:
            start = False
            if line.startswith("#!"):
                result += u"```" + line.replace("#!", "") + os.linesep
            else:
                indent = u"    "
                result += os.linesep + indent + line + os.linesep
        elif line.strip() == u"}}}" and running:
            running = False
            if indent:
                indent = u""
                result += os.linesep
            else:
                result += u"```" + os.linesep
        else:
            result += indent + line + os.linesep
    if running:
        # something went wrong; don't touch the text.
        return original
    return result

re_inlinecode = re.compile(r'\{\{\{([^\n]+?)\}\}\}')
re_h4 = re.compile(r'====\s(.+?)\s====')
re_h3 = re.compile(r'===\s(.+?)\s===')
re_h2 = re.compile(r'==\s(.+?)\s==')
re_h1 = re.compile(r'=\s(.+?)\s=')
re_uri = re.compile(r'\[(?:wiki:)?([^\s]+)\s(.+)\]')
re_CamelCaseUri = re.compile(r'([^"\/\!])(([A-Z][a-z0-9]+){2,})')
re_NoUri = re.compile(r'\!(([A-Z][a-z0-9]+){2,})')
re_strong = re.compile(r"'''(.+)'''")
re_italic = re.compile(r"''(.+)''")
re_ul = re.compile(r'(^\s\*)', re.MULTILINE)
re_ol = re.compile(r'^\s(\d+\.)', re.MULTILINE)

def format_text(text):
    """ converts trac wiki to gollum markdown syntax

    >>> format_text(u"= One =\\n== Two ==\\n=== Three ===\\n==== Four ====")
    u'# One\\n## Two\\n### Three\\n#### Four\\n'
    >>> format_text(u"Paragraph with ''italic'' and '''bold'''.")
    u'Paragraph with *italic* and **bold**.\\n'
    >>> format_text(u"Example with [wiki:a/b one link].")
    u'Example with [[one link|a/b]].\\n'
    >>> format_text(u"Beispiel mit [http://blog.fefe.de Fefes Blog] Link.")
    u'Beispiel mit [[Fefes Blog|http://blog.fefe.de]] Link.\\n'
    >>> format_text(u"Beispiel mit CamelCase Link.")
    u'Beispiel mit [[CamelCase]] Link.\\n'
    >>> format_text(u"Beispiel ohne !CamelCase Link.")
    u'Beispiel ohne CamelCase Link.\\n'
    >>> format_text(u"Test {{{inline code}}}\\n\\nand more {{{inline code}}}.")
    u'Test `inline code`\\n\\nand more `inline code`.\\n'
    >>> format_text(u"\\n * one\\n * two\\n")
    u'\\n* one\\n* two\\n'
    >>> format_text(u"\\n 1. first\\n 2. second\\n")
    u'\\n1. first\\n2. second\\n'
    """
    # TODO: ticket: and source: links are not yet handled
    text = convert_code(text)
    text = re_inlinecode.sub(r'`\1`', text)
    text = re_h4.sub(r'#### \1', text)
    text = re_h3.sub(r'### \1', text)
    text = re_h2.sub(r'## \1', text)
    text = re_h1.sub(r'# \1', text)
    text = re_uri.sub(r'[[\2|' + r'\1]]', text)
    text = re_CamelCaseUri.sub(r'\1[[\2]]', text)
    text = re_NoUri.sub(r'\1', text)
    text = re_strong.sub(r'**\1**', text)
    text = re_italic.sub(r'*\1*', text)
    text = re_ul.sub(r'*', text)
    text = re_ol.sub(r'\1', text)
    return text

def format_page(page):
    """ rename WikiStart to Home

    >>> format_page(u'test')
    u'test'
    >>> format_page(u'WikiStart')
    u'Home'
    """
    if page == u"WikiStart":
        return u"Home"
    # Gollum wiki replaces slash and space with dash:
    return page.replace(u"/", u"-").replace(u" ", u"-")

def read_database(db):
    # get all pages except those generated by the trac system itself (help etc.)
    pages = [x[0] for x in db.execute('select name from wiki where ipnr != "127.0.0.1" group by name', []).fetchall()]
    for page in pages:
        for revision in db.execute('select * from wiki where name is ? order by version', [page]).fetchall():
            yield {
                "page": format_page(revision[0]),
                "version": revision[1],
                "time": format_time(revision[2]),
                "user": format_user(revision),
                "ip": revision[4],
                "text": revision[5],
                "comment": format_comment(revision, final=False),
            }
        latest = db.execute('select name, max(version), time, author, ipnr, text, comment from wiki where name is ?',
                            [page]).fetchall()[0]
        yield {
            "page": format_page(latest[0]),
            "version": latest[1],
            "time": format_time(latest[2]),
            "user": format_user(latest),
            "ip": latest[4],
            "text": format_text(latest[5]),
            "comment": format_comment(latest, final=True),
        }

def main():
    db, target = getargs()
    source = read_database(db)
    for entry in source:
        # make paths conform to local filesystem
        page = os.path.normpath(entry["page"] + u".md")
        if not os.path.supports_unicode_filenames:
            page = page.encode("utf-8")
        try:
            open(os.path.join(target, page), "wb").write(entry["text"].encode("utf-8"))
            subprocess.check_call([GIT, "add", page], cwd=target)
            try:
                subprocess.check_call([GIT, "commit", "--author", entry["user"],
                                       "--date", entry["time"], "-m", entry["comment"]], cwd=target)
            # trying to circumvent strange unicode-encoded file name problems:
            except subprocess.CalledProcessError:
                [subprocess.check_call([GIT, "add", x], cwd=target) for x in os.listdir(target)]
                subprocess.check_call([GIT, "commit", "--author", entry["user"],
                                       "--date", entry["time"], "-m", entry["comment"]], cwd=target)

        except Exception, e:
            print "\n\n\nXXX Problem: ", e
            sys.exit(23)
    # finally garbage collect git repository
    subprocess.check_call([GIT, "gc"], cwd=target)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import translator.common.utils
from translator.toscalib.tests.base import TestCase

class CommonUtilsTest(TestCase):

    MemoryUnit = translator.common.utils.MemoryUnit

    def test_convert_unit_size_to_num(self):
        size = '1 TB'
        num_to_convert = 'GB'
        expected_output = 1000
        output = self.MemoryUnit.convert_unit_size_to_num(size, num_to_convert)
        self.assertEqual(output, expected_output)

        size = '40 GB'
        num_to_convert = 'MB'
        expected_output = 40000
        output = self.MemoryUnit.convert_unit_size_to_num(size, num_to_convert)
        self.assertEqual(output, expected_output)

    def test_validate_unit(self):
        unit = 'AB'
        exp_msg = ('Provided unit "{0}" is not valid. The valid units are '
                   '{1}').format(unit, self.MemoryUnit.UNIT_SIZE_DICT.keys())
        try:
            self.MemoryUnit.validate_unit(unit)
        except Exception as err:
            self.assertTrue(
                isinstance(err, ValueError))
            self.assertEqual(exp_msg, err.__str__())
<commit_msg>Added unit test to cover ValueError in str_to_num<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import translator.common.utils
from translator.toscalib.tests.base import TestCase

class CommonUtilsTest(TestCase):

    MemoryUnit = translator.common.utils.MemoryUnit

    def test_convert_unit_size_to_num(self):
        size = '1 TB'
        num_to_convert = 'GB'
        expected_output = 1000
        output = self.MemoryUnit.convert_unit_size_to_num(size, num_to_convert)
        self.assertEqual(output, expected_output)

        size = '40 GB'
        num_to_convert = 'MB'
        expected_output = 40000
        output = self.MemoryUnit.convert_unit_size_to_num(size, num_to_convert)
        self.assertEqual(output, expected_output)

    def test_validate_unit(self):
        unit = 'AB'
        exp_msg = ('Provided unit "{0}" is not valid. The valid units are '
                   '{1}').format(unit, self.MemoryUnit.UNIT_SIZE_DICT.keys())
        try:
            self.MemoryUnit.validate_unit(unit)
        except Exception as err:
            self.assertTrue(
                isinstance(err, ValueError))
            self.assertEqual(exp_msg, err.__str__())

    def test_str_to_num_value_error(self):
        str_to_convert = '55063.000000'
        expected_output = 55063.0
        output = translator.common.utils.str_to_num(str_to_convert)
        self.assertEquals(output, expected_output)
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Concatenates video files using FFmpeg stream copy.
Uses mkvpropedit to insert chapter markers for each source file.
Optionally wraps the file into a package structure with checksum manifests.
'''
import sys
import subprocess
import os
import argparse
import time
import shutil
import sipcreator
import ififuncs

def parse_args(args_):
    '''
    Parse command line arguments.
    '''
    parser = argparse.ArgumentParser(
        description='Concatenate video files using ffmpeg stream copy'
        ' Written by Kieran O\'Leary.'
    )
    parser.add_argument(
        '-i', nargs='+',
        help='full path of input directory', required=True
    )
    parser.add_argument(
        '-o', '-output',
        help='full path of output directory', required=True
    )
    parser.add_argument(
        '-r', '-recursive',
        help='recursively process all files in subdirectories. This could be potentially a disaster - so use with caution or with XDCAM', action='store_true'
    )
    parser.add_argument(
        '--no-sip',
        help='Do not run sipcreator.py on the resulting file.', action='store_true'
    )
    parser.add_argument(
        '-user',
        help='Declare who you are. If this is not set, you will be prompted.')
    parser.add_argument(
        '-oe',
        help='Enter the Object Entry number for the representation.SIP will be placed in a folder with this name.'
    )
    parsed_args = parser.parse_args(args_)
    return parsed_args

def ffmpeg_concat(concat_file, args, uuid):
    '''
    Launch the actual ffmpeg concatenation command.
    '''
    fmd5_logfile = os.path.join(args.o, '%s_concat.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
    fmd5_env_dict = ififuncs.set_environment(fmd5_logfile)
    print fmd5_logfile
    print fmd5_env_dict
    cmd = [
        'ffmpeg', '-report', '-f', 'concat', '-safe', '0',
        '-i', concat_file,
        '-c', 'copy', '-map', '0:v', '-map', '0:a?',
        os.path.join(args.o, '%s.mkv' % uuid),
        '-f', 'md5', '-map', '0:v', '-map', '0:a?','-c', 'copy',  '-'
    ]
    print cmd
    source_bitstream_md5 = subprocess.check_output(
        cmd, env=fmd5_env_dict
    )
    return source_bitstream_md5.rstrip(), fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')

def recursive_file_list(video_files):
    '''
    Recursively searches through directories for AV files and adds to a list.
    '''
    # check if all inputs are actually directories
    recursive_list = []
    for directory in video_files:
        if not os.path.isdir(directory):
            print 'You have selected the recursive option, but not all of your inputs are directories.'
            sys.exit()
        else:
            for root, _, filenames in os.walk(directory):
                for filename in filenames:
                    if filename.endswith(('.MP4', '.mp4', '.mov', '.mkv', '.mxf')):
                        recursive_list.append(os.path.join(root, filename))
    print recursive_list
    return recursive_list

def make_chapters(video_files):
    '''
    Use mkvpropedit to insert chapter markers for each source video.
    Each chapter's name will reflect the source filename of each clip.
    '''
    millis = ififuncs.get_milliseconds(video_files[0])
    timestamp = ififuncs.convert_millis(int(millis))
    chapter_list = [['00:00:00.000', os.path.basename(video_files[0])], [timestamp, os.path.basename(video_files[1])]]
    count = 2
    for video in video_files[1:]:
        millis += ififuncs.get_milliseconds(video)
        timestamp = ififuncs.convert_millis(int(millis))
        if count == len(video_files):
            continue
        else:
            chapter_list.append([timestamp, os.path.basename(video_files[count])])
        count+=1
    chapter_counter = 1
    # uh use a real path/filename.
    with open('chapters.txt', 'wb') as fo:
        for i in chapter_list:
            fo.write('CHAPTER%s=%s\nCHAPTER%sNAME=%s\n' % (str(chapter_counter).zfill(2), i[0], str(chapter_counter).zfill(2), i[1]))
            chapter_counter += 1

def main(args_):
    '''
    Launches the functions that prepare and execute the concatenation.
    '''
    uuid = ififuncs.create_uuid()
    args = parse_args(args_)
    print args
    log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
    ififuncs.generate_log(log_name_source, 'concat.py started.')
    ififuncs.generate_log(
        log_name_source,
        'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
    ififuncs.generate_log(
        log_name_source,
        'Command line arguments: %s' % args
    )
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    if args.oe:
        if args.oe[:2] != 'oe':
            print 'First two characters must be \'oe\' and last four characters must be four digits'
            object_entry = ififuncs.get_object_entry()
        elif len(args.oe[2:]) != 4:
            print 'First two characters must be \'oe\' and last four characters must be four digits'
            object_entry = ififuncs.get_object_entry()
        elif not args.oe[2:].isdigit():
           object_entry = ififuncs.get_object_entry()
           print 'First two characters must be \'oe\' and last four characters must be four digits'
        else:
            object_entry = args.oe
    else:
        object_entry = ififuncs.get_object_entry()
    ififuncs.generate_log(
        log_name_source,
        'EVENT = agentName=%s' % user
    )
    source_uuid_check = ififuncs.check_for_uuid(args)
    if source_uuid_check == False:
        source_uuid = ififuncs.get_source_uuid()
    else: source_uuid = source_uuid_check
    ififuncs.generate_log(
        log_name_source,
        'Relationship, derivation, has source=%s' % source_uuid
    )
    video_files = args.i
    concat_file = ififuncs.get_temp_concat('concat_stuff')
    ififuncs.generate_log(
        log_name_source,
        'concatenation file=%s' % concat_file)
    if args.r:
        video_files = recursive_file_list(video_files)
    video_files = ififuncs.sanitise_filenames(video_files)
    for source_files in video_files:
        ififuncs.generate_log(
            log_name_source,
            'source_files = %s' % source_files)
    make_chapters(video_files)
    ififuncs.concat_textfile(video_files, concat_file)
    ififuncs.generate_log(
        log_name_source,
        'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
    source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid)
    output_file = os.path.join(args.o, '%s.mkv' % uuid)
    ififuncs.generate_log(
        log_name_source,
        'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
    ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
    validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
    validation_env_dict = ififuncs.set_environment(validation_logfile)
    output_bitstream_md5 = subprocess.check_output([
        'ffmpeg', '-report',
        '-i', output_file,
        '-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
    ], env=validation_env_dict).rstrip()
    ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
    if source_bitstream_md5 == output_bitstream_md5:
        print 'process appears to be lossless'
        print source_bitstream_md5, output_bitstream_md5
        ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, eventOutcome=pass')
    else:
        print 'something went wrong - not lossless!'
        print source_bitstream_md5,output_bitstream_md5
        ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, eventOutcome=fail')
    subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
    ififuncs.generate_log(
        log_name_source,
        'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
    ififuncs.concat_textfile(video_files, concat_file)
    with open(log_name_source, 'r') as concat_log:
        concat_lines = concat_log.readlines()
    if not args.no_sip:
        sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
        shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
        shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
        logs_dir = os.path.dirname(sipcreator_log)
        ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
        ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
        with open(sipcreator_log, 'r') as sipcreator_log_object:
            sipcreator_lines = sipcreator_log_object.readlines()
        with open(sipcreator_log, 'wb') as fo:
            for lines in concat_lines:
                fo.write(lines)
            for remaining_lines in sipcreator_lines:
                fo.write(remaining_lines)
        ififuncs.checksum_replace(sipcreator_manifest, sipcreator_log)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>concat - fixes file inputs again<commit_after>#!/usr/bin/env python
'''
Concatenates video files using FFmpeg stream copy for general use but
particularly for XDCAM workflows in the IFI Irish Film Institute.
Uses mkvpropedit to insert chapter markers for each source file.
Optionally wraps the file into a package structure with checksum manifests.

Written by Kieran O'Leary.
'''
import sys
import subprocess
import os
import argparse
import time
import shutil
import sipcreator
import ififuncs

def parse_args(args_):
    '''
    Parse command line arguments.
    '''
    parser = argparse.ArgumentParser(
        description='Concatenate video files using ffmpeg stream copy'
        ' Written by Kieran O\'Leary.'
    )
    parser.add_argument(
        '-i', nargs='+',
        help='full path of input directory', required=True
    )
    parser.add_argument(
        '-o', '-output',
        help='full path of output directory', required=True
    )
    parser.add_argument(
        '-r', '-recursive',
        help='recursively process all files in subdirectories. This could be potentially a disaster - so use with caution or with XDCAM', action='store_true'
    )
    parser.add_argument(
        '--no-sip',
        help='Do not run sipcreator.py on the resulting file.', action='store_true'
    )
    parser.add_argument(
        '-user',
        help='Declare who you are. If this is not set, you will be prompted.')
    parser.add_argument(
        '-oe',
        help='Enter the Object Entry number for the representation.SIP will be placed in a folder with this name.'
    )
    parsed_args = parser.parse_args(args_)
    return parsed_args

def ffmpeg_concat(concat_file, args, uuid):
    '''
    Launch the actual ffmpeg concatenation command.
    '''
    fmd5_logfile = os.path.join(args.o, '%s_concat.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
    fmd5_env_dict = ififuncs.set_environment(fmd5_logfile)
    print fmd5_logfile
    print fmd5_env_dict
    cmd = [
        'ffmpeg', '-report', '-f', 'concat', '-safe', '0',
        '-i', concat_file,
        '-c', 'copy', '-map', '0:v', '-map', '0:a?',
        os.path.join(args.o, '%s.mkv' % uuid),
        '-f', 'md5', '-map', '0:v', '-map', '0:a?','-c', 'copy',  '-'
    ]
    print cmd
    source_bitstream_md5 = subprocess.check_output(
        cmd, env=fmd5_env_dict
    )
    return source_bitstream_md5.rstrip(), fmd5_logfile.replace('\\\\', '\\').replace('\:', ':')

def recursive_file_list(video_files):
    '''
    Recursively searches through directories for AV files and adds to a list.
    '''
    # check if all inputs are actually directories
    recursive_list = []
    for directory in video_files:
        if not os.path.isdir(directory):
            print 'You have selected the recursive option, but not all of your inputs are directories.'
            sys.exit()
        else:
            for root, _, filenames in os.walk(directory):
                for filename in filenames:
                    if filename.endswith(('.MP4', '.mp4', '.mov', '.mkv', '.mxf')):
                        recursive_list.append(os.path.join(root, filename))
    print recursive_list
    return recursive_list

def make_chapters(video_files):
    '''
    Use mkvpropedit to insert chapter markers for each source video.
    Each chapter's name will reflect the source filename of each clip.
    '''
    millis = ififuncs.get_milliseconds(video_files[0])
    timestamp = ififuncs.convert_millis(int(millis))
    chapter_list = [['00:00:00.000', os.path.basename(video_files[0])], [timestamp, os.path.basename(video_files[1])]]
    count = 2
    for video in video_files[1:]:
        millis += ififuncs.get_milliseconds(video)
        timestamp = ififuncs.convert_millis(int(millis))
        if count == len(video_files):
            continue
        else:
            chapter_list.append([timestamp, os.path.basename(video_files[count])])
        count+=1
    chapter_counter = 1
    # uh use a real path/filename.
    with open('chapters.txt', 'wb') as fo:
        for i in chapter_list:
            fo.write('CHAPTER%s=%s\nCHAPTER%sNAME=%s\n' % (str(chapter_counter).zfill(2), i[0], str(chapter_counter).zfill(2), i[1]))
            chapter_counter += 1

def main(args_):
    '''
    Launches the functions that prepare and execute the concatenation.
    '''
    uuid = ififuncs.create_uuid()
    args = parse_args(args_)
    print args
    log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
    ififuncs.generate_log(log_name_source, 'concat.py started.')
    ififuncs.generate_log(
        log_name_source,
        'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
    ififuncs.generate_log(
        log_name_source,
        'Command line arguments: %s' % args
    )
    if args.user:
        user = args.user
    else:
        user = ififuncs.get_user()
    if args.oe:
        if args.oe[:2] != 'oe':
            print 'First two characters must be \'oe\' and last four characters must be four digits'
            object_entry = ififuncs.get_object_entry()
        elif len(args.oe[2:]) != 4:
            print 'First two characters must be \'oe\' and last four characters must be four digits'
            object_entry = ififuncs.get_object_entry()
        elif not args.oe[2:].isdigit():
           object_entry = ififuncs.get_object_entry()
           print 'First two characters must be \'oe\' and last four characters must be four digits'
        else:
            object_entry = args.oe
    else:
        object_entry = ififuncs.get_object_entry()
    ififuncs.generate_log(
        log_name_source,
        'EVENT = agentName=%s' % user
    )
    source_uuid_check = ''
    if os.path.isfile(args.i[0]):
        source_uuid = ififuncs.get_source_uuid()
    elif os.path.isdir(args.i[0]):
        source_uuid_check = ififuncs.check_for_uuid(args)
    if source_uuid_check == False:
        source_uuid = ififuncs.get_source_uuid()
    else: source_uuid = source_uuid_check
    ififuncs.generate_log(
        log_name_source,
        'Relationship, derivation, has source=%s' % source_uuid
    )
    video_files = args.i
    concat_file = ififuncs.get_temp_concat('concat_stuff')
    ififuncs.generate_log(
        log_name_source,
        'concatenation file=%s' % concat_file)
    if args.r:
        video_files = recursive_file_list(video_files)
    video_files = ififuncs.sanitise_filenames(video_files)
    for source_files in video_files:
        ififuncs.generate_log(
            log_name_source,
            'source_files = %s' % source_files)
    make_chapters(video_files)
    ififuncs.concat_textfile(video_files, concat_file)
    ififuncs.generate_log(
        log_name_source,
        'EVENT = Concatenation, status=started, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
    source_bitstream_md5, fmd5_logfile = ffmpeg_concat(concat_file, args, uuid)
    output_file = os.path.join(args.o, '%s.mkv' % uuid)
    ififuncs.generate_log(
        log_name_source,
        'EVENT = Concatenation, status=finished, eventType=Creation, agentName=ffmpeg, eventDetail=Source media concatenated into a single file output=%s' % os.path.join(args.o, '%s.mkv' % uuid))
    ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, status=started, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
    validation_logfile = os.path.join(args.o, '%s_validation.log' % uuid).replace('\\', '\\\\').replace(':', '\:')
    validation_env_dict = ififuncs.set_environment(validation_logfile)
    output_bitstream_md5 = subprocess.check_output([
        'ffmpeg', '-report',
        '-i', output_file,
        '-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
    ], env=validation_env_dict).rstrip()
    ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, status=finished, eventType=messageDigestCalculation, agentName=ffmpeg, eventDetail=MD5s of AV streams of output file generated for validation')
    if source_bitstream_md5 == output_bitstream_md5:
        print 'process appears to be lossless'
        print source_bitstream_md5, output_bitstream_md5
        ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, eventOutcome=pass')
    else:
        print 'something went wrong - not lossless!'
        print source_bitstream_md5,output_bitstream_md5
        ififuncs.generate_log(
        log_name_source,
        'EVENT = losslessness verification, eventOutcome=fail')
    subprocess.call(['mkvpropedit', output_file, '-c', 'chapters.txt'])
    ififuncs.generate_log(
        log_name_source,
        'EVENT = eventType=modification, agentName=mkvpropedit, eventDetail=Chapters added to file detailing start point of source clips.')
    ififuncs.concat_textfile(video_files, concat_file)
    with open(log_name_source, 'r') as concat_log:
        concat_lines = concat_log.readlines()
    if not args.no_sip:
        sipcreator_log, sipcreator_manifest = sipcreator.main(['-i', output_file, '-u', uuid, '-oe', object_entry, '-user', user, '-o', args.o])
        shutil.move(fmd5_logfile, os.path.dirname(sipcreator_log))
        shutil.move(validation_logfile.replace('\\\\', '\\').replace('\:', ':'), os.path.dirname(sipcreator_log))
        logs_dir = os.path.dirname(sipcreator_log)
        ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir, os.path.basename(fmd5_logfile)))
        ififuncs.manifest_update(sipcreator_manifest, os.path.join(logs_dir,(os.path.basename(validation_logfile.replace('\\\\', '\\').replace('\:', ':')))))
        with open(sipcreator_log, 'r') as sipcreator_log_object:
            sipcreator_lines = sipcreator_log_object.readlines()
        with open(sipcreator_log, 'wb') as fo:
            for lines in concat_lines:
                fo.write(lines)
            for remaining_lines in sipcreator_lines:
                fo.write(remaining_lines)
        ififuncs.checksum_replace(sipcreator_manifest, sipcreator_log)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>