<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2013 OpenStack, LLC
# Copyright 2013 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Provides methods needed by installation script for OpenStack development
virtual environments.

Synced in from openstack-common
"""

import os
import subprocess
import sys

possible_topdir = os.getcwd()
if os.path.exists(os.path.join(possible_topdir, "nova",
                               "__init__.py")):
    sys.path.insert(0, possible_topdir)

from oslo.config import cfg

class InstallVenv(object):

    def __init__(self, root, venv, pip_requires, test_requires, py_version,
                 project):
        self.root = root
        self.venv = venv
        self.pip_requires = pip_requires
        self.test_requires = test_requires
        self.py_version = py_version
        self.project = project

    def die(self, message, *args):
        print >> sys.stderr, message % args
        sys.exit(1)

    def check_python_version(self):
        if sys.version_info < (2, 6):
            self.die("Need Python Version >= 2.6")

    def run_command_with_code(self, cmd, redirect_output=True,
                              check_exit_code=True):
        """Runs a command in an out-of-process shell.

        Returns the output of that command. Working directory is ROOT.
        """
        if redirect_output:
            stdout = subprocess.PIPE
        else:
            stdout = None

        proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
        output = proc.communicate()[0]
        if check_exit_code and proc.returncode != 0:
            self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
        return (output, proc.returncode)

    def run_command(self, cmd, redirect_output=True, check_exit_code=True):
        return self.run_command_with_code(cmd, redirect_output,
                                          check_exit_code)[0]

    def get_distro(self):
        if (os.path.exists('/etc/fedora-release') or
                os.path.exists('/etc/redhat-release')):
            return Fedora(self.root, self.venv, self.pip_requires,
                          self.test_requires, self.py_version, self.project)
        else:
            return Distro(self.root, self.venv, self.pip_requires,
                          self.test_requires, self.py_version, self.project)

    def check_dependencies(self):
        self.get_distro().install_virtualenv()

    def create_virtualenv(self, no_site_packages=True):
        """Creates the virtual environment and installs PIP.

        Creates the virtual environment and installs PIP only into the
        virtual environment.
        """
        if not os.path.isdir(self.venv):
            print 'Creating venv...',
            if no_site_packages:
                self.run_command(['virtualenv', '-q', '--no-site-packages',
                                 self.venv])
            else:
                self.run_command(['virtualenv', '-q', self.venv])
            print 'done.'
            print 'Installing pip in virtualenv...',
            if not self.run_command(['tools/with_venv.sh', 'easy_install',
                                    'pip>1.0']).strip():
                self.die("Failed to install pip.")
            print 'done.'
        else:
            print "venv already exists..."
            pass

    def pip_install(self, *args):
        self.run_command(['tools/with_venv.sh',
                         'pip', 'install', '--upgrade'] + list(args),
                         redirect_output=False)

    def install_dependencies(self):
        print 'Installing dependencies with pip (this can take a while)...'

        # First things first, make sure our venv has the latest pip and
        # distribute.
        # NOTE: we keep pip at version 1.1 since the most recent version causes
        # the .venv creation to fail. See:
        # https://bugs.launchpad.net/nova/+bug/1047120
        self.pip_install('pip==1.1')
        self.pip_install('distribute')

        # Install greenlet by hand - just listing it in the requires file does
        # not
        # get it installed in the right order
        self.pip_install('greenlet')

        self.pip_install('-r', self.pip_requires)
        self.pip_install('-r', self.test_requires)

    def post_process(self):
        self.get_distro().post_process()

    def parse_args(self, argv):
        """Parses command-line arguments."""
        cli_opts = [
            cfg.BoolOpt('no-site-packages',
                        default=False,
                        short='n',
                        help="Do not inherit packages from global Python"
                             "install"),
        ]
        CLI = cfg.ConfigOpts()
        CLI.register_cli_opts(cli_opts)
        CLI(argv[1:])
        return CLI

class Distro(InstallVenv):

    def check_cmd(self, cmd):
        return bool(self.run_command(['which', cmd],
                    check_exit_code=False).strip())

    def install_virtualenv(self):
        if self.check_cmd('virtualenv'):
            return

        if self.check_cmd('easy_install'):
            print 'Installing virtualenv via easy_install...',
            if self.run_command(['easy_install', 'virtualenv']):
                print 'Succeeded'
                return
            else:
                print 'Failed'

        self.die('ERROR: virtualenv not found.\n\n%s development'
                 ' requires virtualenv, please install it using your'
                 ' favorite package management tool' % self.project)

    def post_process(self):
        """Any distribution-specific post-processing gets done here.

        In particular, this is useful for applying patches to code inside
        the venv.
        """
        pass

class Fedora(Distro):
    """This covers all Fedora-based distributions.

    Includes: Fedora, RHEL, CentOS, Scientific Linux
    """

    def check_pkg(self, pkg):
        return self.run_command_with_code(['rpm', '-q', pkg],
                                          check_exit_code=False)[1] == 0

    def yum_install(self, pkg, **kwargs):
        print "Attempting to install '%s' via yum" % pkg
        self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)

    def apply_patch(self, originalfile, patchfile):
        self.run_command(['patch', originalfile, patchfile])

    def install_virtualenv(self):
        if self.check_cmd('virtualenv'):
            return

        if not self.check_pkg('python-virtualenv'):
            self.yum_install('python-virtualenv', check_exit_code=False)

        super(Fedora, self).install_virtualenv()

    def post_process(self):
        """Workaround for a bug in eventlet.

        This currently affects RHEL6.1, but the fix can safely be
        applied to all RHEL and Fedora distributions.

        This can be removed when the fix is applied upstream.

        Nova: https://bugs.launchpad.net/nova/+bug/884915
        Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
        """

        # Install "patch" program if it's not there
        if not self.check_pkg('patch'):
            self.yum_install('patch')

        # Apply the eventlet patch
        self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
                                      'site-packages',
                                      'eventlet/green/subprocess.py'),
                         'contrib/redhat-eventlet.patch')
<commit_msg>Sync latest install_venv_common.py<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2013 OpenStack, LLC
# Copyright 2013 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Provides methods needed by installation script for OpenStack development
virtual environments.

Synced in from openstack-common
"""

import argparse
import os
import subprocess
import sys

class InstallVenv(object):

    def __init__(self, root, venv, pip_requires, test_requires, py_version,
                 project):
        self.root = root
        self.venv = venv
        self.pip_requires = pip_requires
        self.test_requires = test_requires
        self.py_version = py_version
        self.project = project

    def die(self, message, *args):
        print >> sys.stderr, message % args
        sys.exit(1)

    def check_python_version(self):
        if sys.version_info < (2, 6):
            self.die("Need Python Version >= 2.6")

    def run_command_with_code(self, cmd, redirect_output=True,
                              check_exit_code=True):
        """Runs a command in an out-of-process shell.

        Returns the output of that command. Working directory is self.root.
        """
        if redirect_output:
            stdout = subprocess.PIPE
        else:
            stdout = None

        proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
        output = proc.communicate()[0]
        if check_exit_code and proc.returncode != 0:
            self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
        return (output, proc.returncode)

    def run_command(self, cmd, redirect_output=True, check_exit_code=True):
        return self.run_command_with_code(cmd, redirect_output,
                                          check_exit_code)[0]

    def get_distro(self):
        if (os.path.exists('/etc/fedora-release') or
                os.path.exists('/etc/redhat-release')):
            return Fedora(self.root, self.venv, self.pip_requires,
                          self.test_requires, self.py_version, self.project)
        else:
            return Distro(self.root, self.venv, self.pip_requires,
                          self.test_requires, self.py_version, self.project)

    def check_dependencies(self):
        self.get_distro().install_virtualenv()

    def create_virtualenv(self, no_site_packages=True):
        """Creates the virtual environment and installs PIP.

        Creates the virtual environment and installs PIP only into the
        virtual environment.
        """
        if not os.path.isdir(self.venv):
            print 'Creating venv...',
            if no_site_packages:
                self.run_command(['virtualenv', '-q', '--no-site-packages',
                                 self.venv])
            else:
                self.run_command(['virtualenv', '-q', self.venv])
            print 'done.'
            print 'Installing pip in venv...',
            if not self.run_command(['tools/with_venv.sh', 'easy_install',
                                    'pip>1.0']).strip():
                self.die("Failed to install pip.")
            print 'done.'
        else:
            print "venv already exists..."
            pass

    def pip_install(self, *args):
        self.run_command(['tools/with_venv.sh',
                         'pip', 'install', '--upgrade'] + list(args),
                         redirect_output=False)

    def install_dependencies(self):
        print 'Installing dependencies with pip (this can take a while)...'

        # First things first, make sure our venv has the latest pip and
        # distribute.
        # NOTE: we keep pip at version 1.1 since the most recent version causes
        # the .venv creation to fail. See:
        # https://bugs.launchpad.net/nova/+bug/1047120
        self.pip_install('pip==1.1')
        self.pip_install('distribute')

        # Install greenlet by hand - just listing it in the requires file does
        # not
        # get it installed in the right order
        self.pip_install('greenlet')

        self.pip_install('-r', self.pip_requires)
        self.pip_install('-r', self.test_requires)

    def post_process(self):
        self.get_distro().post_process()

    def parse_args(self, argv):
        """Parses command-line arguments."""
        parser = argparse.ArgumentParser()
        parser.add_argument('-n', '--no-site-packages',
                            action='store_true',
                            help="Do not inherit packages from global Python "
                                 "install")
        return parser.parse_args(argv[1:])

class Distro(InstallVenv):

    def check_cmd(self, cmd):
        return bool(self.run_command(['which', cmd],
                    check_exit_code=False).strip())

    def install_virtualenv(self):
        if self.check_cmd('virtualenv'):
            return

        if self.check_cmd('easy_install'):
            print 'Installing virtualenv via easy_install...',
            if self.run_command(['easy_install', 'virtualenv']):
                print 'Succeeded'
                return
            else:
                print 'Failed'

        self.die('ERROR: virtualenv not found.\n\n%s development'
                 ' requires virtualenv, please install it using your'
                 ' favorite package management tool' % self.project)

    def post_process(self):
        """Any distribution-specific post-processing gets done here.

        In particular, this is useful for applying patches to code inside
        the venv.
        """
        pass

class Fedora(Distro):
    """This covers all Fedora-based distributions.

    Includes: Fedora, RHEL, CentOS, Scientific Linux
    """

    def check_pkg(self, pkg):
        return self.run_command_with_code(['rpm', '-q', pkg],
                                          check_exit_code=False)[1] == 0

    def yum_install(self, pkg, **kwargs):
        print "Attempting to install '%s' via yum" % pkg
        self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)

    def apply_patch(self, originalfile, patchfile):
        self.run_command(['patch', originalfile, patchfile])

    def install_virtualenv(self):
        if self.check_cmd('virtualenv'):
            return

        if not self.check_pkg('python-virtualenv'):
            self.yum_install('python-virtualenv', check_exit_code=False)

        super(Fedora, self).install_virtualenv()

    def post_process(self):
        """Workaround for a bug in eventlet.

        This currently affects RHEL6.1, but the fix can safely be
        applied to all RHEL and Fedora distributions.

        This can be removed when the fix is applied upstream.

        Nova: https://bugs.launchpad.net/nova/+bug/884915
        Upstream: https://bitbucket.org/which_linden/eventlet/issue/89
        """

        # Install "patch" program if it's not there
        if not self.check_pkg('patch'):
            self.yum_install('patch')

        # Apply the eventlet patch
        self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
                                      'site-packages',
                                      'eventlet/green/subprocess.py'),
                         'contrib/redhat-eventlet.patch')
<|endoftext|>


<commit_before>import asyncore
import logging
import optparse
import shlex
import socket
import sys
import time

import hiredis

__all__ = ["Redis", "DebugRedis"]

try:
    NullHandler = logging.NullHandler
except AttributeError:
    class NullHandler(logging.Handler):
        def emit(self, record): pass

log = logging.getLogger(__name__)
log.addHandler(NullHandler())

def wirecmd(command, args, separator="\r\n"):
    arglen = 1 + len(args)
    parts = ["*%d" % arglen]
    for arg in (command,) + args:
        arg = str(arg)
        parts.extend(["$%d" % len(arg), arg])
    parts.append('')

    return separator.join(parts)

def fmtcmd(command, args, separator=' '):
    parts = ["%s"]
    parts.extend("%r" for arg in args)
    return ' '.join(parts)

def logcmd(name, command, args, log=None, level=logging.DEBUG):
    parts = ["%s"]
    parts.extend("%r" for arg in args)

    if name is not None:
        log = logging.getLogger(name)
    log.log(level, fmtcmd(command, args), command, *args)

class Redis(asyncore.dispatcher):
    terminator = "\r\n"

    def __init__(self, sock=None, map=None):
        asyncore.dispatcher.__init__(self, sock=sock, map=map)
        self.callbacks = []
        self.buffer = ''
        self.reader = hiredis.Reader()

    def connect(self, host="localhost", port=6379, db=0, callback=None, data=None):
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        asyncore.dispatcher.connect(self, (host, port))
        self.socket.setblocking(0)
        self.set_socket(self.socket, self._map)
        log.debug("connected to %s:%d/%d", host, port, db)
        self.callbacks = [("CONNECT", (), callback, data)]

    def do(self, callback, data, command, *args):
        self.log_send(command, args)
        self.buffer += wirecmd(command, args, separator=self.terminator)
        self.callbacks.insert(1, (command, args, callback, data))

    def log(self, message):
        pass

    def log_info(self, message, type=None):
        pass

    def log_send(self, command, args):
        pass

    def log_recv(self, reply):
        pass

    def handle_connect(self): pass

    def handle_close(self):
        self.close()

    def handle_write(self):
        sent = self.send(self.buffer)
        self.buffer = self.buffer[sent:]

    def handle_read(self):
        self.reader.feed(self.recv(8192))

        while True:
            try:
                reply = self.reader.gets()
            except hiredis.ProtocolError:
                self.close()
                raise
            if reply is False:
                return

            self.log_recv(reply)
            command, args, callback, data = self.callbacks.pop()
            if callback is not None:
                callback(command, args, data, reply)

class DebugRedis(Redis):

    def log(self, message):
        log.debug(message)

    def log_info(self, message, type=None):
        log.debug(message)

    def log_send(self, command, args):
        logcmd("%s.client.tx" % __name__, command, args)

    def log_recv(self, reply):
        logging.getLogger("%s.client.rx" % __name__).debug("%r", reply)

def parseargs(argv):
    """Parse command line arguments.

    Returns a tuple (*opts*, *args*), where *opts* is an
    :class:`optparse.Values` instance and *args* is the list of arguments left
    over after processing.

    :param argv: a list of command line arguments, usually :data:`sys.argv`.
    """
    prog = argv[0]
    usage = "[options] <address>"
    parser = optparse.OptionParser(prog=prog, usage=usage)
    parser.allow_interspersed_args = False

    defaults = {
        "quiet": 0,
        "silent": False,
        "verbose": 0,
    }

    # Global options.
    parser.add_option("-q", "--quiet", dest="quiet",
        default=defaults["quiet"], action="count",
        help="decrease the logging verbosity")
    parser.add_option("-s", "--silent", dest="silent",
        default=defaults["silent"], action="store_true",
        help="silence the logger")
    parser.add_option("-v", "--verbose", dest="verbose",
        default=defaults["verbose"], action="count",
        help="increase the logging verbosity")

    (opts, args) = parser.parse_args(args=argv[1:])
    return (opts, args)

def main(argv, stdin=None, stdout=None, stderr=None):
    """Main entry point.

    Returns a value that can be understood by :func:`sys.exit`.

    :param argv: a list of command line arguments, usually :data:`sys.argv`.
    :param out: stream to write messages; :data:`sys.stdout` if None.
    :param err: stream to write error messages; :data:`sys.stderr` if None.
    """
    if stdin is None: # pragma: nocover
        stdin = sys.stdin
    if stdout is None: # pragma: nocover
        stdout = sys.stdout
    if stderr is None: # pragma: nocover
        stderr = sys.stderr

    (opts, args) = parseargs(argv)
    level = logging.WARNING - ((opts.verbose - opts.quiet) * 10)
    if opts.silent:
        level = logging.CRITICAL + 1
    level = max(1, level)

    format = "%(name)s %(message)s"
    handler = logging.StreamHandler(stderr)
    handler.setFormatter(logging.Formatter(format))
    log.addHandler(handler)
    log.setLevel(level)

    db = Redis()
    db.connect()

    def cb_log(command, args, data, reply):
        (start,) = data
        seconds = time.time() - start
        cmd = fmtcmd(command, args) % ((command,) + args)
        log.debug("Ran %s in %g seconds", cmd, seconds)
        log.debug("Received: %r", reply)

    for line in stdin:
        splitted = shlex.split(line)
        command = splitted[0]
        args = splitted[1:]
        db.do(cb_log, (time.time(),), command, *args)

    asyncore.loop()

def run():
    try:
        ret = main(sys.argv, sys.stdin, sys.stdout, sys.stderr)
    except KeyboardInterrupt:
        ret = None

    sys.exit(ret)

if __name__ == "__main__": # pragma: nocover
    run()
<commit_msg>Respect separator param in fmtcmd().<commit_after>import asyncore
import logging
import optparse
import shlex
import socket
import sys
import time

import hiredis

__all__ = ["Redis", "DebugRedis"]

try:
    NullHandler = logging.NullHandler
except AttributeError:
    class NullHandler(logging.Handler):
        def emit(self, record): pass

log = logging.getLogger(__name__)
log.addHandler(NullHandler())

def wirecmd(command, args, separator="\r\n"):
    arglen = 1 + len(args)
    parts = ["*%d" % arglen]
    for arg in (command,) + args:
        arg = str(arg)
        parts.extend(["$%d" % len(arg), arg])
    parts.append('')

    return separator.join(parts)

def fmtcmd(command, args, separator=' '):
    parts = ["%s"]
    parts.extend("%r" for arg in args)
    return separator.join(parts)

def logcmd(name, command, args, log=None, level=logging.DEBUG):
    parts = ["%s"]
    parts.extend("%r" for arg in args)

    if name is not None:
        log = logging.getLogger(name)
    log.log(level, fmtcmd(command, args), command, *args)

class Redis(asyncore.dispatcher):
    terminator = "\r\n"

    def __init__(self, sock=None, map=None):
        asyncore.dispatcher.__init__(self, sock=sock, map=map)
        self.callbacks = []
        self.buffer = ''
        self.reader = hiredis.Reader()

    def connect(self, host="localhost", port=6379, db=0, callback=None, data=None):
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        asyncore.dispatcher.connect(self, (host, port))
        self.socket.setblocking(0)
        self.set_socket(self.socket, self._map)
        log.debug("connected to %s:%d/%d", host, port, db)
        self.callbacks = [("CONNECT", (), callback, data)]

    def do(self, callback, data, command, *args):
        self.log_send(command, args)
        self.buffer += wirecmd(command, args, separator=self.terminator)
        self.callbacks.insert(1, (command, args, callback, data))

    def log(self, message):
        pass

    def log_info(self, message, type=None):
        pass

    def log_send(self, command, args):
        pass

    def log_recv(self, reply):
        pass

    def handle_connect(self): pass

    def handle_close(self):
        self.close()

    def handle_write(self):
        sent = self.send(self.buffer)
        self.buffer = self.buffer[sent:]

    def handle_read(self):
        self.reader.feed(self.recv(8192))

        while True:
            try:
                reply = self.reader.gets()
            except hiredis.ProtocolError:
                self.close()
                raise
            if reply is False:
                return

            self.log_recv(reply)
            command, args, callback, data = self.callbacks.pop()
            if callback is not None:
                callback(command, args, data, reply)

class DebugRedis(Redis):

    def log(self, message):
        log.debug(message)

    def log_info(self, message, type=None):
        log.debug(message)

    def log_send(self, command, args):
        logcmd("%s.client.tx" % __name__, command, args)

    def log_recv(self, reply):
        logging.getLogger("%s.client.rx" % __name__).debug("%r", reply)

def parseargs(argv):
    """Parse command line arguments.

    Returns a tuple (*opts*, *args*), where *opts* is an
    :class:`optparse.Values` instance and *args* is the list of arguments left
    over after processing.

    :param argv: a list of command line arguments, usually :data:`sys.argv`.
    """
    prog = argv[0]
    usage = "[options] <address>"
    parser = optparse.OptionParser(prog=prog, usage=usage)
    parser.allow_interspersed_args = False

    defaults = {
        "quiet": 0,
        "silent": False,
        "verbose": 0,
    }

    # Global options.
    parser.add_option("-q", "--quiet", dest="quiet",
        default=defaults["quiet"], action="count",
        help="decrease the logging verbosity")
    parser.add_option("-s", "--silent", dest="silent",
        default=defaults["silent"], action="store_true",
        help="silence the logger")
    parser.add_option("-v", "--verbose", dest="verbose",
        default=defaults["verbose"], action="count",
        help="increase the logging verbosity")

    (opts, args) = parser.parse_args(args=argv[1:])
    return (opts, args)

def main(argv, stdin=None, stdout=None, stderr=None):
    """Main entry point.

    Returns a value that can be understood by :func:`sys.exit`.

    :param argv: a list of command line arguments, usually :data:`sys.argv`.
    :param out: stream to write messages; :data:`sys.stdout` if None.
    :param err: stream to write error messages; :data:`sys.stderr` if None.
    """
    if stdin is None: # pragma: nocover
        stdin = sys.stdin
    if stdout is None: # pragma: nocover
        stdout = sys.stdout
    if stderr is None: # pragma: nocover
        stderr = sys.stderr

    (opts, args) = parseargs(argv)
    level = logging.WARNING - ((opts.verbose - opts.quiet) * 10)
    if opts.silent:
        level = logging.CRITICAL + 1
    level = max(1, level)

    format = "%(name)s %(message)s"
    handler = logging.StreamHandler(stderr)
    handler.setFormatter(logging.Formatter(format))
    log.addHandler(handler)
    log.setLevel(level)

    db = Redis()
    db.connect()

    def cb_log(command, args, data, reply):
        (start,) = data
        seconds = time.time() - start
        cmd = fmtcmd(command, args) % ((command,) + args)
        log.debug("Ran %s in %g seconds", cmd, seconds)
        log.debug("Received: %r", reply)

    for line in stdin:
        splitted = shlex.split(line)
        command = splitted[0]
        args = splitted[1:]
        db.do(cb_log, (time.time(),), command, *args)

    asyncore.loop()

def run():
    try:
        ret = main(sys.argv, sys.stdin, sys.stdout, sys.stderr)
    except KeyboardInterrupt:
        ret = None

    sys.exit(ret)

if __name__ == "__main__": # pragma: nocover
    run()
<|endoftext|>


<commit_before>'''
Based off https://github.com/pimutils/python-vdir, which is itself based off
vdirsyncer.
'''

import os
import errno
import uuid

from atomicwrites import atomic_write

class cached_property:
    '''A read-only @property that is only evaluated once. Only usable on class
    instances' methods.
    '''
    def __init__(self, fget, doc=None):
        self.__name__ = fget.__name__
        self.__module__ = fget.__module__
        self.__doc__ = doc or fget.__doc__
        self.fget = fget

    def __get__(self, obj, cls):
        if obj is None:  # pragma: no cover
            return self
        obj.__dict__[self.__name__] = result = self.fget(obj)
        return result

def to_unicode(x, encoding='ascii'):
    if not isinstance(x, str):
        return x.decode(encoding)
    return x

def to_bytes(x, encoding='ascii'):
    if not isinstance(x, bytes):
        return x.encode(encoding)
    return x

SAFE_UID_CHARS = ('abcdefghijklmnopqrstuvwxyz'
                  'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                  '0123456789_.-+')

def _href_safe(uid, safe=SAFE_UID_CHARS):
    return not bool(set(uid) - set(safe))

def _generate_href(uid=None, safe=SAFE_UID_CHARS):
    if not uid or not _href_safe(uid, safe):
        return to_unicode(uuid.uuid4().hex)
    else:
        return uid

def get_etag_from_file(f):
    '''Get mtime-based etag from a filepath or file-like object.

    This function will flush/sync the file as much as necessary to obtain a
    correct mtime.
    '''
    if hasattr(f, 'read'):
        # assure that all internal buffers associated with this file are
        # written to disk
        f.flush()
        os.fsync(f.fileno())
        stat = os.fstat(f.fileno())
    else:
        stat = os.stat(f)

    mtime = getattr(stat, 'st_mtime_ns', None)
    if mtime is None:
        mtime = stat.st_mtime
    return '{:.9f}'.format(mtime)

class VdirError(IOError):
    def __init__(self, *args, **kwargs):
        for key, value in kwargs.items():
            if getattr(self, key, object()) is not None:  # pragma: no cover
                raise TypeError('Invalid argument: {}'.format(key))
            setattr(self, key, value)

        super(VdirError, self).__init__(*args)

class NotFoundError(VdirError):
    pass

class CollectionNotFoundError(VdirError):
    pass

class WrongEtagError(VdirError):
    pass

class AlreadyExistingError(VdirError):
    existing_href = None

class Item:
    def __init__(self, raw):
        assert isinstance(raw, str)
        self.raw = raw

    @cached_property
    def uid(self):
        uid = u''
        lines = iter(self.raw.splitlines())
        for line in lines:
            if line.startswith('UID:'):
                uid += line[4:].strip()
                break

        for line in lines:
            if not line.startswith(u' '):
                break
            uid += line[1:]

        return uid or None

def _normalize_meta_value(value):
    return to_unicode(value or u'').strip()

class VdirBase:
    item_class = Item
    default_mode = 0o750

    def __init__(self, path, fileext, encoding='utf-8'):
        if not os.path.isdir(path):
            raise CollectionNotFoundError(path)
        self.path = path
        self.encoding = encoding
        self.fileext = fileext

    @classmethod
    def discover(cls, path, **kwargs):
        try:
            collections = os.listdir(path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            return

        for collection in collections:
            collection_path = os.path.join(path, collection)
            if os.path.isdir(collection_path):
                yield cls(path=collection_path, **kwargs)

    @classmethod
    def create(cls, collection_name, **kwargs):
        kwargs = dict(kwargs)
        path = kwargs['path']

        path = os.path.join(path, collection_name)
        if not os.path.exists(path):
            os.makedirs(path, mode=cls.default_mode)
        elif not os.path.isdir(path):
            raise IOError('{} is not a directory.'.format(repr(path)))

        kwargs['path'] = path
        return kwargs

    def _get_filepath(self, href):
        return os.path.join(self.path, href)

    def _get_href(self, uid):
        return _generate_href(uid) + self.fileext

    def list(self):
        for fname in os.listdir(self.path):
            fpath = os.path.join(self.path, fname)
            if os.path.isfile(fpath) and fname.endswith(self.fileext):
                yield fname, get_etag_from_file(fpath)

    def get(self, href):
        fpath = self._get_filepath(href)
        try:
            with open(fpath, 'rb') as f:
                return (Item(f.read().decode(self.encoding)),
                        get_etag_from_file(fpath))
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise NotFoundError(href)
            else:
                raise

    def upload(self, item):
        if not isinstance(item.raw, str):
            raise TypeError('item.raw must be a unicode string.')

        try:
            href = self._get_href(item.uid)
            fpath, etag = self._upload_impl(item, href)
        except OSError as e:
            if e.errno in (
                errno.ENAMETOOLONG,  # Unix
                errno.ENOENT  # Windows
            ):
                # random href instead of UID-based
                href = self._get_href(None)
                fpath, etag = self._upload_impl(item, href)
            else:
                raise

        return href, etag

    def _upload_impl(self, item, href):
        fpath = self._get_filepath(href)
        try:
            with atomic_write(fpath, mode='wb', overwrite=False) as f:
                f.write(item.raw.encode(self.encoding))
                return fpath, get_etag_from_file(f)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise AlreadyExistingError(existing_href=href)
            else:
                raise

    def update(self, href, item, etag):
        fpath = self._get_filepath(href)
        if not os.path.exists(fpath):
            raise NotFoundError(item.uid)
        actual_etag = get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)

        if not isinstance(item.raw, str):
            raise TypeError('item.raw must be a unicode string.')

        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(item.raw.encode(self.encoding))
            etag = get_etag_from_file(f)

        return etag

    def delete(self, href, etag):
        fpath = self._get_filepath(href)
        if not os.path.isfile(fpath):
            raise NotFoundError(href)
        actual_etag = get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)
        os.remove(fpath)

    def get_meta(self, key):
        fpath = os.path.join(self.path, key)
        try:
            with open(fpath, 'rb') as f:
                return f.read().decode(self.encoding) or None
        except IOError as e:
            if e.errno == errno.ENOENT:
                return None
            else:
                raise

    def set_meta(self, key, value):
        value = value or u''
        assert isinstance(value, str)
        fpath = os.path.join(self.path, key)
        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(value.encode(self.encoding))

class Color:
    def __init__(self, x):
        if not x:
            raise ValueError('Color is false-ish.')
        if not x.startswith('#'):
            raise ValueError('Color must start with a #.')
        if len(x) != 7:
            raise ValueError('Color must not have shortcuts. '
                             '#ffffff instead of #fff')
        self.raw = x.upper()

    @cached_property
    def rgb(self):
        x = self.raw

        r = x[1:3]
        g = x[3:5]
        b = x[5:8]

        if len(r) == len(g) == len(b) == 2:
            return int(r, 16), int(g, 16), int(b, 16)
        else:
            raise ValueError('Unable to parse color value: {}'
                             .format(self.value))

class ColorMixin:
    color_type = Color

    def get_color(self):
        try:
            return self.color_type(self.get_meta('color'))
        except ValueError:
            return None

    def set_color(self, value):
        self.set_meta('color', self.color_type(value).raw)

class DisplayNameMixin:
    def get_displayname(self):
        return self.get_meta('displayname')

    def set_displayname(self, value):
        self.set_meta('displayname', value)

class Vdir(VdirBase, ColorMixin, DisplayNameMixin):
    pass
<commit_msg>Also fsync() directories<commit_after>'''
Based off https://github.com/pimutils/python-vdir, which is itself based off
vdirsyncer.
'''

import os
import errno
import uuid

from atomicwrites import atomic_write

class cached_property:
    '''A read-only @property that is only evaluated once. Only usable on class
    instances' methods.
    '''
    def __init__(self, fget, doc=None):
        self.__name__ = fget.__name__
        self.__module__ = fget.__module__
        self.__doc__ = doc or fget.__doc__
        self.fget = fget

    def __get__(self, obj, cls):
        if obj is None:  # pragma: no cover
            return self
        obj.__dict__[self.__name__] = result = self.fget(obj)
        return result

def to_unicode(x, encoding='ascii'):
    if not isinstance(x, str):
        return x.decode(encoding)
    return x

def to_bytes(x, encoding='ascii'):
    if not isinstance(x, bytes):
        return x.encode(encoding)
    return x

SAFE_UID_CHARS = ('abcdefghijklmnopqrstuvwxyz'
                  'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                  '0123456789_.-+')

def _href_safe(uid, safe=SAFE_UID_CHARS):
    return not bool(set(uid) - set(safe))

def _generate_href(uid=None, safe=SAFE_UID_CHARS):
    if not uid or not _href_safe(uid, safe):
        return to_unicode(uuid.uuid4().hex)
    else:
        return uid

def get_etag_from_file(f):
    '''Get mtime-based etag from a filepath or file-like object.

    This function will flush/sync the file as much as necessary to obtain a
    correct mtime.
    '''
    # assure that all internal buffers associated with this file are
    # written to disk
    if isinstance(f, str):
        if os.path.isdir(f):
            dir_ = os.open(f, os.O_DIRECTORY)
            os.fsync(dir_)
            os.close(dir_)
            stat = os.stat(f)
        elif os.path.isfile(f):
            with open(f) as file_:
                return get_etag_from_file(file_)
    elif hasattr(f, 'read'):
        f.flush()
        os.fsync(f.fileno())
        stat = os.fstat(f.fileno())
    else:
        raise ValueError('Please debug me')

    mtime = getattr(stat, 'st_mtime_ns', None)
    if mtime is None:
        mtime = stat.st_mtime
    return '{:.9f}'.format(mtime)

class VdirError(IOError):
    def __init__(self, *args, **kwargs):
        for key, value in kwargs.items():
            if getattr(self, key, object()) is not None:  # pragma: no cover
                raise TypeError('Invalid argument: {}'.format(key))
            setattr(self, key, value)

        super(VdirError, self).__init__(*args)

class NotFoundError(VdirError):
    pass

class CollectionNotFoundError(VdirError):
    pass

class WrongEtagError(VdirError):
    pass

class AlreadyExistingError(VdirError):
    existing_href = None

class Item:
    def __init__(self, raw):
        assert isinstance(raw, str)
        self.raw = raw

    @cached_property
    def uid(self):
        uid = u''
        lines = iter(self.raw.splitlines())
        for line in lines:
            if line.startswith('UID:'):
                uid += line[4:].strip()
                break

        for line in lines:
            if not line.startswith(u' '):
                break
            uid += line[1:]

        return uid or None

def _normalize_meta_value(value):
    return to_unicode(value or u'').strip()

class VdirBase:
    item_class = Item
    default_mode = 0o750

    def __init__(self, path, fileext, encoding='utf-8'):
        if not os.path.isdir(path):
            raise CollectionNotFoundError(path)
        self.path = path
        self.encoding = encoding
        self.fileext = fileext

    @classmethod
    def discover(cls, path, **kwargs):
        try:
            collections = os.listdir(path)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
            return

        for collection in collections:
            collection_path = os.path.join(path, collection)
            if os.path.isdir(collection_path):
                yield cls(path=collection_path, **kwargs)

    @classmethod
    def create(cls, collection_name, **kwargs):
        kwargs = dict(kwargs)
        path = kwargs['path']

        path = os.path.join(path, collection_name)
        if not os.path.exists(path):
            os.makedirs(path, mode=cls.default_mode)
        elif not os.path.isdir(path):
            raise IOError('{} is not a directory.'.format(repr(path)))

        kwargs['path'] = path
        return kwargs

    def _get_filepath(self, href):
        return os.path.join(self.path, href)

    def _get_href(self, uid):
        return _generate_href(uid) + self.fileext

    def list(self):
        for fname in os.listdir(self.path):
            fpath = os.path.join(self.path, fname)
            if os.path.isfile(fpath) and fname.endswith(self.fileext):
                yield fname, get_etag_from_file(fpath)

    def get(self, href):
        fpath = self._get_filepath(href)
        try:
            with open(fpath, 'rb') as f:
                return (Item(f.read().decode(self.encoding)),
                        get_etag_from_file(fpath))
        except IOError as e:
            if e.errno == errno.ENOENT:
                raise NotFoundError(href)
            else:
                raise

    def upload(self, item):
        if not isinstance(item.raw, str):
            raise TypeError('item.raw must be a unicode string.')

        try:
            href = self._get_href(item.uid)
            fpath, etag = self._upload_impl(item, href)
        except OSError as e:
            if e.errno in (
                errno.ENAMETOOLONG,  # Unix
                errno.ENOENT  # Windows
            ):
                # random href instead of UID-based
                href = self._get_href(None)
                fpath, etag = self._upload_impl(item, href)
            else:
                raise

        return href, etag

    def _upload_impl(self, item, href):
        fpath = self._get_filepath(href)
        try:
            with atomic_write(fpath, mode='wb', overwrite=False) as f:
                f.write(item.raw.encode(self.encoding))
                return fpath, get_etag_from_file(f)
        except OSError as e:
            if e.errno == errno.EEXIST:
                raise AlreadyExistingError(existing_href=href)
            else:
                raise

    def update(self, href, item, etag):
        fpath = self._get_filepath(href)
        if not os.path.exists(fpath):
            raise NotFoundError(item.uid)
        actual_etag = get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)

        if not isinstance(item.raw, str):
            raise TypeError('item.raw must be a unicode string.')

        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(item.raw.encode(self.encoding))
            etag = get_etag_from_file(f)

        return etag

    def delete(self, href, etag):
        fpath = self._get_filepath(href)
        if not os.path.isfile(fpath):
            raise NotFoundError(href)
        actual_etag = get_etag_from_file(fpath)
        if etag != actual_etag:
            raise WrongEtagError(etag, actual_etag)
        os.remove(fpath)

    def get_meta(self, key):
        fpath = os.path.join(self.path, key)
        try:
            with open(fpath, 'rb') as f:
                return f.read().decode(self.encoding) or None
        except IOError as e:
            if e.errno == errno.ENOENT:
                return None
            else:
                raise

    def set_meta(self, key, value):
        value = value or u''
        assert isinstance(value, str)
        fpath = os.path.join(self.path, key)
        with atomic_write(fpath, mode='wb', overwrite=True) as f:
            f.write(value.encode(self.encoding))

class Color:
    def __init__(self, x):
        if not x:
            raise ValueError('Color is false-ish.')
        if not x.startswith('#'):
            raise ValueError('Color must start with a #.')
        if len(x) != 7:
            raise ValueError('Color must not have shortcuts. '
                             '#ffffff instead of #fff')
        self.raw = x.upper()

    @cached_property
    def rgb(self):
        x = self.raw

        r = x[1:3]
        g = x[3:5]
        b = x[5:8]

        if len(r) == len(g) == len(b) == 2:
            return int(r, 16), int(g, 16), int(b, 16)
        else:
            raise ValueError('Unable to parse color value: {}'
                             .format(self.value))

class ColorMixin:
    color_type = Color

    def get_color(self):
        try:
            return self.color_type(self.get_meta('color'))
        except ValueError:
            return None

    def set_color(self, value):
        self.set_meta('color', self.color_type(value).raw)

class DisplayNameMixin:
    def get_displayname(self):
        return self.get_meta('displayname')

    def set_displayname(self, value):
        self.set_meta('displayname', value)

class Vdir(VdirBase, ColorMixin, DisplayNameMixin):
    pass
<|endoftext|>


<commit_before>"""
Implements Lilliefors corrected Kolmogorov-Smirnov tests for normal and
exponential distributions.

`kstest_fit` is provided as a top-level function to access both tests.
`kstest_normal` and `kstest_exponential` are provided as convenience functions
with the appropriate test as the default.
`lilliefors` is provided as an alias for `kstest_fit`.

Created on Sat Oct 01 13:16:49 2011

Author: Josef Perktold
License: BSD-3

pvalues for Lilliefors test are based on formula and table in

An Analytic Approximation to the Distribution of Lilliefors's Test Statistic
for Normality
Author(s): Gerard E. Dallal and Leland WilkinsonSource: The American
Statistician, Vol. 40, No. 4 (Nov., 1986), pp. 294-296
Published by: American Statistical Association
Stable URL: http://www.jstor.org/stable/2684607 .

On the Kolmogorov-Smirnov Test for Normality with Mean and Variance Unknown
Hubert W. Lilliefors
Journal of the American Statistical Association, Vol. 62, No. 318.
(Jun., 1967), pp. 399-402.

---

Updated 2017-07-23
Jacob C. Kimmel

Ref:
Lilliefors, H.W.
On the Kolmogorov-Smirnov test for the exponential distribution with mean
unknown. Journal of the American Statistical Association, Vol 64, No. 325.
(1969), pp. 387–389.
"""
from functools import partial

import numpy as np
from scipy import stats

from statsmodels.tools.validation import string_like
from ._lilliefors_critical_values import (critical_values,
                                          asymp_critical_values,
                                          PERCENTILES)
from .tabledist import TableDist

def _make_asymptotic_function(params):
    """
    Generates an asymptotic distribution callable from a param matrix

    Polynomial is a[0] * x**(-1/2) + a[1] * x**(-1) + a[2] * x**(-3/2)

    Parameters
    ----------
    params : ndarray
        Array with shape (nalpha, 3) where nalpha is the number of
        significance levels
    """

    def f(n):
        poly = np.array([1, np.log(n), np.log(n) ** 2])
        return np.exp(poly.dot(params.T))

    return f

def ksstat(x, cdf, alternative='two_sided', args=()):
    """
    Calculate statistic for the Kolmogorov-Smirnov test for goodness of fit

    This calculates the test statistic for a test of the distribution G(x) of
    an observed variable against a given distribution F(x). Under the null
    hypothesis the two distributions are identical, G(x)=F(x). The
    alternative hypothesis can be either 'two_sided' (default), 'less'
    or 'greater'. The KS test is only valid for continuous distributions.

    Parameters
    ----------
    x : array_like, 1d
        array of observations
    cdf : str or callable
        string: name of a distribution in scipy.stats
        callable: function to evaluate cdf
    alternative : 'two_sided' (default), 'less' or 'greater'
        defines the alternative hypothesis (see explanation)
    args : tuple, sequence
        distribution parameters for call to cdf

    Returns
    -------
    D : float
        KS test statistic, either D, D+ or D-

    See Also
    --------
    scipy.stats.kstest

    Notes
    -----

    In the one-sided test, the alternative is that the empirical
    cumulative distribution function of the random variable is "less"
    or "greater" than the cumulative distribution function F(x) of the
    hypothesis, G(x)<=F(x), resp. G(x)>=F(x).

    In contrast to scipy.stats.kstest, this function only calculates the
    statistic which can be used either as distance measure or to implement
    case specific p-values.
    """
    nobs = float(len(x))

    if isinstance(cdf, str):
        cdf = getattr(stats.distributions, cdf).cdf
    elif hasattr(cdf, 'cdf'):
        cdf = getattr(cdf, 'cdf')

    x = np.sort(x)
    cdfvals = cdf(x, *args)

    d_plus = (np.arange(1.0, nobs + 1) / nobs - cdfvals).max()
    d_min = (cdfvals - np.arange(0.0, nobs) / nobs).max()
    if alternative == 'greater':
        return d_plus
    elif alternative == 'less':
        return d_min

    return np.max([d_plus, d_min])

def get_lilliefors_table(dist='norm'):
    """
    Generates tables for significance levels of Lilliefors test statistics

    Tables for available normal and exponential distribution testing,
    as specified in Lilliefors references above

    Parameters
    ----------
    dist : str
        distribution being tested in set {'norm', 'exp'}.

    Returns
    -------
    lf : TableDist object.
        table of critical values
    """
    # function just to keep things together
    # for this test alpha is sf probability, i.e. right tail probability

    alpha = 1 - np.array(PERCENTILES) / 100.0
    alpha = alpha[::-1]
    dist = 'normal' if dist == 'norm' else dist
    if dist not in critical_values:
        raise ValueError("Invalid dist parameter. Must be 'norm' or 'exp'")
    cv_data = critical_values[dist]
    acv_data = asymp_critical_values[dist]

    size = np.array(sorted(cv_data), dtype=float)
    crit_lf = np.array([cv_data[key] for key in sorted(cv_data)])
    crit_lf = crit_lf[:, ::-1]

    asym_params = np.array([acv_data[key] for key in sorted(acv_data)])
    asymp_fn = _make_asymptotic_function((asym_params[::-1]))

    lf = TableDist(alpha, size, crit_lf, asymptotic=asymp_fn)
    return lf

lilliefors_table_norm = get_lilliefors_table(dist='norm')
lilliefors_table_expon = get_lilliefors_table(dist='exp')

def pval_lf(d_max, n):
    """
    Approximate pvalues for Lilliefors test

    This is only valid for pvalues smaller than 0.1 which is not checked in
    this function.

    Parameters
    ----------
    d_max : array_like
        two-sided Kolmogorov-Smirnov test statistic
    n : int or float
        sample size

    Returns
    -------
    p-value : float or ndarray
        pvalue according to approximation formula of Dallal and Wilkinson.

    Notes
    -----
    This is mainly a helper function where the calling code should dispatch
    on bound violations. Therefore it does not check whether the pvalue is in
    the valid range.

    Precision for the pvalues is around 2 to 3 decimals. This approximation is
    also used by other statistical packages (e.g. R:fBasics) but might not be
    the most precise available.

    References
    ----------
    DallalWilkinson1986
    """
    # todo: check boundaries, valid range for n and Dmax
    if n > 100:
        d_max *= (n / 100.) ** 0.49
        n = 100
    pval = np.exp(-7.01256 * d_max ** 2 * (n + 2.78019)
                  + 2.99587 * d_max * np.sqrt(n + 2.78019) - 0.122119
                  + 0.974598 / np.sqrt(n) + 1.67997 / n)
    return pval

def kstest_fit(x, dist='norm', pvalmethod="table"):
    """
    Test assumed normal or exponential distribution using Lilliefors' test.

    Lilliefors' test is a Kolmogorov-Smirnov test with estimated parameters.

    Parameters
    ----------
    x : array_like, 1d
        Data to test.
    dist : {'norm', 'exp'}, optional
        The assumed distribution.
    pvalmethod : {'approx', 'table'}, optional
        The method used to compute the p-value of the test statistic. In
        general, 'table' is preferred and makes use of a very large simulation.
        'approx' is only valid for normality. if `dist = 'exp'` `table` is
        always used. 'approx' uses the approximation formula of Dalal and
        Wilkinson, valid for pvalues < 0.1. If the pvalue is larger than 0.1,
        then the result of `table` is returned.

    Returns
    -------
    ksstat : float
        Kolmogorov-Smirnov test statistic with estimated mean and variance.
    pvalue : float
        If the pvalue is lower than some threshold, e.g. 0.05, then we can
        reject the Null hypothesis that the sample comes from a normal
        distribution.

    Notes
    -----
    'table' uses an improved table based on 10,000,000 simulations. The
    critical values are approximated using
    log(cv_alpha) = b_alpha + c[0] log(n) + c[1] log(n)**2
    where cv_alpha is the critical value for a test with size alpha,
    b_alpha is an alpha-specific intercept term and c[1] and c[2] are
    coefficients that are shared all alphas.
    Values in the table are linearly interpolated. Values outside the
    range are be returned as bounds, 0.990 for large and 0.001 for small
    pvalues.

    For implementation details, see  lilliefors_critical_value_simulation.py in
    the test directory.
    """
    pvalmethod = string_like(pvalmethod,
                             "pvalmethod",
                             options=("approx", "table"))
    x = np.asarray(x)
    nobs = len(x)

    if dist == 'norm':
        z = (x - x.mean()) / x.std(ddof=1)
        test_d = stats.norm.cdf
        lilliefors_table = lilliefors_table_norm
    elif dist == 'exp':
        z = x / x.mean()
        test_d = stats.expon.cdf
        lilliefors_table = lilliefors_table_expon
        pvalmethod = 'table'
    else:
        raise ValueError("Invalid dist parameter, must be 'norm' or 'exp'")

    min_nobs = 4 if dist == 'norm' else 3
    if nobs < min_nobs:
        raise ValueError('Test for distribution {0} requires at least {1} '
                         'observations'.format(dist, min_nobs))

    d_ks = ksstat(z, test_d, alternative='two_sided')

    if pvalmethod == 'approx':
        pval = pval_lf(d_ks, nobs)
        # check pval is in desired range
        if pval > 0.1:
            pval = lilliefors_table.prob(d_ks, nobs)
    else:  # pvalmethod == 'table'
        pval = lilliefors_table.prob(d_ks, nobs)

    return d_ks, pval

lilliefors = kstest_fit
kstest_normal = kstest_fit
kstest_exponential = partial(kstest_fit, dist='exp')
<commit_msg>Squeeze kstest_fit input to 1d and raise ValueError when unsuccesful<commit_after>"""
Implements Lilliefors corrected Kolmogorov-Smirnov tests for normal and
exponential distributions.

`kstest_fit` is provided as a top-level function to access both tests.
`kstest_normal` and `kstest_exponential` are provided as convenience functions
with the appropriate test as the default.
`lilliefors` is provided as an alias for `kstest_fit`.

Created on Sat Oct 01 13:16:49 2011

Author: Josef Perktold
License: BSD-3

pvalues for Lilliefors test are based on formula and table in

An Analytic Approximation to the Distribution of Lilliefors's Test Statistic
for Normality
Author(s): Gerard E. Dallal and Leland WilkinsonSource: The American
Statistician, Vol. 40, No. 4 (Nov., 1986), pp. 294-296
Published by: American Statistical Association
Stable URL: http://www.jstor.org/stable/2684607 .

On the Kolmogorov-Smirnov Test for Normality with Mean and Variance Unknown
Hubert W. Lilliefors
Journal of the American Statistical Association, Vol. 62, No. 318.
(Jun., 1967), pp. 399-402.

---

Updated 2017-07-23
Jacob C. Kimmel

Ref:
Lilliefors, H.W.
On the Kolmogorov-Smirnov test for the exponential distribution with mean
unknown. Journal of the American Statistical Association, Vol 64, No. 325.
(1969), pp. 387–389.
"""
from functools import partial

import numpy as np
from scipy import stats

from statsmodels.tools.validation import string_like
from ._lilliefors_critical_values import (critical_values,
                                          asymp_critical_values,
                                          PERCENTILES)
from .tabledist import TableDist

def _make_asymptotic_function(params):
    """
    Generates an asymptotic distribution callable from a param matrix

    Polynomial is a[0] * x**(-1/2) + a[1] * x**(-1) + a[2] * x**(-3/2)

    Parameters
    ----------
    params : ndarray
        Array with shape (nalpha, 3) where nalpha is the number of
        significance levels
    """

    def f(n):
        poly = np.array([1, np.log(n), np.log(n) ** 2])
        return np.exp(poly.dot(params.T))

    return f

def ksstat(x, cdf, alternative='two_sided', args=()):
    """
    Calculate statistic for the Kolmogorov-Smirnov test for goodness of fit

    This calculates the test statistic for a test of the distribution G(x) of
    an observed variable against a given distribution F(x). Under the null
    hypothesis the two distributions are identical, G(x)=F(x). The
    alternative hypothesis can be either 'two_sided' (default), 'less'
    or 'greater'. The KS test is only valid for continuous distributions.

    Parameters
    ----------
    x : array_like, 1d
        array of observations
    cdf : str or callable
        string: name of a distribution in scipy.stats
        callable: function to evaluate cdf
    alternative : 'two_sided' (default), 'less' or 'greater'
        defines the alternative hypothesis (see explanation)
    args : tuple, sequence
        distribution parameters for call to cdf

    Returns
    -------
    D : float
        KS test statistic, either D, D+ or D-

    See Also
    --------
    scipy.stats.kstest

    Notes
    -----

    In the one-sided test, the alternative is that the empirical
    cumulative distribution function of the random variable is "less"
    or "greater" than the cumulative distribution function F(x) of the
    hypothesis, G(x)<=F(x), resp. G(x)>=F(x).

    In contrast to scipy.stats.kstest, this function only calculates the
    statistic which can be used either as distance measure or to implement
    case specific p-values.
    """
    nobs = float(len(x))

    if isinstance(cdf, str):
        cdf = getattr(stats.distributions, cdf).cdf
    elif hasattr(cdf, 'cdf'):
        cdf = getattr(cdf, 'cdf')

    x = np.sort(x)
    cdfvals = cdf(x, *args)

    d_plus = (np.arange(1.0, nobs + 1) / nobs - cdfvals).max()
    d_min = (cdfvals - np.arange(0.0, nobs) / nobs).max()
    if alternative == 'greater':
        return d_plus
    elif alternative == 'less':
        return d_min

    return np.max([d_plus, d_min])

def get_lilliefors_table(dist='norm'):
    """
    Generates tables for significance levels of Lilliefors test statistics

    Tables for available normal and exponential distribution testing,
    as specified in Lilliefors references above

    Parameters
    ----------
    dist : str
        distribution being tested in set {'norm', 'exp'}.

    Returns
    -------
    lf : TableDist object.
        table of critical values
    """
    # function just to keep things together
    # for this test alpha is sf probability, i.e. right tail probability

    alpha = 1 - np.array(PERCENTILES) / 100.0
    alpha = alpha[::-1]
    dist = 'normal' if dist == 'norm' else dist
    if dist not in critical_values:
        raise ValueError("Invalid dist parameter. Must be 'norm' or 'exp'")
    cv_data = critical_values[dist]
    acv_data = asymp_critical_values[dist]

    size = np.array(sorted(cv_data), dtype=float)
    crit_lf = np.array([cv_data[key] for key in sorted(cv_data)])
    crit_lf = crit_lf[:, ::-1]

    asym_params = np.array([acv_data[key] for key in sorted(acv_data)])
    asymp_fn = _make_asymptotic_function((asym_params[::-1]))

    lf = TableDist(alpha, size, crit_lf, asymptotic=asymp_fn)
    return lf

lilliefors_table_norm = get_lilliefors_table(dist='norm')
lilliefors_table_expon = get_lilliefors_table(dist='exp')

def pval_lf(d_max, n):
    """
    Approximate pvalues for Lilliefors test

    This is only valid for pvalues smaller than 0.1 which is not checked in
    this function.

    Parameters
    ----------
    d_max : array_like
        two-sided Kolmogorov-Smirnov test statistic
    n : int or float
        sample size

    Returns
    -------
    p-value : float or ndarray
        pvalue according to approximation formula of Dallal and Wilkinson.

    Notes
    -----
    This is mainly a helper function where the calling code should dispatch
    on bound violations. Therefore it does not check whether the pvalue is in
    the valid range.

    Precision for the pvalues is around 2 to 3 decimals. This approximation is
    also used by other statistical packages (e.g. R:fBasics) but might not be
    the most precise available.

    References
    ----------
    DallalWilkinson1986
    """
    # todo: check boundaries, valid range for n and Dmax
    if n > 100:
        d_max *= (n / 100.) ** 0.49
        n = 100
    pval = np.exp(-7.01256 * d_max ** 2 * (n + 2.78019)
                  + 2.99587 * d_max * np.sqrt(n + 2.78019) - 0.122119
                  + 0.974598 / np.sqrt(n) + 1.67997 / n)
    return pval

def kstest_fit(x, dist='norm', pvalmethod="table"):
    """
    Test assumed normal or exponential distribution using Lilliefors' test.

    Lilliefors' test is a Kolmogorov-Smirnov test with estimated parameters.

    Parameters
    ----------
    x : array_like, 1d
        Data to test.
    dist : {'norm', 'exp'}, optional
        The assumed distribution.
    pvalmethod : {'approx', 'table'}, optional
        The method used to compute the p-value of the test statistic. In
        general, 'table' is preferred and makes use of a very large simulation.
        'approx' is only valid for normality. if `dist = 'exp'` `table` is
        always used. 'approx' uses the approximation formula of Dalal and
        Wilkinson, valid for pvalues < 0.1. If the pvalue is larger than 0.1,
        then the result of `table` is returned.

    Returns
    -------
    ksstat : float
        Kolmogorov-Smirnov test statistic with estimated mean and variance.
    pvalue : float
        If the pvalue is lower than some threshold, e.g. 0.05, then we can
        reject the Null hypothesis that the sample comes from a normal
        distribution.

    Notes
    -----
    'table' uses an improved table based on 10,000,000 simulations. The
    critical values are approximated using
    log(cv_alpha) = b_alpha + c[0] log(n) + c[1] log(n)**2
    where cv_alpha is the critical value for a test with size alpha,
    b_alpha is an alpha-specific intercept term and c[1] and c[2] are
    coefficients that are shared all alphas.
    Values in the table are linearly interpolated. Values outside the
    range are be returned as bounds, 0.990 for large and 0.001 for small
    pvalues.

    For implementation details, see  lilliefors_critical_value_simulation.py in
    the test directory.
    """
    pvalmethod = string_like(pvalmethod,
                             "pvalmethod",
                             options=("approx", "table"))
    x = np.squeeze(np.asarray(x))
    if x.ndim != 1:
        raise ValueError(f"Array `x` with shape {x.shape} cannot be squeezed"
                         f" into a 1d array")
    nobs = len(x)

    if dist == 'norm':
        z = (x - x.mean()) / x.std(ddof=1)
        test_d = stats.norm.cdf
        lilliefors_table = lilliefors_table_norm
    elif dist == 'exp':
        z = x / x.mean()
        test_d = stats.expon.cdf
        lilliefors_table = lilliefors_table_expon
        pvalmethod = 'table'
    else:
        raise ValueError("Invalid dist parameter, must be 'norm' or 'exp'")

    min_nobs = 4 if dist == 'norm' else 3
    if nobs < min_nobs:
        raise ValueError('Test for distribution {0} requires at least {1} '
                         'observations'.format(dist, min_nobs))

    d_ks = ksstat(z, test_d, alternative='two_sided')

    if pvalmethod == 'approx':
        pval = pval_lf(d_ks, nobs)
        # check pval is in desired range
        if pval > 0.1:
            pval = lilliefors_table.prob(d_ks, nobs)
    else:  # pvalmethod == 'table'
        pval = lilliefors_table.prob(d_ks, nobs)

    return d_ks, pval

lilliefors = kstest_fit
kstest_normal = kstest_fit
kstest_exponential = partial(kstest_fit, dist='exp')
<|endoftext|>


<commit_before>"""project related functions"""

import os
import shutil
import subprocess

from mod import log, util, config, dep, template
from mod.tools import git, cmake, make, ninja, xcodebuild, ccmake, cmake_gui

#-------------------------------------------------------------------------------
def init(fips_dir, proj_name) :
    """initialize an existing project directory as a fips directory by
    copying essential files and creating or updating .gitignore

    :param fips_dir:    absolute path to fips
    :param proj_name:   project directory name (dir must exist)
    :returns:           True if the project was successfully initialized
    """
    ws_dir = util.get_workspace_dir(fips_dir)
    proj_dir = util.get_project_dir(fips_dir, proj_name)
    if os.path.isdir(proj_dir) :
        templ_values = {
            'project': proj_name
        }
        for f in ['CMakeLists.txt', 'fips', 'fips.cmd', 'fips.yml'] :
            template.copy_template_file(fips_dir, proj_dir, f, templ_values)
        os.chmod(proj_dir + '/fips', 0o744)
        gitignore_entries = ['.fips-*', '*.pyc']
        template.write_git_ignore(proj_dir, gitignore_entries)
    else :
        log.error("project dir '{}' does not exist".format(proj_dir))
        return False

#-------------------------------------------------------------------------------
def clone(fips_dir, url) :
    """clone an existing fips project with git, do NOT fetch dependencies

    :param fips_dir:    absolute path to fips
    :param url:         git url to clone from
    :return:            True if project was successfully cloned
    """
    ws_dir = util.get_workspace_dir(fips_dir)
    proj_name = util.get_project_name_from_url(url)
    proj_dir = util.get_project_dir(fips_dir, proj_name)
    if not os.path.isdir(proj_dir) :
        if git.clone(url, proj_name, ws_dir) :
            # fetch imports
            dep.fetch_imports(fips_dir, proj_dir)
            return True
        else :
            log.error("failed to 'git clone {}' into '{}'".format(url, proj_dir))
            return False
    else :
        log.error("project dir '{}' already exists".format(proj_dir))
        return False

#-------------------------------------------------------------------------------
def gen_project(fips_dir, proj_dir, cfg) :
    """private: generate build files for one config"""

    log.colored(log.YELLOW, "=== generating: {}".format(cfg['name']))
    proj_name = util.get_project_name_from_dir(proj_dir)
    build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
    if not os.path.isdir(build_dir) :
        os.makedirs(build_dir)
    toolchain_path = config.get_toolchain_for_platform(fips_dir, cfg['platform'])
    return cmake.run_gen(cfg, proj_dir, build_dir, toolchain_path)

#-------------------------------------------------------------------------------
def gen(fips_dir, proj_dir, cfg_name) :
    """generate build files with cmake

    :param fips_dir:    absolute path to fips
    :param proj_dir:    absolute path to project
    :param cfg_name:    config name or pattern (e.g. osx-make-debug)
    :returns:           True if successful
    """

    # prepare
    dep.fetch_imports(fips_dir, proj_dir)
    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)
    dep.write_imports_files(fips_dir, proj_dir)

    # load the config(s)
    configs = config.load(cfg_name, [fips_dir])
    num_valid_configs = 0
    if configs :
        for cfg in configs :
            # check if config is valid
            if config.check_config_valid(cfg) :
                if gen_project(fips_dir, proj_dir, cfg) :
                    num_valid_configs += 1
                else :
                    log.error("failed to generate build files for config '{}'".format(cfg['name']), False)
            else :
                log.error("'{}' is not a valid config".format(cfg['name']), False)
    else :
        log.error("No configs found for '{}'".format(cfg_name))

    if num_valid_configs != len(configs) :
        log.error('{} out of {} configs failed!'.format(len(configs) - num_valid_configs, len(configs)))
        return False
    else :
        log.colored(log.GREEN, '{} configs generated'.format(num_valid_configs))
        return True

#-------------------------------------------------------------------------------
def configure(fips_dir, proj_dir, cfg_name) :
    """run ccmake or cmake-gui on the provided project and config

    :param fips_dir:    absolute fips path
    :param proj_dir:    absolute project dir
    :cfg_name:          build config name
    """

    dep.fetch_imports(fips_dir, proj_dir)
    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)
    dep.write_imports_files(fips_dir, proj_dir)

    # load configs, if more then one, only use first one
    configs = config.load(cfg_name, [fips_dir])
    if configs :
        cfg = configs[0]
        log.colored(log.YELLOW, '=== configuring: {}'.format(cfg['name']))

        # generate build files
        if not gen_project(fips_dir, proj_dir, cfg) :
            log.error("Failed to generate '{}' of project '{}'".format(cfg['name'], proj_name))

        # run ccmake or cmake-gui
        build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
        if ccmake.check_exists() :
            ccmake.run(build_dir)
        elif cmake_gui.check_exists() :
            cmake_gui.run(build_dir)
        else :
            log.error("Neither 'ccmake' nor 'cmake-gui' found (run 'fips diag')")
    else :
        log.error("No configs found for '{}'".format(cfg_name))

#-------------------------------------------------------------------------------
def build(fips_dir, proj_dir, cfg_name, target=None) :
    """perform a build of config(s) in project

    :param fips_dir:    absolute path of fips
    :param proj_dir:    absolute path of project dir
    :param cfg_name:    config name or pattern
    :param target:      optional target name (build all if None)
    :returns:           True if build was successful
    """

    # prepare
    dep.fetch_imports(fips_dir, proj_dir)
    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)
    dep.write_imports_files(fips_dir, proj_dir)

    # load the config(s)
    configs = config.load(cfg_name, [fips_dir])
    num_valid_configs = 0
    if configs :
        for cfg in configs :
            # check if config is valid
            if config.check_config_valid(cfg) :
                log.colored(log.YELLOW, "=== building: {}".format(cfg['name']))

                if not gen_project(fips_dir, proj_dir, cfg) :
                    log.error("Failed to generate '{}' of project '{}'".format(cfg['name'], proj_name))

                # select and run build tool
                # FIXME: make number of jobs configurable
                build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
                num_jobs = 3
                result = False
                if cfg['build_tool'] == make.name :
                    result = make.run_build(target, build_dir, num_jobs)
                elif cfg['build_tool'] == ninja.name :
                    result = ninja.run_build(target, build_dir, num_jobs)
                elif cfg['build_tool'] == xcodebuild.name :
                    result = xcodebuild.run_build(target, cfg['build_type'], build_dir, num_jobs)
                else :
                    result = cmake.run_build(target, cfg['build_type'], build_dir)

                if result :
                    num_valid_configs += 1
                else :
                    log.error("Failed to build config '{}' of project '{}'".format(cfg['name'], proj_name))
    else :
        log.error("No valid configs found for '{}'".format(cfg_name))

    if num_valid_configs != len(configs) :
        log.error('{} out of {} configs failed!'.format(len(configs) - num_valid_configs, len(configs)))
        return False
    else :
        log.colored(log.GREEN, '{} configs built'.format(num_valid_configs))
        return True

#-------------------------------------------------------------------------------
def run(fips_dir, proj_dir, cfg_name, target_name) :
    """run a build target executable

    :param fips_dir:    absolute path of fips
    :param proj_dir:    absolute path of project dir
    :param cfg_name:    config name or pattern
    :param target_name: the target name
    :returns:           True if build was successful
    """

    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)

    # load the config(s)
    configs = config.load(cfg_name, [fips_dir])
    if configs :
        for cfg in configs :
            log.colored(log.YELLOW, "=== run '{}' (config: {}, project: {}):".format(target_name, cfg['name'], proj_name))

            # find deploy dir where executables live
            deploy_dir = util.get_deploy_dir(fips_dir, proj_name, cfg)

            cmd_line = []
            if cfg['platform'] in ['emscripten', 'pnacl'] :
                # special case: emscripten app
                if cfg['platform'] == 'emscripten' :
                    html_name = target_name + '.html'
                else :
                    html_name = target_name + '_pnacl.html'
                if config.get_host_platform() == 'osx' :
                    try :
                        subprocess.call(
                            ['open http://localhost:8000/{} ; python {}/mod/httpserver.py'.format(html_name, fips_dir)],
                            cwd = deploy_dir, shell=True)
                    except KeyboardInterrupt :
                        pass
                elif config.get_host_platform() == 'win' :
                    log.error('FIXME: start HTML app on Windows')
                elif config.get_host_platform() == 'linux' :
                    log.error('FIXME: start HTML app on Linux')
                else :
                    log.error("don't know how to start HTML app on this platform")
            elif os.path.isdir('{}/{}.app'.format(deploy_dir, target_name)) :
                # special case: Mac app
                cmd_line = ['open', '{}/{}.app'.format(deploy_dir, target_name)]
            else :
                cmd_line = [ '{}/{}'.format(deploy_dir, target_name) ]
            if cmd_line :
                try:
                    subprocess.call(args=cmd_line, cwd=deploy_dir)
                except OSError, e:
                    log.error("Failed to execute '{}' with '{}'".format(target_name, e.strerror))
    else :
        log.error("No valid configs found for '{}'".format(cfg_name))

#-------------------------------------------------------------------------------
def clean(fips_dir, proj_dir, cfg_name) :
    """clean build files

    :param fips_dir:    absolute path of fips
    :param proj_dir:    absolute project path
    :param cfg_name:    config name (or pattern)
    """
    proj_name = util.get_project_name_from_dir(proj_dir)
    configs = config.load(cfg_name, [fips_dir])
    if configs :
        for cfg in configs :
            log.colored(log.YELLOW, "=== clean: {}".format(cfg['name']))

            build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
            if os.path.isdir(build_dir) :
                shutil.rmtree(build_dir)
                log.info("  deleted '{}'".format(build_dir))

            deploy_dir = util.get_deploy_dir(fips_dir, proj_name, cfg)
            if os.path.isdir(deploy_dir) :
                shutil.rmtree(deploy_dir)
                log.info("  deleted '{}'".format(deploy_dir))
    else :
        log.error("No valid configs found for '{}'".format(cfg_name))

<commit_msg>Fixed 'fips run' on Linux for browser platforms<commit_after>"""project related functions"""

import os
import shutil
import subprocess

from mod import log, util, config, dep, template
from mod.tools import git, cmake, make, ninja, xcodebuild, ccmake, cmake_gui

#-------------------------------------------------------------------------------
def init(fips_dir, proj_name) :
    """initialize an existing project directory as a fips directory by
    copying essential files and creating or updating .gitignore

    :param fips_dir:    absolute path to fips
    :param proj_name:   project directory name (dir must exist)
    :returns:           True if the project was successfully initialized
    """
    ws_dir = util.get_workspace_dir(fips_dir)
    proj_dir = util.get_project_dir(fips_dir, proj_name)
    if os.path.isdir(proj_dir) :
        templ_values = {
            'project': proj_name
        }
        for f in ['CMakeLists.txt', 'fips', 'fips.cmd', 'fips.yml'] :
            template.copy_template_file(fips_dir, proj_dir, f, templ_values)
        os.chmod(proj_dir + '/fips', 0o744)
        gitignore_entries = ['.fips-*', '*.pyc']
        template.write_git_ignore(proj_dir, gitignore_entries)
    else :
        log.error("project dir '{}' does not exist".format(proj_dir))
        return False

#-------------------------------------------------------------------------------
def clone(fips_dir, url) :
    """clone an existing fips project with git, do NOT fetch dependencies

    :param fips_dir:    absolute path to fips
    :param url:         git url to clone from
    :return:            True if project was successfully cloned
    """
    ws_dir = util.get_workspace_dir(fips_dir)
    proj_name = util.get_project_name_from_url(url)
    proj_dir = util.get_project_dir(fips_dir, proj_name)
    if not os.path.isdir(proj_dir) :
        if git.clone(url, proj_name, ws_dir) :
            # fetch imports
            dep.fetch_imports(fips_dir, proj_dir)
            return True
        else :
            log.error("failed to 'git clone {}' into '{}'".format(url, proj_dir))
            return False
    else :
        log.error("project dir '{}' already exists".format(proj_dir))
        return False

#-------------------------------------------------------------------------------
def gen_project(fips_dir, proj_dir, cfg) :
    """private: generate build files for one config"""

    log.colored(log.YELLOW, "=== generating: {}".format(cfg['name']))
    proj_name = util.get_project_name_from_dir(proj_dir)
    build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
    if not os.path.isdir(build_dir) :
        os.makedirs(build_dir)
    toolchain_path = config.get_toolchain_for_platform(fips_dir, cfg['platform'])
    return cmake.run_gen(cfg, proj_dir, build_dir, toolchain_path)

#-------------------------------------------------------------------------------
def gen(fips_dir, proj_dir, cfg_name) :
    """generate build files with cmake

    :param fips_dir:    absolute path to fips
    :param proj_dir:    absolute path to project
    :param cfg_name:    config name or pattern (e.g. osx-make-debug)
    :returns:           True if successful
    """

    # prepare
    dep.fetch_imports(fips_dir, proj_dir)
    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)
    dep.write_imports_files(fips_dir, proj_dir)

    # load the config(s)
    configs = config.load(cfg_name, [fips_dir])
    num_valid_configs = 0
    if configs :
        for cfg in configs :
            # check if config is valid
            if config.check_config_valid(cfg) :
                if gen_project(fips_dir, proj_dir, cfg) :
                    num_valid_configs += 1
                else :
                    log.error("failed to generate build files for config '{}'".format(cfg['name']), False)
            else :
                log.error("'{}' is not a valid config".format(cfg['name']), False)
    else :
        log.error("No configs found for '{}'".format(cfg_name))

    if num_valid_configs != len(configs) :
        log.error('{} out of {} configs failed!'.format(len(configs) - num_valid_configs, len(configs)))
        return False
    else :
        log.colored(log.GREEN, '{} configs generated'.format(num_valid_configs))
        return True

#-------------------------------------------------------------------------------
def configure(fips_dir, proj_dir, cfg_name) :
    """run ccmake or cmake-gui on the provided project and config

    :param fips_dir:    absolute fips path
    :param proj_dir:    absolute project dir
    :cfg_name:          build config name
    """

    dep.fetch_imports(fips_dir, proj_dir)
    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)
    dep.write_imports_files(fips_dir, proj_dir)

    # load configs, if more then one, only use first one
    configs = config.load(cfg_name, [fips_dir])
    if configs :
        cfg = configs[0]
        log.colored(log.YELLOW, '=== configuring: {}'.format(cfg['name']))

        # generate build files
        if not gen_project(fips_dir, proj_dir, cfg) :
            log.error("Failed to generate '{}' of project '{}'".format(cfg['name'], proj_name))

        # run ccmake or cmake-gui
        build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
        if ccmake.check_exists() :
            ccmake.run(build_dir)
        elif cmake_gui.check_exists() :
            cmake_gui.run(build_dir)
        else :
            log.error("Neither 'ccmake' nor 'cmake-gui' found (run 'fips diag')")
    else :
        log.error("No configs found for '{}'".format(cfg_name))

#-------------------------------------------------------------------------------
def build(fips_dir, proj_dir, cfg_name, target=None) :
    """perform a build of config(s) in project

    :param fips_dir:    absolute path of fips
    :param proj_dir:    absolute path of project dir
    :param cfg_name:    config name or pattern
    :param target:      optional target name (build all if None)
    :returns:           True if build was successful
    """

    # prepare
    dep.fetch_imports(fips_dir, proj_dir)
    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)
    dep.write_imports_files(fips_dir, proj_dir)

    # load the config(s)
    configs = config.load(cfg_name, [fips_dir])
    num_valid_configs = 0
    if configs :
        for cfg in configs :
            # check if config is valid
            if config.check_config_valid(cfg) :
                log.colored(log.YELLOW, "=== building: {}".format(cfg['name']))

                if not gen_project(fips_dir, proj_dir, cfg) :
                    log.error("Failed to generate '{}' of project '{}'".format(cfg['name'], proj_name))

                # select and run build tool
                # FIXME: make number of jobs configurable
                build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
                num_jobs = 3
                result = False
                if cfg['build_tool'] == make.name :
                    result = make.run_build(target, build_dir, num_jobs)
                elif cfg['build_tool'] == ninja.name :
                    result = ninja.run_build(target, build_dir, num_jobs)
                elif cfg['build_tool'] == xcodebuild.name :
                    result = xcodebuild.run_build(target, cfg['build_type'], build_dir, num_jobs)
                else :
                    result = cmake.run_build(target, cfg['build_type'], build_dir)

                if result :
                    num_valid_configs += 1
                else :
                    log.error("Failed to build config '{}' of project '{}'".format(cfg['name'], proj_name))
    else :
        log.error("No valid configs found for '{}'".format(cfg_name))

    if num_valid_configs != len(configs) :
        log.error('{} out of {} configs failed!'.format(len(configs) - num_valid_configs, len(configs)))
        return False
    else :
        log.colored(log.GREEN, '{} configs built'.format(num_valid_configs))
        return True

#-------------------------------------------------------------------------------
def run(fips_dir, proj_dir, cfg_name, target_name) :
    """run a build target executable

    :param fips_dir:    absolute path of fips
    :param proj_dir:    absolute path of project dir
    :param cfg_name:    config name or pattern
    :param target_name: the target name
    :returns:           True if build was successful
    """

    proj_name = util.get_project_name_from_dir(proj_dir)
    util.ensure_valid_project_dir(proj_dir)

    # load the config(s)
    configs = config.load(cfg_name, [fips_dir])
    if configs :
        for cfg in configs :
            log.colored(log.YELLOW, "=== run '{}' (config: {}, project: {}):".format(target_name, cfg['name'], proj_name))

            # find deploy dir where executables live
            deploy_dir = util.get_deploy_dir(fips_dir, proj_name, cfg)

            cmd_line = []
            if cfg['platform'] in ['emscripten', 'pnacl'] :
                # special case: emscripten app
                if cfg['platform'] == 'emscripten' :
                    html_name = target_name + '.html'
                else :
                    html_name = target_name + '_pnacl.html'
                if config.get_host_platform() == 'osx' :
                    try :
                        subprocess.call(
                            ['open http://localhost:8000/{} ; python {}/mod/httpserver.py'.format(html_name, fips_dir)],
                            cwd = deploy_dir, shell=True)
                    except KeyboardInterrupt :
                        pass
                elif config.get_host_platform() == 'win' :
                    log.error('FIXME: start HTML app on Windows')
                elif config.get_host_platform() == 'linux' :
                    try :
                        subprocess.call(
                            ['xdg-open http://localhost:8000/{}; python {}/mod/httpserver.py'.format(html_name, fips_dir)],
                            cwd = deploy_dir, shell=True)
                    except KeyboardInterrupt :
                        pass
                else :
                    log.error("don't know how to start HTML app on this platform")
            elif os.path.isdir('{}/{}.app'.format(deploy_dir, target_name)) :
                # special case: Mac app
                cmd_line = ['open', '{}/{}.app'.format(deploy_dir, target_name)]
            else :
                cmd_line = [ '{}/{}'.format(deploy_dir, target_name) ]
            if cmd_line :
                try:
                    subprocess.call(args=cmd_line, cwd=deploy_dir)
                except OSError, e:
                    log.error("Failed to execute '{}' with '{}'".format(target_name, e.strerror))
    else :
        log.error("No valid configs found for '{}'".format(cfg_name))

#-------------------------------------------------------------------------------
def clean(fips_dir, proj_dir, cfg_name) :
    """clean build files

    :param fips_dir:    absolute path of fips
    :param proj_dir:    absolute project path
    :param cfg_name:    config name (or pattern)
    """
    proj_name = util.get_project_name_from_dir(proj_dir)
    configs = config.load(cfg_name, [fips_dir])
    if configs :
        for cfg in configs :
            log.colored(log.YELLOW, "=== clean: {}".format(cfg['name']))

            build_dir = util.get_build_dir(fips_dir, proj_name, cfg)
            if os.path.isdir(build_dir) :
                shutil.rmtree(build_dir)
                log.info("  deleted '{}'".format(build_dir))

            deploy_dir = util.get_deploy_dir(fips_dir, proj_name, cfg)
            if os.path.isdir(deploy_dir) :
                shutil.rmtree(deploy_dir)
                log.info("  deleted '{}'".format(deploy_dir))
    else :
        log.error("No valid configs found for '{}'".format(cfg_name))

<|endoftext|>


<commit_before>
#
# soaplib - Copyright (C) Soaplib contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

import soaplib

from lxml import etree

_ns_xs = soaplib.nsmap['xs']

def nillable_value(func):
    def wrapper(cls, value, tns, *args, **kwargs):
        if value is None:
            return Null.to_xml(value, tns, *args, **kwargs)
        return func(cls, value, tns, *args, **kwargs)
    return wrapper

def nillable_element(func):
    def wrapper(cls, element):
        if bool(element.get('{%s}nil' % soaplib.ns_xsi)):
            #element.text is None and len(element.getchildren()) == 0:
            return None
        return func(cls, element)
    return wrapper

def string_to_xml(cls, value, tns, name):
    assert isinstance(value, str) or isinstance(value, unicode), "'value' must " \
                    "be string or unicode. it is instead '%s'" % repr(value)

    retval = etree.Element("{%s}%s" % (tns,name))

    retval.set('{%s}type' % soaplib.ns_xsi, cls.get_type_name_ns())
    retval.text = value

    return retval

class Base(object):
    __namespace__ = None
    __type_name__ = None

    nillable = True

    class Empty(object):
        pass

    @classmethod
    def get_namespace_prefix(cls):
        ns = cls.get_namespace()

        assert ns != "__main__", cls
        assert ns != "soaplib.serializers.base", cls

        retval = soaplib.get_namespace_prefix(ns)

        return retval

    @classmethod
    def get_namespace(cls):
        retval = cls.__namespace__

        if retval is None:
            retval = cls.__module__

        return retval

    @classmethod
    def resolve_namespace(cls, default_ns):
        if cls.__namespace__ is None:
            cls.__namespace__ = cls.__module__

            if (cls.__namespace__.startswith("soaplib") or cls.__namespace__ == '__main__'):
                cls.__namespace__ = default_ns

    @classmethod
    def get_type_name(cls):
        retval = cls.__type_name__
        if retval is None:
            retval = cls.__name__.lower()

        return retval

    @classmethod
    def get_type_name_ns(cls):
        return "%s:%s" % (cls.get_namespace_prefix(), cls.get_type_name())

    @classmethod
    def to_xml(cls, value, tns, name='retval'):
        return string_to_xml(cls, value, tns, name)

    @classmethod
    def add_to_schema(cls, schema_entries):
        '''
        Nothing needs to happen when the type is a standard schema element
        '''
        pass

    @classmethod
    def customize(cls, **kwargs):
        """
        This function duplicates and customizes the class it belongs to. The
        original class remains intact.

        An example where this is useful:
            Array(String) normally creates an instance of Array class. This is
            inconsistent with the rest of the type definitions (e.g. String)
            being classes and not instances.

            Thanks to this function, one will get a new Array class instead,
            with its serializer set to String.
        """

        cls_dup = type(cls.__name__, (cls,), kwargs)

        return cls_dup

class Null(Base):
    @classmethod
    def to_xml(cls, value, tns, name='retval'):
        element = etree.Element("{%s}%s" % (tns,name))
        element.set('{%s}nil' % soaplib.ns_xsi, '1')
        return element

    @classmethod
    def from_xml(cls, element):
        return None

class SimpleType(Base):
    __namespace__ = "http://www.w3.org/2001/XMLSchema"
    __base_type__ = None
    values = set()

    def __new__(cls, **kwargs):
        """
        Overriden so that any attempt to instantiate a primitive will return a
        customized class instead of an instance.

        See serializers.base.Base for more information.
        """

        retval = cls.customize(**kwargs)

        retval.values = kwargs.get("values", SimpleType.values)

        if not retval.is_default():
            retval.__base_type__ = cls.get_type_name_ns()
            retval.__type_name__ = kwargs.get("type_name", Base.Empty)

        return retval

    @classmethod
    def is_default(cls):
        return cls.values == SimpleType.values

    @classmethod
    def get_restriction_tag(cls, schema_entries):
        simple_type = etree.Element('{%s}simpleType' % _ns_xs)
        simple_type.set('name', cls.get_type_name())
        schema_entries.add_simple_type(cls, simple_type)

        restriction = etree.SubElement(simple_type, '{%s}restriction' % _ns_xs)
        restriction.set('base', cls.__base_type__)

        for v in cls.values:
            enumeration = etree.SubElement(restriction, '{%s}enumeration' % _ns_xs)
            enumeration.set('value', v)

        return restriction

    @classmethod
    def add_to_schema(cls, schema_entries):
        if not schema_entries.has_class(cls) and not cls.is_default():
            cls.get_restriction_tag(schema_entries)
<commit_msg>not important<commit_after>
#
# soaplib - Copyright (C) Soaplib contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

import soaplib

from lxml import etree

_ns_xs = soaplib.nsmap['xs']

def nillable_value(func):
    def wrapper(cls, value, tns, *args, **kwargs):
        if value is None:
            return Null.to_xml(value, tns, *args, **kwargs)
        return func(cls, value, tns, *args, **kwargs)
    return wrapper

def nillable_element(func):
    def wrapper(cls, element):
        if bool(element.get('{%s}nil' % soaplib.ns_xsi)): # or (element.text is None and len(element.getchildren()) == 0):
            return None
        return func(cls, element)
    return wrapper

def string_to_xml(cls, value, tns, name):
    assert isinstance(value, str) or isinstance(value, unicode), "'value' must " \
                    "be string or unicode. it is instead '%s'" % repr(value)

    retval = etree.Element("{%s}%s" % (tns,name))

    retval.set('{%s}type' % soaplib.ns_xsi, cls.get_type_name_ns())
    retval.text = value

    return retval

class Base(object):
    __namespace__ = None
    __type_name__ = None

    nillable = True

    class Empty(object):
        pass

    @classmethod
    def get_namespace_prefix(cls):
        ns = cls.get_namespace()

        assert ns != "__main__", cls
        assert ns != "soaplib.serializers.base", cls

        retval = soaplib.get_namespace_prefix(ns)

        return retval

    @classmethod
    def get_namespace(cls):
        retval = cls.__namespace__

        if retval is None:
            retval = cls.__module__

        return retval

    @classmethod
    def resolve_namespace(cls, default_ns):
        if cls.__namespace__ is None:
            cls.__namespace__ = cls.__module__

            if (cls.__namespace__.startswith("soaplib") or cls.__namespace__ == '__main__'):
                cls.__namespace__ = default_ns

    @classmethod
    def get_type_name(cls):
        retval = cls.__type_name__
        if retval is None:
            retval = cls.__name__.lower()

        return retval

    @classmethod
    def get_type_name_ns(cls):
        return "%s:%s" % (cls.get_namespace_prefix(), cls.get_type_name())

    @classmethod
    def to_xml(cls, value, tns, name='retval'):
        return string_to_xml(cls, value, tns, name)

    @classmethod
    def add_to_schema(cls, schema_entries):
        '''
        Nothing needs to happen when the type is a standard schema element
        '''
        pass

    @classmethod
    def customize(cls, **kwargs):
        """
        This function duplicates and customizes the class it belongs to. The
        original class remains intact.

        An example where this is useful:
            Array(String) normally creates an instance of Array class. This is
            inconsistent with the rest of the type definitions (e.g. String)
            being classes and not instances.

            Thanks to this function, one will get a new Array class instead,
            with its serializer set to String.
        """

        cls_dup = type(cls.__name__, (cls,), kwargs)

        return cls_dup

class Null(Base):
    @classmethod
    def to_xml(cls, value, tns, name='retval'):
        element = etree.Element("{%s}%s" % (tns,name))
        element.set('{%s}nil' % soaplib.ns_xsi, '1')
        return element

    @classmethod
    def from_xml(cls, element):
        return None

class SimpleType(Base):
    __namespace__ = "http://www.w3.org/2001/XMLSchema"
    __base_type__ = None
    values = set()

    def __new__(cls, **kwargs):
        """
        Overriden so that any attempt to instantiate a primitive will return a
        customized class instead of an instance.

        See serializers.base.Base for more information.
        """

        retval = cls.customize(**kwargs)

        retval.values = kwargs.get("values", SimpleType.values)

        if not retval.is_default():
            retval.__base_type__ = cls.get_type_name_ns()
            retval.__type_name__ = kwargs.get("type_name", Base.Empty)

        return retval

    @classmethod
    def is_default(cls):
        return cls.values == SimpleType.values

    @classmethod
    def get_restriction_tag(cls, schema_entries):
        simple_type = etree.Element('{%s}simpleType' % _ns_xs)
        simple_type.set('name', cls.get_type_name())
        schema_entries.add_simple_type(cls, simple_type)

        restriction = etree.SubElement(simple_type, '{%s}restriction' % _ns_xs)
        restriction.set('base', cls.__base_type__)

        for v in cls.values:
            enumeration = etree.SubElement(restriction, '{%s}enumeration' % _ns_xs)
            enumeration.set('value', v)

        return restriction

    @classmethod
    def add_to_schema(cls, schema_entries):
        if not schema_entries.has_class(cls) and not cls.is_default():
            cls.get_restriction_tag(schema_entries)
<|endoftext|>