<commit_before>"""
Utilities for fast persistence of big data, with optional compression.
"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.

import pickle
import traceback
import sys
import os
import zlib
import warnings

from ._compat import _basestring

from io import BytesIO

if sys.version_info[0] >= 3:
    Unpickler = pickle._Unpickler
    Pickler = pickle._Pickler

    def asbytes(s):
        if isinstance(s, bytes):
            return s
        return s.encode('latin1')
else:
    Unpickler = pickle.Unpickler
    Pickler = pickle.Pickler
    asbytes = str

_MEGA = 2 ** 20
_MAX_LEN = len(hex(2 ** 64))

# To detect file types
_ZFILE_PREFIX = asbytes('ZF')

###############################################################################
# Compressed file with Zlib

def _read_magic(file_handle):
    """ Utility to check the magic signature of a file identifying it as a
        Zfile
    """
    magic = file_handle.read(len(_ZFILE_PREFIX))
    # Pickling needs file-handles at the beginning of the file
    file_handle.seek(0)
    return magic

def read_zfile(file_handle):
    """Read the z-file and return the content as a string

    Z-files are raw data compressed with zlib used internally by joblib
    for persistence. Backward compatibility is not guaranteed. Do not
    use for external purposes.
    """
    file_handle.seek(0)
    assert _read_magic(file_handle) == _ZFILE_PREFIX, \
        "File does not have the right magic"
    length = file_handle.read(len(_ZFILE_PREFIX) + _MAX_LEN)
    length = length[len(_ZFILE_PREFIX):]
    length = int(length, 16)
    # We use the known length of the data to tell Zlib the size of the
    # buffer to allocate.
    data = zlib.decompress(file_handle.read(), 15, length)
    assert len(data) == length, (
        "Incorrect data length while decompressing %s."
        "The file could be corrupted." % file_handle)
    return data

def write_zfile(file_handle, data, compress=1):
    """Write the data in the given file as a Z-file.

    Z-files are raw data compressed with zlib used internally by joblib
    for persistence. Backward compatibility is not guarantied. Do not
    use for external purposes.
    """
    file_handle.write(_ZFILE_PREFIX)
    length = hex(len(data))
    if sys.version_info[0] < 3 and type(length) is long:
        # We need to remove the trailing 'L' in the hex representation
        length = length[:-1]
    # Store the length of the data
    file_handle.write(asbytes(length.ljust(_MAX_LEN)))
    file_handle.write(zlib.compress(asbytes(data), compress))

###############################################################################
# Utility objects for persistence.

class NDArrayWrapper(object):
    """ An object to be persisted instead of numpy arrays.

        The only thing this object does, is to carry the filename in which
        the array has been persisted, and the array subclass.
    """
    def __init__(self, filename, subclass):
        "Store the useful information for later"
        self.filename = filename
        self.subclass = subclass

    def read(self, unpickler):
        "Reconstruct the array"
        filename = os.path.join(unpickler._dirname, self.filename)
        # Load the array from the disk
        np_ver = [int(x) for x in unpickler.np.__version__.split('.', 2)[:2]]
        if np_ver >= [1, 3]:
            array = unpickler.np.load(filename,
                            mmap_mode=unpickler.mmap_mode)
        else:
            # Numpy does not have mmap_mode before 1.3
            array = unpickler.np.load(filename)
        # Reconstruct subclasses. This does not work with old
        # versions of numpy
        if (hasattr(array, '__array_prepare__')
                and not self.subclass in (unpickler.np.ndarray,
                                      unpickler.np.memmap)):
            # We need to reconstruct another subclass
            new_array = unpickler.np.core.multiarray._reconstruct(
                    self.subclass, (0,), 'b')
            new_array.__array_prepare__(array)
            array = new_array
        return array

    #def __reduce__(self):
    #    return None

class ZNDArrayWrapper(NDArrayWrapper):
    """An object to be persisted instead of numpy arrays.

    This object store the Zfile filename in which
    the data array has been persisted, and the meta information to
    retrieve it.

    The reason that we store the raw buffer data of the array and
    the meta information, rather than array representation routine
    (tostring) is that it enables us to use completely the strided
    model to avoid memory copies (a and a.T store as fast). In
    addition saving the heavy information separately can avoid
    creating large temporary buffers when unpickling data with
    large arrays.
    """
    def __init__(self, filename, init_args, state):
        "Store the useful information for later"
        self.filename = filename
        self.state = state
        self.init_args = init_args

    def read(self, unpickler):
        "Reconstruct the array from the meta-information and the z-file"
        # Here we a simply reproducing the unpickling mechanism for numpy
        # arrays
        filename = os.path.join(unpickler._dirname, self.filename)
        array = unpickler.np.core.multiarray._reconstruct(*self.init_args)
        with open(filename, 'rb') as f:
            data = read_zfile(f)
        state = self.state + (data,)
        array.__setstate__(state)
        return array

###############################################################################
# Pickler classes

class NumpyPickler(Pickler):
    """A pickler to persist of big data efficiently.

        The main features of this object are:

         * persistence of numpy arrays in separate .npy files, for which
           I/O is fast.

         * optional compression using Zlib, with a special care on avoid
           temporaries.
    """

    def __init__(self, filename, compress=0, cache_size=10):
        self._filename = filename
        self._filenames = [filename, ]
        self.cache_size = cache_size
        self.compress = compress
        if not self.compress:
            self.file = open(filename, 'wb')
        else:
            self.file = BytesIO()
        # Count the number of npy files that we have created:
        self._npy_counter = 0
        Pickler.__init__(self, self.file,
                                protocol=pickle.HIGHEST_PROTOCOL)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def _write_array(self, array, filename):
        if not self.compress:
            self.np.save(filename, array)
            container = NDArrayWrapper(os.path.basename(filename),
                                       type(array))
        else:
            filename += '.z'
            # Efficient compressed storage:
            # The meta data is stored in the container, and the core
            # numerics in a z-file
            _, init_args, state = array.__reduce__()
            # the last entry of 'state' is the data itself
            with open(filename, 'wb') as zfile:
                write_zfile(zfile, state[-1], compress=self.compress)
            state = state[:-1]
            container = ZNDArrayWrapper(os.path.basename(filename),
                                            init_args, state)
        return container, filename

    def save(self, obj):
        """ Subclass the save method, to save ndarray subclasses in npy
            files, rather than pickling them. Of course, this is a
            total abuse of the Pickler class.
        """
        if self.np is not None and type(obj) in (self.np.ndarray,
                                            self.np.matrix, self.np.memmap):
            size = obj.size * obj.itemsize
            if self.compress and size < self.cache_size * _MEGA:
                # When compressing, as we are not writing directly to the
                # disk, it is more efficient to use standard pickling
                if type(obj) is self.np.memmap:
                    # Pickling doesn't work with memmaped arrays
                    obj = self.np.asarray(obj)
                return Pickler.save(self, obj)
            self._npy_counter += 1
            try:
                filename = '%s_%02i.npy' % (self._filename,
                                            self._npy_counter)
                # This converts the array in a container
                obj, filename = self._write_array(obj, filename)
                self._filenames.append(filename)
            except:
                self._npy_counter -= 1
                # XXX: We should have a logging mechanism
                print('Failed to save %s to .npy file:\n%s' % (
                        type(obj),
                        traceback.format_exc()))
        return Pickler.save(self, obj)

    def close(self):
        if self.compress:
            with open(self._filename, 'wb') as zfile:
                write_zfile(zfile, self.file.getvalue(), self.compress)

class NumpyUnpickler(Unpickler):
    """A subclass of the Unpickler to unpickle our numpy pickles.
    """
    dispatch = Unpickler.dispatch.copy()

    def __init__(self, filename, file_handle, mmap_mode=None):
        self._filename = os.path.basename(filename)
        self._dirname = os.path.dirname(filename)
        self.mmap_mode = mmap_mode
        self.file_handle = self._open_pickle(file_handle)
        Unpickler.__init__(self, self.file_handle)
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def _open_pickle(self, file_handle):
        return file_handle

    def load_build(self):
        """ This method is called to set the state of a newly created
            object.

            We capture it to replace our place-holder objects,
            NDArrayWrapper, by the array we are interested in. We
            replace them directly in the stack of pickler.
        """
        Unpickler.load_build(self)
        if isinstance(self.stack[-1], NDArrayWrapper):
            if self.np is None:
                raise ImportError('Trying to unpickle an ndarray, '
                        "but numpy didn't import correctly")
            nd_array_wrapper = self.stack.pop()
            array = nd_array_wrapper.read(self)
            self.stack.append(array)

    # Be careful to register our new method.
    if sys.version_info[0] >= 3:
        dispatch[pickle.BUILD[0]] = load_build
    else:
        dispatch[pickle.BUILD] = load_build

class ZipNumpyUnpickler(NumpyUnpickler):
    """A subclass of our Unpickler to unpickle on the fly from
    compressed storage."""

    def __init__(self, filename, file_handle):
        NumpyUnpickler.__init__(self, filename,
                                file_handle,
                                mmap_mode=None)

    def _open_pickle(self, file_handle):
        return BytesIO(read_zfile(file_handle))

###############################################################################
# Utility functions

def dump(value, filename, compress=0, cache_size=100):
    """Fast persistence of an arbitrary Python object into a files, with
    dedicated storage for numpy arrays.

    Parameters
    -----------
    value: any Python object
        The object to store to disk
    filename: string
        The name of the file in which it is to be stored
    compress: integer for 0 to 9, optional
        Optional compression level for the data. 0 is no compression.
        Higher means more compression, but also slower read and
        write times. Using a value of 3 is often a good compromise.
        See the notes for more details.
    cache_size: positive number, optional
        Fixes the order of magnitude (in megabytes) of the cache used
        for in-memory compression. Note that this is just an order of
        magnitude estimate and that for big arrays, the code will go
        over this value at dump and at load time.

    Returns
    -------
    filenames: list of strings
        The list of file names in which the data is stored. If
        compress is false, each array is stored in a different file.

    See Also
    --------
    joblib.load : corresponding loader

    Notes
    -----
    Memmapping on load cannot be used for compressed files. Thus
    using compression can significantly slow down loading. In
    addition, compressed files take extra extra memory during
    dump and load.
    """
    if compress is True:
        # By default, if compress is enabled, we want to be using 3 by
        # default
        compress = 3
    if not isinstance(filename, _basestring):
        # People keep inverting arguments, and the resulting error is
        # incomprehensible
        raise ValueError(
              'Second argument should be a filename, %s (type %s) was given'
              % (filename, type(filename))
            )
    try:
        pickler = NumpyPickler(filename, compress=compress,
                               cache_size=cache_size)
        pickler.dump(value)
        pickler.close()
    finally:
        if 'pickler' in locals() and hasattr(pickler, 'file'):
            pickler.file.flush()
            pickler.file.close()
    return pickler._filenames

def load(filename, mmap_mode=None):
    """Reconstruct a Python object from a file persisted with joblib.dump.

    Parameters
    -----------
    filename: string
        The name of the file from which to load the object
    mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
        If not None, the arrays are memory-mapped from the disk. This
        mode has not effect for compressed files. Note that in this
        case the reconstructed object might not longer match exactly
        the originally pickled object.

    Returns
    -------
    result: any Python object
        The object stored in the file.

    See Also
    --------
    joblib.dump : function to save an object

    Notes
    -----

    This function can load numpy array files saved separately during the
    dump. If the mmap_mode argument is given, it is passed to np.load and
    arrays are loaded as memmaps. As a consequence, the reconstructed
    object might not match the original pickled object. Note that if the
    file was saved with compression, the arrays cannot be memmaped.
    """
    with open(filename, 'rb') as file_handle:
        # We are careful to open the file handle early and keep it open to
        # avoid race-conditions on renames. That said, if data are stored in
        # companion files, moving the directory will create a race when
        # joblib tries to access the companion files.
        if _read_magic(file_handle) == _ZFILE_PREFIX:
            if mmap_mode is not None:
                warnings.warn('file "%(filename)s" appears to be a zip, '
                              'ignoring mmap_mode "%(mmap_mode)s" flag passed'
                              % locals(), Warning, stacklevel=2)
            unpickler = ZipNumpyUnpickler(filename, file_handle=file_handle)
        else:
            unpickler = NumpyUnpickler(filename, file_handle=file_handle,
                                       mmap_mode=mmap_mode)

        try:
            obj = unpickler.load()
        finally:
            if hasattr(unpickler, 'file_handle'):
                unpickler.file_handle.close()
        return obj
<commit_msg>Typo in numpy_pickle/load()<commit_after>"""
Utilities for fast persistence of big data, with optional compression.
"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2009 Gael Varoquaux
# License: BSD Style, 3 clauses.

import pickle
import traceback
import sys
import os
import zlib
import warnings

from ._compat import _basestring

from io import BytesIO

if sys.version_info[0] >= 3:
    Unpickler = pickle._Unpickler
    Pickler = pickle._Pickler

    def asbytes(s):
        if isinstance(s, bytes):
            return s
        return s.encode('latin1')
else:
    Unpickler = pickle.Unpickler
    Pickler = pickle.Pickler
    asbytes = str

_MEGA = 2 ** 20
_MAX_LEN = len(hex(2 ** 64))

# To detect file types
_ZFILE_PREFIX = asbytes('ZF')

###############################################################################
# Compressed file with Zlib

def _read_magic(file_handle):
    """ Utility to check the magic signature of a file identifying it as a
        Zfile
    """
    magic = file_handle.read(len(_ZFILE_PREFIX))
    # Pickling needs file-handles at the beginning of the file
    file_handle.seek(0)
    return magic

def read_zfile(file_handle):
    """Read the z-file and return the content as a string

    Z-files are raw data compressed with zlib used internally by joblib
    for persistence. Backward compatibility is not guaranteed. Do not
    use for external purposes.
    """
    file_handle.seek(0)
    assert _read_magic(file_handle) == _ZFILE_PREFIX, \
        "File does not have the right magic"
    length = file_handle.read(len(_ZFILE_PREFIX) + _MAX_LEN)
    length = length[len(_ZFILE_PREFIX):]
    length = int(length, 16)
    # We use the known length of the data to tell Zlib the size of the
    # buffer to allocate.
    data = zlib.decompress(file_handle.read(), 15, length)
    assert len(data) == length, (
        "Incorrect data length while decompressing %s."
        "The file could be corrupted." % file_handle)
    return data

def write_zfile(file_handle, data, compress=1):
    """Write the data in the given file as a Z-file.

    Z-files are raw data compressed with zlib used internally by joblib
    for persistence. Backward compatibility is not guarantied. Do not
    use for external purposes.
    """
    file_handle.write(_ZFILE_PREFIX)
    length = hex(len(data))
    if sys.version_info[0] < 3 and type(length) is long:
        # We need to remove the trailing 'L' in the hex representation
        length = length[:-1]
    # Store the length of the data
    file_handle.write(asbytes(length.ljust(_MAX_LEN)))
    file_handle.write(zlib.compress(asbytes(data), compress))

###############################################################################
# Utility objects for persistence.

class NDArrayWrapper(object):
    """ An object to be persisted instead of numpy arrays.

        The only thing this object does, is to carry the filename in which
        the array has been persisted, and the array subclass.
    """
    def __init__(self, filename, subclass):
        "Store the useful information for later"
        self.filename = filename
        self.subclass = subclass

    def read(self, unpickler):
        "Reconstruct the array"
        filename = os.path.join(unpickler._dirname, self.filename)
        # Load the array from the disk
        np_ver = [int(x) for x in unpickler.np.__version__.split('.', 2)[:2]]
        if np_ver >= [1, 3]:
            array = unpickler.np.load(filename,
                            mmap_mode=unpickler.mmap_mode)
        else:
            # Numpy does not have mmap_mode before 1.3
            array = unpickler.np.load(filename)
        # Reconstruct subclasses. This does not work with old
        # versions of numpy
        if (hasattr(array, '__array_prepare__')
                and not self.subclass in (unpickler.np.ndarray,
                                      unpickler.np.memmap)):
            # We need to reconstruct another subclass
            new_array = unpickler.np.core.multiarray._reconstruct(
                    self.subclass, (0,), 'b')
            new_array.__array_prepare__(array)
            array = new_array
        return array

    #def __reduce__(self):
    #    return None

class ZNDArrayWrapper(NDArrayWrapper):
    """An object to be persisted instead of numpy arrays.

    This object store the Zfile filename in which
    the data array has been persisted, and the meta information to
    retrieve it.

    The reason that we store the raw buffer data of the array and
    the meta information, rather than array representation routine
    (tostring) is that it enables us to use completely the strided
    model to avoid memory copies (a and a.T store as fast). In
    addition saving the heavy information separately can avoid
    creating large temporary buffers when unpickling data with
    large arrays.
    """
    def __init__(self, filename, init_args, state):
        "Store the useful information for later"
        self.filename = filename
        self.state = state
        self.init_args = init_args

    def read(self, unpickler):
        "Reconstruct the array from the meta-information and the z-file"
        # Here we a simply reproducing the unpickling mechanism for numpy
        # arrays
        filename = os.path.join(unpickler._dirname, self.filename)
        array = unpickler.np.core.multiarray._reconstruct(*self.init_args)
        with open(filename, 'rb') as f:
            data = read_zfile(f)
        state = self.state + (data,)
        array.__setstate__(state)
        return array

###############################################################################
# Pickler classes

class NumpyPickler(Pickler):
    """A pickler to persist of big data efficiently.

        The main features of this object are:

         * persistence of numpy arrays in separate .npy files, for which
           I/O is fast.

         * optional compression using Zlib, with a special care on avoid
           temporaries.
    """

    def __init__(self, filename, compress=0, cache_size=10):
        self._filename = filename
        self._filenames = [filename, ]
        self.cache_size = cache_size
        self.compress = compress
        if not self.compress:
            self.file = open(filename, 'wb')
        else:
            self.file = BytesIO()
        # Count the number of npy files that we have created:
        self._npy_counter = 0
        Pickler.__init__(self, self.file,
                                protocol=pickle.HIGHEST_PROTOCOL)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def _write_array(self, array, filename):
        if not self.compress:
            self.np.save(filename, array)
            container = NDArrayWrapper(os.path.basename(filename),
                                       type(array))
        else:
            filename += '.z'
            # Efficient compressed storage:
            # The meta data is stored in the container, and the core
            # numerics in a z-file
            _, init_args, state = array.__reduce__()
            # the last entry of 'state' is the data itself
            with open(filename, 'wb') as zfile:
                write_zfile(zfile, state[-1], compress=self.compress)
            state = state[:-1]
            container = ZNDArrayWrapper(os.path.basename(filename),
                                            init_args, state)
        return container, filename

    def save(self, obj):
        """ Subclass the save method, to save ndarray subclasses in npy
            files, rather than pickling them. Of course, this is a
            total abuse of the Pickler class.
        """
        if self.np is not None and type(obj) in (self.np.ndarray,
                                            self.np.matrix, self.np.memmap):
            size = obj.size * obj.itemsize
            if self.compress and size < self.cache_size * _MEGA:
                # When compressing, as we are not writing directly to the
                # disk, it is more efficient to use standard pickling
                if type(obj) is self.np.memmap:
                    # Pickling doesn't work with memmaped arrays
                    obj = self.np.asarray(obj)
                return Pickler.save(self, obj)
            self._npy_counter += 1
            try:
                filename = '%s_%02i.npy' % (self._filename,
                                            self._npy_counter)
                # This converts the array in a container
                obj, filename = self._write_array(obj, filename)
                self._filenames.append(filename)
            except:
                self._npy_counter -= 1
                # XXX: We should have a logging mechanism
                print('Failed to save %s to .npy file:\n%s' % (
                        type(obj),
                        traceback.format_exc()))
        return Pickler.save(self, obj)

    def close(self):
        if self.compress:
            with open(self._filename, 'wb') as zfile:
                write_zfile(zfile, self.file.getvalue(), self.compress)

class NumpyUnpickler(Unpickler):
    """A subclass of the Unpickler to unpickle our numpy pickles.
    """
    dispatch = Unpickler.dispatch.copy()

    def __init__(self, filename, file_handle, mmap_mode=None):
        self._filename = os.path.basename(filename)
        self._dirname = os.path.dirname(filename)
        self.mmap_mode = mmap_mode
        self.file_handle = self._open_pickle(file_handle)
        Unpickler.__init__(self, self.file_handle)
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np

    def _open_pickle(self, file_handle):
        return file_handle

    def load_build(self):
        """ This method is called to set the state of a newly created
            object.

            We capture it to replace our place-holder objects,
            NDArrayWrapper, by the array we are interested in. We
            replace them directly in the stack of pickler.
        """
        Unpickler.load_build(self)
        if isinstance(self.stack[-1], NDArrayWrapper):
            if self.np is None:
                raise ImportError('Trying to unpickle an ndarray, '
                        "but numpy didn't import correctly")
            nd_array_wrapper = self.stack.pop()
            array = nd_array_wrapper.read(self)
            self.stack.append(array)

    # Be careful to register our new method.
    if sys.version_info[0] >= 3:
        dispatch[pickle.BUILD[0]] = load_build
    else:
        dispatch[pickle.BUILD] = load_build

class ZipNumpyUnpickler(NumpyUnpickler):
    """A subclass of our Unpickler to unpickle on the fly from
    compressed storage."""

    def __init__(self, filename, file_handle):
        NumpyUnpickler.__init__(self, filename,
                                file_handle,
                                mmap_mode=None)

    def _open_pickle(self, file_handle):
        return BytesIO(read_zfile(file_handle))

###############################################################################
# Utility functions

def dump(value, filename, compress=0, cache_size=100):
    """Fast persistence of an arbitrary Python object into a files, with
    dedicated storage for numpy arrays.

    Parameters
    -----------
    value: any Python object
        The object to store to disk
    filename: string
        The name of the file in which it is to be stored
    compress: integer for 0 to 9, optional
        Optional compression level for the data. 0 is no compression.
        Higher means more compression, but also slower read and
        write times. Using a value of 3 is often a good compromise.
        See the notes for more details.
    cache_size: positive number, optional
        Fixes the order of magnitude (in megabytes) of the cache used
        for in-memory compression. Note that this is just an order of
        magnitude estimate and that for big arrays, the code will go
        over this value at dump and at load time.

    Returns
    -------
    filenames: list of strings
        The list of file names in which the data is stored. If
        compress is false, each array is stored in a different file.

    See Also
    --------
    joblib.load : corresponding loader

    Notes
    -----
    Memmapping on load cannot be used for compressed files. Thus
    using compression can significantly slow down loading. In
    addition, compressed files take extra extra memory during
    dump and load.
    """
    if compress is True:
        # By default, if compress is enabled, we want to be using 3 by
        # default
        compress = 3
    if not isinstance(filename, _basestring):
        # People keep inverting arguments, and the resulting error is
        # incomprehensible
        raise ValueError(
              'Second argument should be a filename, %s (type %s) was given'
              % (filename, type(filename))
            )
    try:
        pickler = NumpyPickler(filename, compress=compress,
                               cache_size=cache_size)
        pickler.dump(value)
        pickler.close()
    finally:
        if 'pickler' in locals() and hasattr(pickler, 'file'):
            pickler.file.flush()
            pickler.file.close()
    return pickler._filenames

def load(filename, mmap_mode=None):
    """Reconstruct a Python object from a file persisted with joblib.dump.

    Parameters
    -----------
    filename: string
        The name of the file from which to load the object
    mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
        If not None, the arrays are memory-mapped from the disk. This
        mode has no effect for compressed files. Note that in this
        case the reconstructed object might not longer match exactly
        the originally pickled object.

    Returns
    -------
    result: any Python object
        The object stored in the file.

    See Also
    --------
    joblib.dump : function to save an object

    Notes
    -----

    This function can load numpy array files saved separately during the
    dump. If the mmap_mode argument is given, it is passed to np.load and
    arrays are loaded as memmaps. As a consequence, the reconstructed
    object might not match the original pickled object. Note that if the
    file was saved with compression, the arrays cannot be memmaped.
    """
    with open(filename, 'rb') as file_handle:
        # We are careful to open the file handle early and keep it open to
        # avoid race-conditions on renames. That said, if data are stored in
        # companion files, moving the directory will create a race when
        # joblib tries to access the companion files.
        if _read_magic(file_handle) == _ZFILE_PREFIX:
            if mmap_mode is not None:
                warnings.warn('file "%(filename)s" appears to be a zip, '
                              'ignoring mmap_mode "%(mmap_mode)s" flag passed'
                              % locals(), Warning, stacklevel=2)
            unpickler = ZipNumpyUnpickler(filename, file_handle=file_handle)
        else:
            unpickler = NumpyUnpickler(filename, file_handle=file_handle,
                                       mmap_mode=mmap_mode)

        try:
            obj = unpickler.load()
        finally:
            if hasattr(unpickler, 'file_handle'):
                unpickler.file_handle.close()
        return obj
<|endoftext|>


<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Command-line interface to the Tuskar API.
"""

from __future__ import print_function

import argparse
import logging
import logging.handlers
import sys

import six

import tuskarclient
from tuskarclient import client
import tuskarclient.common.utils as utils
from tuskarclient.openstack.common.apiclient import exceptions as exc

logging.basicConfig()
logger = logging.getLogger(__name__)

class TuskarShell(object):

    def __init__(self, raw_args,
                 argument_parser_class=argparse.ArgumentParser):
        self.raw_args = raw_args
        self.argument_parser_class = argument_parser_class

        self.partial_args = None
        self.parser = None
        self.subparsers = None
        self._prepare_parsers()

    def _prepare_parsers(self):
        nonversioned_parser = self._nonversioned_parser()
        self.partial_args = (
            nonversioned_parser.parse_known_args(self.raw_args)[0])
        self.parser, self.subparsers = (
            self._parser(self.partial_args.tuskar_api_version))

    def run(self):
        '''Run the CLI. Parse arguments and do the respective action.'''

        # run self.do_help() if we have no raw_args at all or just -h/--help
        if (not self.raw_args
                or self.raw_args in (['-h'], ['--help'])):
            self.do_help(self.partial_args)
            return 0

        args = self.parser.parse_args(self.raw_args)

        # run self.do_help() if we have help subcommand or -h/--help option
        if args.func == self.do_help or args.help:
            self.do_help(args)
            return 0

        self._ensure_auth_info(args)

        tuskar_client = client.get_client(
            self.partial_args.tuskar_api_version, **args.__dict__)
        args.func(tuskar_client, args)

    def _ensure_auth_info(self, args):
        '''Ensure that authentication information is provided. Two variants
        of authentication are supported:
        - provide username, password, tenant and auth url
        - provide token and tuskar url (or auth url instead of tuskar url)
        '''
        if not args.os_auth_token:
            if not args.os_username:
                raise exc.CommandError("You must provide username via either "
                                       "--os-username or env[OS_USERNAME]")

            if not args.os_password:
                raise exc.CommandError("You must provide password via either "
                                       "--os-password or env[OS_PASSWORD]")

            if not args.os_tenant_id and not args.os_tenant_name:
                raise exc.CommandError("You must provide tenant via either "
                                       "--os-tenant-name or --os-tenant-id or "
                                       "env[OS_TENANT_NAME] or "
                                       "env[OS_TENANT_ID]")

            if not args.os_auth_url:
                raise exc.CommandError("You must provide auth URL via either "
                                       "--os-auth-url or env[OS_AUTH_URL]")
        else:
            if not args.tuskar_url and not args.os_auth_url:
                raise exc.CommandError("You must provide either "
                                       "--tuskar-url or --os-auth-url or "
                                       "env[TUSKAR_URL] or env[OS_AUTH_URL]")

    def _parser(self, version):
        '''Create a top level argument parser.

        :param version: version of Tuskar API (and corresponding CLI
            commands) to use
        :return: main parser and subparsers
        :rtype: (Parser, Subparsers)
        '''
        parser = self._nonversioned_parser()
        subparsers = parser.add_subparsers(metavar='<subcommand>')
        versioned_shell = utils.import_versioned_module(version, 'shell')
        versioned_shell.enhance_parser(parser, subparsers)
        utils.define_commands_from_module(subparsers, self)
        return parser, subparsers

    def _nonversioned_parser(self):
        '''Create a basic parser that doesn't contain version-specific
        subcommands. This one is mainly useful for parsing which API
        version should be used for the versioned full blown parser and
        defining common version-agnostic options.
        '''
        parser = self.argument_parser_class(
            prog='tuskar',
            description='OpenStack Management CLI',
            add_help=False,
        )

        parser.add_argument('-h', '--help',
                            action='store_true',
                            help="Print this help message and exit.",
                            )

        parser.add_argument('--version',
                            action='version',
                            version=tuskarclient.__version__,
                            help="Shows the client version and exits.")

        parser.add_argument('--os-username',
                            default=utils.env('OS_USERNAME'),
                            help='Defaults to env[OS_USERNAME]',
                            )

        parser.add_argument('--os_username',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-password',
                            default=utils.env('OS_PASSWORD'),
                            help='Defaults to env[OS_PASSWORD]',
                            )

        parser.add_argument('--os_password',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-tenant-id',
                            default=utils.env('OS_TENANT_ID'),
                            help='Defaults to env[OS_TENANT_ID]',
                            )

        parser.add_argument('--os_tenant_id',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-tenant-name',
                            default=utils.env('OS_TENANT_NAME'),
                            help='Defaults to env[OS_TENANT_NAME]',
                            )

        parser.add_argument('--os_tenant_name',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-auth-url',
                            default=utils.env('OS_AUTH_URL'),
                            help='Defaults to env[OS_AUTH_URL]',
                            )

        parser.add_argument('--os_auth_url',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-auth-token',
                            default=utils.env('OS_AUTH_TOKEN'),
                            help='Defaults to env[OS_AUTH_TOKEN]')

        parser.add_argument('--os_auth_token',
                            help=argparse.SUPPRESS)

        parser.add_argument('--tuskar-url',
                            default=utils.env('TUSKAR_URL'),
                            help='Defaults to env[TUSKAR_URL]')

        parser.add_argument('--tuskar_url',
                            help=argparse.SUPPRESS)

        parser.add_argument('--tuskar-api-version',
                            default=utils.env('TUSKAR_API_VERSION',
                                              default='2'),
                            help='Defaults to env[TUSKAR_API_VERSION] '
                            'or 2')

        parser.add_argument('--tuskar_api_version',
                            help=argparse.SUPPRESS)

        return parser

    @utils.arg(
        'command', metavar='<subcommand>', nargs='?',
        help='Display help for <subcommand>')
    def do_help(self, args):
        """Display help about this program or one of its subcommands."""
        if getattr(args, 'command', None):
            if args.command in self.subparsers.choices:
                # print help for subcommand
                self.subparsers.choices[args.command].print_help()
            else:
                raise exc.CommandError("'%s' is not a valid subcommand" %
                                       args.command)
        else:
            # print general help
            self.parser.print_help()

def main():
    try:
        TuskarShell(sys.argv[1:]).run()
    except exc.CommandError as e:
        print(six.text_type(e), file=sys.stderr)
    except Exception as e:
        logger.exception("Exiting due to an error:")
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>Add a --debug option<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Command-line interface to the Tuskar API.
"""

from __future__ import print_function

import argparse
import logging
import logging.handlers
import sys

import six

import tuskarclient
from tuskarclient import client
import tuskarclient.common.utils as utils
from tuskarclient.openstack.common.apiclient import exceptions as exc

logger = logging.getLogger(__name__)

class TuskarShell(object):

    def __init__(self, raw_args,
                 argument_parser_class=argparse.ArgumentParser):
        self.raw_args = raw_args
        self.argument_parser_class = argument_parser_class

        self.partial_args = None
        self.parser = None
        self.subparsers = None
        self._prepare_parsers()

    def _prepare_parsers(self):
        nonversioned_parser = self._nonversioned_parser()
        self.partial_args = (
            nonversioned_parser.parse_known_args(self.raw_args)[0])
        self.parser, self.subparsers = (
            self._parser(self.partial_args.tuskar_api_version))

    def run(self):
        '''Run the CLI. Parse arguments and do the respective action.'''

        # run self.do_help() if we have no raw_args at all or just -h/--help
        if (not self.raw_args
                or self.raw_args in (['-h'], ['--help'])):
            self.do_help(self.partial_args)
            return 0

        args = self.parser.parse_args(self.raw_args)

        self._setup_logging(args.debug)

        # run self.do_help() if we have help subcommand or -h/--help option
        if args.func == self.do_help or args.help:
            self.do_help(args)
            return 0

        self._ensure_auth_info(args)

        tuskar_client = client.get_client(
            self.partial_args.tuskar_api_version, **args.__dict__)
        args.func(tuskar_client, args)

    def _ensure_auth_info(self, args):
        '''Ensure that authentication information is provided. Two variants
        of authentication are supported:
        - provide username, password, tenant and auth url
        - provide token and tuskar url (or auth url instead of tuskar url)
        '''
        if not args.os_auth_token:
            if not args.os_username:
                raise exc.CommandError("You must provide username via either "
                                       "--os-username or env[OS_USERNAME]")

            if not args.os_password:
                raise exc.CommandError("You must provide password via either "
                                       "--os-password or env[OS_PASSWORD]")

            if not args.os_tenant_id and not args.os_tenant_name:
                raise exc.CommandError("You must provide tenant via either "
                                       "--os-tenant-name or --os-tenant-id or "
                                       "env[OS_TENANT_NAME] or "
                                       "env[OS_TENANT_ID]")

            if not args.os_auth_url:
                raise exc.CommandError("You must provide auth URL via either "
                                       "--os-auth-url or env[OS_AUTH_URL]")
        else:
            if not args.tuskar_url and not args.os_auth_url:
                raise exc.CommandError("You must provide either "
                                       "--tuskar-url or --os-auth-url or "
                                       "env[TUSKAR_URL] or env[OS_AUTH_URL]")

    def _parser(self, version):
        '''Create a top level argument parser.

        :param version: version of Tuskar API (and corresponding CLI
            commands) to use
        :return: main parser and subparsers
        :rtype: (Parser, Subparsers)
        '''
        parser = self._nonversioned_parser()
        subparsers = parser.add_subparsers(metavar='<subcommand>')
        versioned_shell = utils.import_versioned_module(version, 'shell')
        versioned_shell.enhance_parser(parser, subparsers)
        utils.define_commands_from_module(subparsers, self)
        return parser, subparsers

    def _nonversioned_parser(self):
        '''Create a basic parser that doesn't contain version-specific
        subcommands. This one is mainly useful for parsing which API
        version should be used for the versioned full blown parser and
        defining common version-agnostic options.
        '''
        parser = self.argument_parser_class(
            prog='tuskar',
            description='OpenStack Management CLI',
            add_help=False,
        )

        parser.add_argument('-h', '--help',
                            action='store_true',
                            help="Print this help message and exit.",
                            )

        parser.add_argument('--version',
                            action='version',
                            version=tuskarclient.__version__,
                            help="Shows the client version and exits.")

        parser.add_argument('-d', '--debug',
                            default=bool(utils.env('TUSKARCLIENT_DEBUG')),
                            action='store_true',
                            help='Defaults to env[TUSKARCLIENT_DEBUG].')

        parser.add_argument('--os-username',
                            default=utils.env('OS_USERNAME'),
                            help='Defaults to env[OS_USERNAME]',
                            )

        parser.add_argument('--os_username',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-password',
                            default=utils.env('OS_PASSWORD'),
                            help='Defaults to env[OS_PASSWORD]',
                            )

        parser.add_argument('--os_password',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-tenant-id',
                            default=utils.env('OS_TENANT_ID'),
                            help='Defaults to env[OS_TENANT_ID]',
                            )

        parser.add_argument('--os_tenant_id',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-tenant-name',
                            default=utils.env('OS_TENANT_NAME'),
                            help='Defaults to env[OS_TENANT_NAME]',
                            )

        parser.add_argument('--os_tenant_name',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-auth-url',
                            default=utils.env('OS_AUTH_URL'),
                            help='Defaults to env[OS_AUTH_URL]',
                            )

        parser.add_argument('--os_auth_url',
                            help=argparse.SUPPRESS,
                            )

        parser.add_argument('--os-auth-token',
                            default=utils.env('OS_AUTH_TOKEN'),
                            help='Defaults to env[OS_AUTH_TOKEN]')

        parser.add_argument('--os_auth_token',
                            help=argparse.SUPPRESS)

        parser.add_argument('--tuskar-url',
                            default=utils.env('TUSKAR_URL'),
                            help='Defaults to env[TUSKAR_URL]')

        parser.add_argument('--tuskar_url',
                            help=argparse.SUPPRESS)

        parser.add_argument('--tuskar-api-version',
                            default=utils.env('TUSKAR_API_VERSION',
                                              default='2'),
                            help='Defaults to env[TUSKAR_API_VERSION] '
                            'or 2')

        parser.add_argument('--tuskar_api_version',
                            help=argparse.SUPPRESS)

        return parser

    @utils.arg(
        'command', metavar='<subcommand>', nargs='?',
        help='Display help for <subcommand>')
    def do_help(self, args):
        """Display help about this program or one of its subcommands."""
        if getattr(args, 'command', None):
            if args.command in self.subparsers.choices:
                # print help for subcommand
                self.subparsers.choices[args.command].print_help()
            else:
                raise exc.CommandError("'%s' is not a valid subcommand" %
                                       args.command)
        else:
            # print general help
            self.parser.print_help()

    def _setup_logging(self, debug):
        log_lvl = logging.DEBUG if debug else logging.WARNING
        logging.basicConfig(
            format="%(levelname)s (%(module)s) %(message)s",
            level=log_lvl)

def main():
    try:
        TuskarShell(sys.argv[1:]).run()
    except exc.CommandError as e:
        print(six.text_type(e), file=sys.stderr)
    except Exception as e:
        logger.exception("Exiting due to an error:")
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2014, 2015, 2016, 2017 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

#
# just a collection of random utility subprograms
#

import sys
import os
import subprocess
try:
    import ConfigParser as configparser
except ImportError:
    import configparser
try:
    from imp import reload
except ImportError:
    pass
import shutil
import errno
import curses
import curses.textpad
import traceback
import code
import signal

from setproctitle import setproctitle
from os.path import expanduser

def load_config(configfn = '.nlprc'):

    home_path = expanduser("~")

    config = configparser.RawConfigParser()
    config.read("%s/%s" % (home_path, configfn))

    return config

def _debug(sig, frame):
    """Interrupt running process, and provide a python prompt for
    interactive debugging.

    source: http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
    """
    d={'_frame':frame}         # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    i = code.InteractiveConsole(d)
    message  = "Signal received : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    i.interact(message)

def init_app (proc_title):

    setproctitle (proc_title)

    if sys.version_info < (3, 0):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    # install signal handler so SIGUSR1 will enter pdb

    signal.signal(signal.SIGUSR1, _debug)  # Register handler

def compress_ws (s):

    vc = True

    res = ''

    for c in s:

        if c == ' ':
            vc = False
        else:
            if vc:
                res = res + c
            else:
                res = res + ' ' + c
            vc = True

    return res

def run_command(command):
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    return iter(p.stdout.readline, b'')

tex_umlaut_map = { u'ä': '"a', u'ü': '"u', u'ö': '"o', u'Ä':'"A', u'Ü':'"U', u'Ö':'"O', u'ß':'"s' }

def tex_encode (u):

    s = ''

    for c in u:

        if c in tex_umlaut_map:
            s += tex_umlaut_map[c]
        else:
            s += str(c)

    return s

def tex_decode (s):

    u = ''

    pos = 0
    while (pos < len(s)):

        found = False

        for umlaut in tex_umlaut_map:
            v = tex_umlaut_map[umlaut]
            if s[pos:].startswith(v):
                u += umlaut
                pos += len(v)
                found = True
                break

        if not found:
            u += unicode(s[pos])
            pos += 1

    return u

def symlink(targetfn, linkfn):
    try:
        os.symlink(targetfn, linkfn)
    except OSError as e:
        if e.errno == errno.EEXIST:
            logging.debug('symlink %s -> %s already exists' % (targetfn, linkfn))

def mkdirs(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def copy_file (src, dst):
    logging.debug("copying %s to %s" % (src, dst))
    shutil.copy(src, dst)

def edit_distance (s, t):
    # https://en.wikipedia.org/wiki/Wagner%E2%80%93Fischer_algorithm

    # for all i and j, d[i,j] will hold the Levenshtein distance between
    # the first i words of s and the first j words of t;
    # note that d has (m+1)x(n+1) values

    m = len(s)
    n = len(t)

    d = [[0 for i in range(n+1)] for j in range(m+1)]

    for i in range (m+1):
        d[i][0] = i                        # the distance of any first seq to an empty second seq
    for j in range (n+1):
        d[0][j] = j                         # the distance of any second seq to an empty first seq

    for j in range (1, n+1):
        for i in range (1, m+1):

            if s[i-1] == t[j-1]:
                d[i][j] = d[i-1][j-1]       # no operation required
            else:
                d[i][j] = min ([
                            d[i-1][j] + 1,       # a deletion
                            d[i][j-1] + 1,       # an insertion
                            d[i-1][j-1] + 1      # a substitution
                         ])

    return d[m][n]

def limit_str(s, limit):

    l = len(s)

    if l<=limit:
        return s

    l = limit-3

    return s[:l] + '...'

#
# curses utils
#

def edit_popup (stdscr, title, s):

    my, mx = stdscr.getmaxyx()

    ww = mx * 9 / 10
    wh = 3

    wox = mx / 2 - ww/2
    woy = my / 2 - wh/2

    win = curses.newwin(wh, ww, woy, wox)
    win.box()
    win.addstr(0, 3, title)

    win.refresh()

    swin = win.derwin (1, ww-4, 1, 2)

    tb = curses.textpad.Textbox(swin, insert_mode=True)

    swin.insstr (0, 0, tex_encode(s))

    swin.refresh()

    s = tex_decode(tb.edit())

    return s.rstrip()

<commit_msg>allow for defaults when reading config files<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2014, 2015, 2016, 2017 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

#
# just a collection of random utility subprograms
#

import sys
import os
import subprocess
try:
    import ConfigParser as configparser
except ImportError:
    import configparser
try:
    from imp import reload
except ImportError:
    pass
import shutil
import errno
import curses
import curses.textpad
import traceback
import code
import signal

from setproctitle import setproctitle
from os.path import expanduser

def load_config(configfn = '.nlprc', defaults={}):

    home_path = expanduser("~")

    config = configparser.ConfigParser(defaults)
    config.read("%s/%s" % (home_path, configfn))

    return config

def _debug(sig, frame):
    """Interrupt running process, and provide a python prompt for
    interactive debugging.

    source: http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application
    """
    d={'_frame':frame}         # Allow access to frame object.
    d.update(frame.f_globals)  # Unless shadowed by global
    d.update(frame.f_locals)

    i = code.InteractiveConsole(d)
    message  = "Signal received : entering python shell.\nTraceback:\n"
    message += ''.join(traceback.format_stack(frame))
    i.interact(message)

def init_app (proc_title):

    setproctitle (proc_title)

    if sys.version_info < (3, 0):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)

    # install signal handler so SIGUSR1 will enter pdb

    signal.signal(signal.SIGUSR1, _debug)  # Register handler

def compress_ws (s):

    vc = True

    res = ''

    for c in s:

        if c == ' ':
            vc = False
        else:
            if vc:
                res = res + c
            else:
                res = res + ' ' + c
            vc = True

    return res

def run_command(command):
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    return iter(p.stdout.readline, b'')

tex_umlaut_map = { u'ä': '"a', u'ü': '"u', u'ö': '"o', u'Ä':'"A', u'Ü':'"U', u'Ö':'"O', u'ß':'"s' }

def tex_encode (u):

    s = ''

    for c in u:

        if c in tex_umlaut_map:
            s += tex_umlaut_map[c]
        else:
            s += str(c)

    return s

def tex_decode (s):

    u = ''

    pos = 0
    while (pos < len(s)):

        found = False

        for umlaut in tex_umlaut_map:
            v = tex_umlaut_map[umlaut]
            if s[pos:].startswith(v):
                u += umlaut
                pos += len(v)
                found = True
                break

        if not found:
            u += unicode(s[pos])
            pos += 1

    return u

def symlink(targetfn, linkfn):
    try:
        os.symlink(targetfn, linkfn)
    except OSError as e:
        if e.errno == errno.EEXIST:
            logging.debug('symlink %s -> %s already exists' % (targetfn, linkfn))

def mkdirs(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def copy_file (src, dst):
    logging.debug("copying %s to %s" % (src, dst))
    shutil.copy(src, dst)

def edit_distance (s, t):
    # https://en.wikipedia.org/wiki/Wagner%E2%80%93Fischer_algorithm

    # for all i and j, d[i,j] will hold the Levenshtein distance between
    # the first i words of s and the first j words of t;
    # note that d has (m+1)x(n+1) values

    m = len(s)
    n = len(t)

    d = [[0 for i in range(n+1)] for j in range(m+1)]

    for i in range (m+1):
        d[i][0] = i                        # the distance of any first seq to an empty second seq
    for j in range (n+1):
        d[0][j] = j                         # the distance of any second seq to an empty first seq

    for j in range (1, n+1):
        for i in range (1, m+1):

            if s[i-1] == t[j-1]:
                d[i][j] = d[i-1][j-1]       # no operation required
            else:
                d[i][j] = min ([
                            d[i-1][j] + 1,       # a deletion
                            d[i][j-1] + 1,       # an insertion
                            d[i-1][j-1] + 1      # a substitution
                         ])

    return d[m][n]

def limit_str(s, limit):

    l = len(s)

    if l<=limit:
        return s

    l = limit-3

    return s[:l] + '...'

#
# curses utils
#

def edit_popup (stdscr, title, s):

    my, mx = stdscr.getmaxyx()

    ww = mx * 9 / 10
    wh = 3

    wox = mx / 2 - ww/2
    woy = my / 2 - wh/2

    win = curses.newwin(wh, ww, woy, wox)
    win.box()
    win.addstr(0, 3, title)

    win.refresh()

    swin = win.derwin (1, ww-4, 1, 2)

    tb = curses.textpad.Textbox(swin, insert_mode=True)

    swin.insstr (0, 0, tex_encode(s))

    swin.refresh()

    s = tex_decode(tb.edit())

    return s.rstrip()

<|endoftext|>


<commit_before>"""
Module to generate URLs to manage tasks, checlists and projects in 2DoApp.

see http://www.2doapp.com

"""
from __future__ import unicode_literals
import sys
from datetime import datetime

PY3 = sys.version_info > (3,)
if PY3:
    # noinspection PyUnresolvedReferences
    from urllib.parse import quote  # pragma: no cover
else:
    from urllib import quote

showall_url = 'twodo://x-callback-url/showAll'
showtoday_url = 'twodo://x-callback-url/showToday'
showstarred_url = 'twodo://x-callback-url/showStarred'
showscheduled_url = 'twodo://x-callback-url/showScheduled'

def get_add_url(task_title):
    """Return a url to add the task, represented by the arguments."""
    return TwoDoTask.BASE_URL.format('task=' + quote(task_title))

class TwoDoTask(object):
    """Represents all attributes of a task in the 2DoApp."""

    BASE_URL = 'twodo://x-callback-url/add?{}'
    TASK_TYPE = '0'
    PROJECT_TYPE = '1'
    CHECKLIST_TYPE = '2'

    PRIO_NONE = '0'
    PRIO_LOW = '1'
    PRIO_MEDIUM = '2'
    PRIO_HIGH = '3'

    def __init__(self, task, task_type=TASK_TYPE, for_list=None,
                 for_parent_task=None, note=None, priority=PRIO_NONE,
                 starred=False, tags=None, due=None, dueTime=None, start=None,
                 repeat=None, action=None, ignoreDefaults=False):
        """Create a 2DoApp-task."""
        self.task = task
        self.type = task_type
        self.for_list = for_list
        self.forParentTask = for_parent_task
        self.note = note
        self.priority = priority
        self.starred = starred
        self.tags = tags
        self.due = due
        self.dueTime = dueTime
        self.start = start
        self.repeat = repeat
        self.action = action
        self.ignoreDefaults = ignoreDefaults

    def url(self):
        """Return the URL for the task of this object."""
        urlpath = 'task=' + quote(self.task)
        if self.type != '0':
            urlpath += '&type={}'.format(self.type)
        if self.for_list is not None:
            urlpath += '&for_list={}'.format(quote(self.for_list))
        if self.note is not None:
            urlpath += '&note={}'.format(quote(self.note))
        if self.priority != '0':
            urlpath += '&priority={}'.format(self.priority)
        if self.starred == '1':
            urlpath += '&starred=1'
        if self.tags is not None:
            urlpath += '&tags={}'.format(quote(self.tags))
        if self.due is not None:
            urlpath += '&due={}'.format(quote(self.due))
        if self.dueTime is not None:
            urlpath += '&dueTime={}'.format(quote(self.dueTime))
        return self.BASE_URL.format(urlpath)

    @property
    def task(self):
        """The task's title text."""
        return self._task

    @task.setter
    def task(self, task_title):
        if task_title is None or len(task_title) == 0:
            raise ValueError('Task title must have content!')
        self._task = task_title

    @task.deleter
    def task(self):
        del self._task

    @property
    def type(self):
        """Type of the task.

        One of TASK_TYPE, PROJECT_TYPE or CHECKLIST_TYPE.
        """
        return self._type

    @type.setter
    def type(self, task_type):
        task_type = str(task_type)
        if task_type not in [self.TASK_TYPE, self.PROJECT_TYPE,
                             self.CHECKLIST_TYPE]:
            msg = "Task type must be one of "
            msg += "'{}', '{}' or '{}', not '{}'".format(self.TASK_TYPE,
                                                         self.PROJECT_TYPE,
                                                         self.CHECKLIST_TYPE,
                                                         task_type)
            raise ValueError(msg)
        self._type = task_type

    @type.deleter
    def type(self):
        del self._type

    @property
    def priority(self):
        """Priority of the task.

        One of PRIO_NONE, PRIO_LOW, PRIO_MEDIUM or PRIO_HIGH.
        """
        return self._priority

    @priority.setter
    def priority(self, priority):
        priority = str(priority)
        if priority not in [self.PRIO_NONE, self.PRIO_LOW, self.PRIO_MEDIUM,
                            self.PRIO_HIGH]:
            msg = "Task priority must be one of '{}', '{}', '{}' or '{}'"
            msg += ", not '{}'".format(self.PRIO_NONE,
                                       self.PRIO_LOW,
                                       self.PRIO_MEDIUM,
                                       self.PRIO_HIGH,
                                       priority)
            raise ValueError(msg)
        self._priority = priority

    @priority.deleter
    def priority(self):
        del self._priority

    @property
    def starred(self):
        """'1' if starred, otherwise '0'."""
        return self._starred

    @starred.setter
    def starred(self, starred):
        """Determine the value for self._starred.

        for starred in [True, '1', 1]
        """
        self._starred = '1' if starred in [True, '1', 1] else '0'

    @starred.deleter
    def starred(self):
        del self._starred

    @property
    def ignoreDefaults(self):
        """If '1', ignore date and time defaults when storing task."""
        return self._ignoreDefaults

    @ignoreDefaults.setter
    def ignoreDefaults(self, ignoreDefaults):
        self._ignoreDefaults = '1' if ignoreDefaults in [True, '1', 1] else '0'

    @ignoreDefaults.deleter
    def ignoreDefaults(self):
        del self._ignoreDefaults

    @property
    def due(self):
        """The due date represented as: YYYY-MM-DD or as number of days."""
        return self._due

    @due.setter
    def due(self, due):
        # raise ValueError if wrong format (no int, no date)
        self._due = None
        if due is not None:
            try:
                int(due)
            except ValueError:
                datetime.strptime(due, '%Y-%m-%d')
            self._due = str(due)

    @due.deleter
    def due(self):
        del self._due
<commit_msg>removed deleters<commit_after>"""
Module to generate URLs to manage tasks, checlists and projects in 2DoApp.

see http://www.2doapp.com

"""
from __future__ import unicode_literals
import sys
from datetime import datetime

PY3 = sys.version_info > (3,)
if PY3:
    # noinspection PyUnresolvedReferences
    from urllib.parse import quote  # pragma: no cover
else:
    from urllib import quote

showall_url = 'twodo://x-callback-url/showAll'
showtoday_url = 'twodo://x-callback-url/showToday'
showstarred_url = 'twodo://x-callback-url/showStarred'
showscheduled_url = 'twodo://x-callback-url/showScheduled'

def get_add_url(task_title):
    """Return a url to add the task, represented by the arguments."""
    return TwoDoTask.BASE_URL.format('task=' + quote(task_title))

class TwoDoTask(object):
    """Represents all attributes of a task in the 2DoApp."""

    BASE_URL = 'twodo://x-callback-url/add?{}'
    TASK_TYPE = '0'
    PROJECT_TYPE = '1'
    CHECKLIST_TYPE = '2'

    PRIO_NONE = '0'
    PRIO_LOW = '1'
    PRIO_MEDIUM = '2'
    PRIO_HIGH = '3'

    DAILY = '1'
    WEEKLY = '2'
    BI_WEEKLY = '3'
    MONTHLY = '4'

    def __init__(self, task, task_type=TASK_TYPE, for_list=None,
                 for_parent_task=None, note=None, priority=PRIO_NONE,
                 starred=False, tags=None, due=None, dueTime=None, start=None,
                 repeat=None, action=None, ignoreDefaults=False):
        """Create a 2DoApp-task."""
        self.task = task
        self.type = task_type
        self.for_list = for_list
        self.forParentTask = for_parent_task
        self.note = note
        self.priority = priority
        self.starred = starred
        self.tags = tags
        self.due = due
        self.dueTime = dueTime
        self.start = start
        self.repeat = repeat
        self.action = action
        self.ignoreDefaults = ignoreDefaults

    def url(self):
        """Return the URL for the task of this object."""
        urlpath = 'task=' + quote(self.task)
        if self.type != self.TASK_TYPE:
            urlpath += '&type={}'.format(self.type)
        if self.for_list is not None:
            urlpath += '&for_list={}'.format(quote(self.for_list))
        if self.note is not None:
            urlpath += '&note={}'.format(quote(self.note))
        if self.priority != self.PRIO_NONE:
            urlpath += '&priority={}'.format(self.priority)
        if self.starred == '1':
            urlpath += '&starred=1'
        if self.tags is not None:
            urlpath += '&tags={}'.format(quote(self.tags))
        if self.due is not None:
            urlpath += '&due={}'.format(quote(self.due))
        if self.dueTime is not None:
            urlpath += '&dueTime={}'.format(quote(self.dueTime))
        # TODO: add start
        if self.repeat is not None:
            urlpath += '&repeat={}'.format(self.repeat)
        return self.BASE_URL.format(urlpath)

    @property
    def task(self):
        """The task's title text."""
        return self._task

    @task.setter
    def task(self, task_title):
        if task_title is None or len(task_title) == 0:
            raise ValueError('Task title must have content!')
        self._task = task_title

    @property
    def type(self):
        """Type of the task.

        One of TwoDoTask.TASK_TYPE, TwoDoTask.PROJECT_TYPE or
        TwoDoTask.CHECKLIST_TYPE.
        """
        return self._type

    @type.setter
    def type(self, task_type):
        task_type = str(task_type)
        if task_type not in [self.TASK_TYPE, self.PROJECT_TYPE,
                             self.CHECKLIST_TYPE]:
            msg = "Task type must be one of "
            msg += "'{}', '{}' or '{}', not '{}'".format(self.TASK_TYPE,
                                                         self.PROJECT_TYPE,
                                                         self.CHECKLIST_TYPE,
                                                         task_type)
            raise ValueError(msg)
        self._type = task_type

    @property
    def priority(self):
        """Priority of the task.

        One of TwoDoTask.PRIO_NONE, TwoDoTask.PRIO_LOW, TwoDoTask.PRIO_MEDIUM
        or TwoDoTask.PRIO_HIGH.
        """
        return self._priority

    @priority.setter
    def priority(self, priority):
        priority = str(priority)
        if priority not in [self.PRIO_NONE, self.PRIO_LOW, self.PRIO_MEDIUM,
                            self.PRIO_HIGH]:
            msg = "Task priority must be one of '{}', '{}', '{}' or '{}'"
            msg += ", not '{}'".format(self.PRIO_NONE,
                                       self.PRIO_LOW,
                                       self.PRIO_MEDIUM,
                                       self.PRIO_HIGH,
                                       priority)
            raise ValueError(msg)
        self._priority = priority

    @property
    def starred(self):
        """'1' if starred, otherwise '0'."""
        return self._starred

    @starred.setter
    def starred(self, starred):
        """Determine the value for self._starred.

        for starred in [True, '1', 1]
        """
        self._starred = '1' if starred in [True, '1', 1] else '0'

    @property
    def ignoreDefaults(self):
        """If '1', ignore date and time defaults when storing task."""
        return self._ignoreDefaults

    @ignoreDefaults.setter
    def ignoreDefaults(self, ignoreDefaults):
        self._ignoreDefaults = '1' if ignoreDefaults in [True, '1', 1] else '0'

    @property
    def due(self):
        """The due date represented as: YYYY-MM-DD or as number of days."""
        return self._due

    @due.setter
    def due(self, due):
        # raise ValueError if wrong format (no int, no date)
        self._due = None
        if due is not None:
            try:
                int(due)
            except ValueError:
                datetime.strptime(due, '%Y-%m-%d')
            self._due = str(due)

    @property
    def repeat(self):
        """Repetition intervall of the task.

        One of TwoDoTask.DAILY, TwoDoTask.WEEKLY, TwoDoTask.BI_WEEKLY,
        TwoDoTask.MONTHLY.
        """
        return self._repeat

    @repeat.setter
    def repeat(self, repeat):
        if repeat not in [None, self.DAILY, self.WEEKLY, self.BI_WEEKLY,
                          self.MONTHLY]:
            msg = "Task repetition (repeat) must be one of "
            msg += "'{}', '{}', '{}' or '{}', not '{}'"
            msg = msg.format(self.DAILY, self.WEEKLY, self.BI_WEEKLY,
                             self.MONTHLY, repeat)
            raise ValueError(msg)
        self._repeat = repeat
<|endoftext|>


<commit_before>#!/usr/bin/env python

DEFAULT_MC_API_URL='http://localhost:9002'

import json
import cbor
import base64
from tornado import gen
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
from urlparse import urljoin
from tornado.ioloop import IOLoop

def ensure_list(x):
    if x is None:
        return []
    if isinstance(x, list):
        return x
    return [x]

def ensure_unicode(val):
    if isinstance(val, basestring):
        return unicode(val)
    if isinstance(val, dict):
        new = {}
        for k, v in val.iteritems():
            new[ensure_unicode(k)] = ensure_unicode(v)
        return new
    if isinstance(val, list):
        return map(ensure_unicode, val)
    return val

def encode_object(obj):
    obj = ensure_unicode(obj)
    return base64.b64encode(cbor.dumps(obj))

class MediachainWriter(object):
    """
    Writes data to a mediachain node using the HTTP API.
    If no API url is given, assumes the node is running on
    localhost with the default port (9002).
    """

    def __init__(self,
                 mc_api_url = False,
                 default_namespace = False,
                 ):
        if mc_api_url is False:
            mc_api_url = DEFAULT_MC_API_URL
        self.mc_api_url = mc_api_url

        self.default_namespace = default_namespace

        self.client = AsyncHTTPClient()

    def _post(self, path, body):
        url = urljoin(self.mc_api_url, path)
        return self.client.fetch(
            HTTPRequest(
                url=url,
                method='POST',
                body=body))

    @gen.coroutine
    def put_data(self, obj):
        """
        Store a data object in the mediachain node
        :param obj: a data object dict to encode and add to the node
        :return: tornado future that resolves to object multihash string
        """
        encoded = encode_object(obj)
        req = json.dumps({'data': encoded})
        result = yield self._post('data/put', req)
        obj_hash = result.body.strip()
        print('put object, hash: ', obj, obj_hash)
        raise gen.Return(obj_hash)

    @gen.coroutine
    def publish(self, namespace=False, data=False, refs=None, deps=None, tags=None):
        """
        Converts `data` to CBOR and writes it, and a statement referencing it,
        to the mediachain node.
        Returns a tornado Future, which will resolve to a string statement ID.
        :param namespace: string namespace to publish to
        :param data: a dict that will be encoded to CBOR and used as the "object" of the MC statement.
        :param refs: a string or list of strings to use as "WKI"s (external ids) for the statement. e.g. the cccoin post id
        :param deps: if the statement depends on other data objects, `deps` should contain their hashes
        :param tags: string or list of strings to use as keywords for queries
        :return: tornado Future, will resolve to a string statement ID on success.
        :raise: HTTPError if request fails
        """
        if namespace is False:
            namespace = self.default_namespace

        assert namespace is not False
        assert data is not False

        object_ref = yield self.put_data(data)
        stmt = {'object': object_ref,
                'refs': ensure_list(refs),
                'deps': ensure_list(deps),
                'tags': ensure_list(tags)}

        response = yield self._post('publish/' + namespace, json.dumps(stmt))
        raise gen.Return(response.body.strip())

def test_mc_simple():
    print ('test_mc_write()')

    w = MediachainWriter()

    rr = IOLoop.current().run_sync(lambda: w.publish('scratch.pytest', {'hello': 'from_python'}))

    print ('GOT', rr)

from threading import current_thread,Thread
from Queue import Queue
from time import sleep

class MediachainQueue:
    """
    Convenience threaded background pusher. Especially useful when you don't already have a tornado event loop.
    Can be used as a context manager, to ensure all items are pushed prior to exit.

    NOTES:
    - Didn't bother to make this parallel for now, but could be easily done.
    - Unneeded at this point, but could do either of:
      + Add an `out_q` to push back permanent failures.
      + Immediately return request_ids that can be polled later for result status.
    """

    def __init__(self,
                 mc_api_url = False,
                 default_namespace = False,
                 sleep_time = 0.1,
                 max_size = 0,
                 ):
        """ Set max_size to > 0 to throttle input. """
        self.sleep_time_orig = sleep_time
        self.sleep_time = sleep_time
        self.in_q = Queue(max_size)
        self.mw = MediachainWriter(mc_api_url = mc_api_url,
                                   default_namespace = default_namespace,
                                   )
        self.start()

    def start(self):
        self.t = Thread(target = self.worker)
        self.t.daemon = True
        self.t.start()

    def push(self, *args, **kw):
        self.in_q.put((args, kw))

    def worker(self):
        while True:
            args, kw = self.in_q.get(block = True)
            IOLoop.current().run_sync(lambda: self.mw.publish(*args, **kw))

    def wait_for_completion(self):
        while True:
            if not self.in_q.size():
                break
            sleep(self.sleep_time)

    def __enter__(self):
        pass

    def __exit__(self):
        self.wait_for_completion()

    def __del__(self):
        self.wait_for_completion()

def test_mc_threaded():
    print ('test_mc_write()')

    print 'STARTING...'

    with MediachainQueue() as mcq:
        for x in xrange(10):
            mcq.put('test_namespace', {'hello': 'from_python_' + str(x)})

        print 'WAITING FOR EXIT..'

    print ('DONE')

if __name__ == '__main__':
    test_mc_simple()
    test_mc_threaded()
<commit_msg>create new IOLoop for MC writer thread<commit_after>#!/usr/bin/env python

DEFAULT_MC_API_URL='http://localhost:9002'

import json
import cbor
import base64
from tornado import gen
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
from urlparse import urljoin
from tornado.ioloop import IOLoop

def ensure_list(x):
    if x is None:
        return []
    if isinstance(x, list):
        return x
    return [x]

def ensure_unicode(val):
    if isinstance(val, basestring):
        return unicode(val)
    if isinstance(val, dict):
        new = {}
        for k, v in val.iteritems():
            new[ensure_unicode(k)] = ensure_unicode(v)
        return new
    if isinstance(val, list):
        return map(ensure_unicode, val)
    return val

def encode_object(obj):
    obj = ensure_unicode(obj)
    return base64.b64encode(cbor.dumps(obj))

class MediachainWriter(object):
    """
    Writes data to a mediachain node using the HTTP API.
    If no API url is given, assumes the node is running on
    localhost with the default port (9002).
    """

    def __init__(self,
                 mc_api_url = False,
                 default_namespace = False,
                 ):
        if mc_api_url is False:
            mc_api_url = DEFAULT_MC_API_URL
        self.mc_api_url = mc_api_url

        self.default_namespace = default_namespace

        self.client = AsyncHTTPClient()

    def _post(self, path, body):
        url = urljoin(self.mc_api_url, path)
        return self.client.fetch(
            HTTPRequest(
                url=url,
                method='POST',
                body=body))

    @gen.coroutine
    def put_data(self, obj):
        """
        Store a data object in the mediachain node
        :param obj: a data object dict to encode and add to the node
        :return: tornado future that resolves to object multihash string
        """
        encoded = encode_object(obj)
        req = json.dumps({'data': encoded})
        result = yield self._post('data/put', req)
        obj_hash = result.body.strip()
        print('put object, hash: ', obj, obj_hash)
        raise gen.Return(obj_hash)

    @gen.coroutine
    def publish(self, namespace=False, data=False, refs=None, deps=None, tags=None):
        """
        Converts `data` to CBOR and writes it, and a statement referencing it,
        to the mediachain node.
        Returns a tornado Future, which will resolve to a string statement ID.
        :param namespace: string namespace to publish to
        :param data: a dict that will be encoded to CBOR and used as the "object" of the MC statement.
        :param refs: a string or list of strings to use as "WKI"s (external ids) for the statement. e.g. the cccoin post id
        :param deps: if the statement depends on other data objects, `deps` should contain their hashes
        :param tags: string or list of strings to use as keywords for queries
        :return: tornado Future, will resolve to a string statement ID on success.
        :raise: HTTPError if request fails
        """
        if namespace is False:
            namespace = self.default_namespace

        assert namespace is not False
        assert data is not False

        object_ref = yield self.put_data(data)
        stmt = {'object': object_ref,
                'refs': ensure_list(refs),
                'deps': ensure_list(deps),
                'tags': ensure_list(tags)}

        response = yield self._post('publish/' + namespace, json.dumps(stmt))
        raise gen.Return(response.body.strip())

def test_mc_simple():
    print ('test_mc_write()')

    w = MediachainWriter()

    rr = IOLoop.current().run_sync(lambda: w.publish('scratch.pytest', {'hello': 'from_python'}))

    print ('GOT', rr)

from threading import current_thread,Thread
from Queue import Queue
from time import sleep

class MediachainQueue:
    """
    Convenience threaded background pusher. Especially useful when you don't already have a tornado event loop.
    Can be used as a context manager, to ensure all items are pushed prior to exit.

    NOTES:
    - Didn't bother to make this parallel for now, but could be easily done.
    - Unneeded at this point, but could do either of:
      + Add an `out_q` to push back permanent failures.
      + Immediately return request_ids that can be polled later for result status.
    """

    def __init__(self,
                 mc_api_url = False,
                 default_namespace = False,
                 sleep_time = 0.1,
                 max_size = 0,
                 ):
        """ Set max_size to > 0 to throttle input. """
        self.sleep_time_orig = sleep_time
        self.sleep_time = sleep_time
        self.in_q = Queue(max_size)
        self.mw = MediachainWriter(mc_api_url = mc_api_url,
                                   default_namespace = default_namespace,
                                   )
        self.start()

    def start(self):
        self.t = Thread(target = self.worker)
        self.t.daemon = True
        self.t.start()

    def push(self, *args, **kw):
        self.in_q.put((args, kw))

    def worker(self):
        io_loop = IOLoop()
        io_loop.start()
        while True:
            args, kw = self.in_q.get(block = True)
            io_loop.run_sync(lambda: self.mw.publish(*args, **kw))

    def wait_for_completion(self):
        while True:
            if not self.in_q.size():
                break
            sleep(self.sleep_time)

    def __enter__(self):
        pass

    def __exit__(self):
        self.wait_for_completion()

    def __del__(self):
        self.wait_for_completion()

def test_mc_threaded():
    print ('test_mc_write()')

    print 'STARTING...'

    with MediachainQueue() as mcq:
        for x in xrange(10):
            mcq.put('test_namespace', {'hello': 'from_python_' + str(x)})

        print 'WAITING FOR EXIT..'

    print ('DONE')

if __name__ == '__main__':
    test_mc_simple()
    test_mc_threaded()
<|endoftext|>


<commit_before>from requests.models import RequestEncodingMixin
from oauthlib.oauth1.rfc5849.utils import (parse_authorization_header,
                                           unescape)
from oauthlib.oauth1.rfc5849.signature import (collect_parameters,
                                               normalize_parameters,
                                               normalize_base_string_uri,
                                               construct_base_string,
                                               sign_hmac_sha1)

def validate_request(request, consumer_getter):
    headers = request.headers()
    auth_header = headers.get('Authorization')
    request_values = request.values()
    if auth_header:
        authorization = dict(parse_authorization_header(auth_header))
        key = authorization['oauth_consumer_key']
        signature = unescape(authorization['oauth_signature'])
    else:
        key = request_values.get('oauth_consumer_key', None)
        signature = request_values.get('oauth_signature', None)

    if key is None or signature is None:
        raise InvalidRequest((
            "Failed to authenticate since necessary parameters were not supplied"
        ))

    consumer = consumer_getter(key)
    if consumer is None:
        raise InvalidRequest('Consumer not found')
    return build_signature(request, consumer['secret']) == unescape(signature)

def build_signature(request, consumer_secret):
    headers = request.headers()
    auth_headers = {k: v for k, v in headers.iteritems() if k == 'Authorization'}
    body = request.form_data()
    params = [(k, v) for k, v in request.params() if k != 'oauth_signature']
    qs = RequestEncodingMixin._encode_params(params)
    collected_params = collect_parameters(qs, body, auth_headers)
    normalized_params = normalize_parameters(collected_params)
    host = headers.get('Host', None)
    normalized_uri = normalize_base_string_uri(request.base_url(), host)
    base_string = construct_base_string(unicode(request.method()),
                                        normalized_uri,
                                        normalized_params)
    return sign_hmac_sha1(base_string, consumer_secret, None)

class Request(object):

    def base_url(self):
        """Base URL of the request

        :rtype : string

        """
        raise NotImplementedError

    def method(self):
        """Request method

        :rtype : unicode

        """
        raise NotImplementedError

    def headers(self):
        """Requests headers

        :rtype : dict like object

        """
        raise NotImplementedError

    def params(self):
        """GET parameters

        :rtype : dict like object

        """
        raise NotImplementedError

    def form_data(self):
        """POST data

        :rtype : dict like object

        """
        raise NotImplementedError

    def values(self):
        """Combined POST and GET data

        :rtype : dict like object

        """
        raise NotImplementedError

class InvalidRequest(Exception):
    pass
<commit_msg>Document code in twolegged module<commit_after>from requests.models import RequestEncodingMixin
from oauthlib.oauth1.rfc5849.utils import (parse_authorization_header,
                                           unescape)
from oauthlib.oauth1.rfc5849.signature import (collect_parameters,
                                               normalize_parameters,
                                               normalize_base_string_uri,
                                               construct_base_string,
                                               sign_hmac_sha1)

def validate_request(request, consumer_getter):
    """Checks whether the request has come from an authorized consumer

    The `request` object is an instance of the twolegged.Request class
    and is framework agnostic. `consumer_getter` is a simple function
    to lookup a consumer from it's key.

    If the request is invalid for any reason other than non-matching
    signatures, a `InvalidRequest` exception will be raised.

    :param request         : a twolegged.Request subclass
    :param consumer_getter : function
    :rtype                 : boolean

    """
    headers = request.headers()
    auth_header = headers.get('Authorization')
    request_values = request.values()
    if auth_header:
        authorization = dict(parse_authorization_header(auth_header))
        key = authorization['oauth_consumer_key']
        signature = unescape(authorization['oauth_signature'])
    else:
        key = request_values.get('oauth_consumer_key', None)
        signature = request_values.get('oauth_signature', None)

    if key is None or signature is None:
        raise InvalidRequest((
            "Failed to authenticate since necessary parameters were not supplied"
        ))

    consumer = consumer_getter(key)
    if consumer is None:
        raise InvalidRequest('Consumer not found')
    return build_signature(request, consumer['secret']) == unescape(signature)

def build_signature(request, consumer_secret):
    """Uses the request object and consumer secret to build a signature

    This is an internal function and it's highly unlikely than an end
    user would ever need to call this

    :param request         : a twolegged.Request subclass
    :param consumer_secret : string
    :rtype                 : string

    """
    headers = request.headers()
    auth_headers = {k: v for k, v in headers.iteritems() if k == 'Authorization'}
    body = request.form_data()
    params = [(k, v) for k, v in request.params() if k != 'oauth_signature']
    qs = RequestEncodingMixin._encode_params(params)
    collected_params = collect_parameters(qs, body, auth_headers)
    normalized_params = normalize_parameters(collected_params)
    host = headers.get('Host', None)
    normalized_uri = normalize_base_string_uri(request.base_url(), host)
    base_string = construct_base_string(unicode(request.method()),
                                        normalized_uri,
                                        normalized_params)
    return sign_hmac_sha1(base_string, consumer_secret, None)

class Request(object):
    """Interface that the functions of this lib understand

    User's of this lib must subclass this class and implement the
    interface. This helps to keep the lib framework agnostic. For
    eg. if you are using django, create a class that would expose
    required aspects of a django request as per this interface which
    this library understands

    """

    def base_url(self):
        """Base URL of the request

        :rtype : string

        """
        raise NotImplementedError

    def method(self):
        """Request method

        :rtype : unicode

        """
        raise NotImplementedError

    def headers(self):
        """Requests headers

        :rtype : dict like object

        """
        raise NotImplementedError

    def params(self):
        """GET parameters

        :rtype : dict like object

        """
        raise NotImplementedError

    def form_data(self):
        """POST data

        :rtype : dict like object

        """
        raise NotImplementedError

    def values(self):
        """Combined POST and GET data

        :rtype : dict like object

        """
        raise NotImplementedError

class InvalidRequest(Exception):
    """Will be raised if the request is invalid for some reason other than
    non-matching signatures

    """
    pass
<|endoftext|>