<commit_before>import os
import sys
import re
import hashlib

from unidecode import unidecode
from datetime import date
from tempfile import NamedTemporaryFile
from lacli.adf import Auth, make_adf
from lacli.crypt import CryptIO
from lacli.cipher import get_cipher
from lacli.hash import HashIO
from lacli.enc import get_unicode
from shutil import copyfileobj
from zipfile import ZipFile, ZIP_DEFLATED
from itertools import starmap

_slugify_strip_re = re.compile(r'[^\w\s-]')
_slugify_hyphenate_re = re.compile(r'[-\s]+')

def _slugify(value):
    """
    Normalizes string, converts to lowercase, removes non-alpha characters,
    and converts spaces to hyphens.
    From Django's "django/template/defaultfilters.py".
    """
    if not isinstance(value, unicode):
        value = unicode(value)
    return unicode(_slugify_hyphenate_re.sub(
        '-', _slugify_strip_re.sub('', unidecode(value))
        ).strip().lower())

def archive_slug(archive):
    return "{}-{}".format(date.today().isoformat(),
                          _slugify(archive.title))

def archive_handle(docs):
    h = HashIO()
    make_adf(docs, out=h)
    return h.getvalue().encode('hex')

def restore_archive(archive, path, cert, folder, tmpdir, cb=None):
    cipher = get_cipher(archive, cert)
    with open(path, 'rb') as infile:
        with NamedTemporaryFile() as dst:
            with CryptIO(infile, cipher) as cf:
                copyfileobj(cf, dst)
            dst.flush()
            dst.seek(0)
            with ZipFile(dst) as zf:
                map(cb,
                    map(lambda zi: zf.extract(zi, folder), zf.infolist()))

class MyHashObj(object):
    hashf = None

    def __init__(self, hashf='sha512'):
        self.md5 = hashlib.md5()
        if hasattr(hashlib, hashf):
            self.hashf = hashf
            setattr(self, hashf, getattr(hashlib, hashf)())

    def update(self, data):
        self.md5.update(data)
        if self.hashf:
            getattr(self, self.hashf).update(data)

    def auth(self):
        args = {'md5': self.md5.digest()}
        if self.hashf:
            args[self.hashf] = getattr(self, self.hashf).digest()
        return Auth(**args)

def dump_archive(archive, items, cert, cb=None, tmpdir='/tmp',
                 hashf='sha512'):
    name = archive_handle([archive, cert])
    cipher = get_cipher(archive, cert)
    hashobj = MyHashObj(hashf)

    path, writer = _writer(name, items,
                           cipher, tmpdir, hashobj)
    try:
        list(starmap(cb, writer))
    except Exception:
        if os.path.exists(path):
            os.unlink(path)  # don't leave trash
        raise
    return (name, path, hashobj.auth())

def walk_folders(folders):
    for folder in folders:
        if not os.path.isdir(folder):
            yield (folder, os.path.basename(folder))
        else:
            for root, _, fs in os.walk(folder):
                for f in fs:
                    path = os.path.join(root, f)
                    strip = os.path.dirname(folder)
                    rel = os.path.relpath(path, strip)
                    yield (path, get_unicode(rel))

def _writer(name, items, cipher, tmpdir, hashobj=None):
    tmpargs = {'delete': False,
               'suffix': ".longaccess",
               'dir': tmpdir,
               'prefix': name}
    dst = NamedTemporaryFile(**tmpargs)

    if sys.platform.startswith('win'):
        # windows has unicode file system api
        items = map(unicode, items)

    def _enc():
        # do it in two passes now as zip can't easily handle streaming
        tmpargs = {'prefix': name,
                   'dir': tmpdir}
        with NamedTemporaryFile(**tmpargs) as zf:
            with ZipFile(zf, 'w', ZIP_DEFLATED, True) as zpf:
                for path, rel in walk_folders(map(os.path.abspath, items)):
                    try:
                        zpf.write(path, rel.encode('utf-8'))
                    except Exception as e:
                        if not hasattr(e, 'filename'):
                            setattr(e, 'filename', path)
                        dst.close()
                        raise
                    yield (path, rel)
            zf.flush()
            zf.seek(0)
            yield (None, None)  # the end
            with CryptIO(dst, cipher, hashobj=hashobj) as fdst:
                copyfileobj(zf, fdst, 1024)
        dst.close()
    return (dst.name, _enc())
<commit_msg>return file handle in writer to close in case of exception<commit_after>import os
import sys
import re
import hashlib

from unidecode import unidecode
from datetime import date
from tempfile import NamedTemporaryFile
from lacli.adf import Auth, make_adf
from lacli.crypt import CryptIO
from lacli.cipher import get_cipher
from lacli.hash import HashIO
from lacli.enc import get_unicode
from shutil import copyfileobj
from zipfile import ZipFile, ZIP_DEFLATED
from itertools import starmap

_slugify_strip_re = re.compile(r'[^\w\s-]')
_slugify_hyphenate_re = re.compile(r'[-\s]+')

def _slugify(value):
    """
    Normalizes string, converts to lowercase, removes non-alpha characters,
    and converts spaces to hyphens.
    From Django's "django/template/defaultfilters.py".
    """
    if not isinstance(value, unicode):
        value = unicode(value)
    return unicode(_slugify_hyphenate_re.sub(
        '-', _slugify_strip_re.sub('', unidecode(value))
        ).strip().lower())

def archive_slug(archive):
    return "{}-{}".format(date.today().isoformat(),
                          _slugify(archive.title))

def archive_handle(docs):
    h = HashIO()
    make_adf(docs, out=h)
    return h.getvalue().encode('hex')

def restore_archive(archive, path, cert, folder, tmpdir, cb=None):
    cipher = get_cipher(archive, cert)
    with open(path, 'rb') as infile:
        with NamedTemporaryFile() as dst:
            with CryptIO(infile, cipher) as cf:
                copyfileobj(cf, dst)
            dst.flush()
            dst.seek(0)
            with ZipFile(dst) as zf:
                map(cb,
                    map(lambda zi: zf.extract(zi, folder), zf.infolist()))

class MyHashObj(object):
    hashf = None

    def __init__(self, hashf='sha512'):
        self.md5 = hashlib.md5()
        if hasattr(hashlib, hashf):
            self.hashf = hashf
            setattr(self, hashf, getattr(hashlib, hashf)())

    def update(self, data):
        self.md5.update(data)
        if self.hashf:
            getattr(self, self.hashf).update(data)

    def auth(self):
        args = {'md5': self.md5.digest()}
        if self.hashf:
            args[self.hashf] = getattr(self, self.hashf).digest()
        return Auth(**args)

def dump_archive(archive, items, cert, cb=None, tmpdir='/tmp',
                 hashf='sha512'):
    name = archive_handle([archive, cert])
    cipher = get_cipher(archive, cert)
    hashobj = MyHashObj(hashf)

    dst, writer = _writer(name, items,
                           cipher, tmpdir, hashobj)
    try:
        list(starmap(cb, writer))
    except Exception as e:
        path = dst.name
        dst.close()
        if os.path.exists(path):
            os.unlink(path)  # don't leave trash
        raise
    return (name, dst.name, hashobj.auth())

def walk_folders(folders):
    for folder in folders:
        if not os.path.isdir(folder):
            yield (folder, os.path.basename(folder))
        else:
            for root, _, fs in os.walk(folder):
                for f in fs:
                    path = os.path.join(root, f)
                    strip = os.path.dirname(folder)
                    rel = os.path.relpath(path, strip)
                    yield (path, get_unicode(rel))

def _writer(name, items, cipher, tmpdir, hashobj=None):
    tmpargs = {'delete': False,
               'suffix': ".longaccess",
               'dir': tmpdir,
               'prefix': name}
    dst = NamedTemporaryFile(**tmpargs)

    if sys.platform.startswith('win'):
        # windows has unicode file system api
        items = map(unicode, items)

    def _enc():
        # do it in two passes now as zip can't easily handle streaming
        tmpargs = {'prefix': name,
                   'dir': tmpdir}
        with NamedTemporaryFile(**tmpargs) as zf:
            with ZipFile(zf, 'w', ZIP_DEFLATED, True) as zpf:
                for path, rel in walk_folders(map(os.path.abspath, items)):
                    try:
                        zpf.write(path, rel.encode('utf-8'))
                    except Exception as e:
                        if not hasattr(e, 'filename'):
                            setattr(e, 'filename', path)
                        dst.close()
                        raise
                    yield (path, rel)
            zf.flush()
            zf.seek(0)
            yield (None, None)  # the end
            with CryptIO(dst, cipher, hashobj=hashobj) as fdst:
                copyfileobj(zf, fdst, 1024)
        dst.close()
    return (dst, _enc())
<|endoftext|>


<commit_before>"""hug/introspect.py

Defines built in hug functions to aid in introspection

Copyright (C) 2016  Timothy Edmund Crosley

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

"""
from __future__ import absolute_import

try:  # pragma: no cover
    import asyncio

    asyncio_iscoroutinefunction = asyncio.iscoroutinefunction

except ImportError:  # pragma: no cover

    def asyncio_iscoroutinefunction(function):
        return False

import inspect
from types import MethodType

def is_method(function):
    """Returns True if the passed in function is identified as a method (NOT a function)"""
    return isinstance(function, MethodType)

def arguments(function, extra_arguments=0):
    """Returns the name of all arguments a function takes"""
    if not hasattr(function, '__code__'):
        return ()

    if asyncio_iscoroutinefunction(function):  # pragma: no cover
        signature = inspect.signature(function)
        if extra_arguments:
            excluded_types = ()
        else:
            excluded_types = (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
        return [p.name for p in signature.parameters.values() if p.kind not in excluded_types]

    return function.__code__.co_varnames[:function.__code__.co_argcount + extra_arguments]

def takes_kwargs(function):
    """Returns True if the supplied function takes keyword arguments"""
    if asyncio_iscoroutinefunction(function):   # pragma: no cover
        signature = inspect.signature(function)
        return any(p for p in signature.parameters.values()
                   if p.kind == inspect.Parameter.VAR_KEYWORD)

    return bool(function.__code__.co_flags & 0x08)

def takes_kargs(function):
    """Returns True if the supplied functions takes extra non-keyword arguments"""
    if asyncio_iscoroutinefunction(function):   # pragma: no cover
        signature = inspect.signature(function)
        return any(p for p in signature.parameters.values()
                   if p.kind == inspect.Parameter.VAR_POSITIONAL)

    return bool(function.__code__.co_flags & 0x04)

def takes_arguments(function, *named_arguments):
    """Returns the arguments that a function takes from a list of requested arguments"""
    return set(named_arguments).intersection(arguments(function))

def takes_all_arguments(function, *named_arguments):
    """Returns True if all supplied arguments are found in the function"""
    return bool(takes_arguments(function, *named_arguments) == set(named_arguments))

def generate_accepted_kwargs(function, *named_arguments):
    """Dynamically creates a function that when called with dictionary of arguments will produce a kwarg that's
       compatible with the supplied function
    """
    if hasattr(function, '__code__') and takes_kwargs(function):
        function_takes_kwargs = True
        function_takes_arguments = []
    else:
        function_takes_kwargs = False
        function_takes_arguments = takes_arguments(function, *named_arguments)

    def accepted_kwargs(kwargs):
        if function_takes_kwargs:
            return kwargs
        elif function_takes_arguments:
            return {key: value for key, value in kwargs.items() if key in function_takes_arguments}
        else:
            return {}
    return accepted_kwargs
<commit_msg>Test removal of inspect usage<commit_after>"""hug/introspect.py

Defines built in hug functions to aid in introspection

Copyright (C) 2016  Timothy Edmund Crosley

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or
substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

"""
from __future__ import absolute_import

import inspect
from types import MethodType

def is_method(function):
    """Returns True if the passed in function is identified as a method (NOT a function)"""
    return isinstance(function, MethodType)

def is_coroutine(function):
    """Returns True if the passed in function is a coroutine"""
    return function.__code__.co_flags & 0x0080 or getattr(function, '_is_coroutine', False)

def arguments(function, extra_arguments=0):
    """Returns the name of all arguments a function takes"""
    if not hasattr(function, '__code__'):
        return ()

    """#if asyncio_iscoroutinefunction(function):  # pragma: no cover
        #signature = inspect.signature(function)
        #if extra_arguments:
            #excluded_types = ()
        #else:
            #excluded_types = (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
        #return [p.name for p in signature.parameters.values() if p.kind not in excluded_types]"""

    return function.__code__.co_varnames[:function.__code__.co_argcount + extra_arguments]

def takes_kwargs(function):
    """Returns True if the supplied function takes keyword arguments"""
    """#if asyncio_iscoroutinefunction(function):   # pragma: no cover
        #signature = inspect.signature(function)
        #return any(p for p in signature.parameters.values()
                   #if p.kind == inspect.Parameter.VAR_KEYWORD)"""

    return bool(function.__code__.co_flags & 0x08)

def takes_kargs(function):
    """Returns True if the supplied functions takes extra non-keyword arguments"""
    """#if asyncio_iscoroutinefunction(function):   # pragma: no cover
        #signature = inspect.signature(function)
        #return any(p for p in signature.parameters.values()
                   #if p.kind == inspect.Parameter.VAR_POSITIONAL)"""

    return bool(function.__code__.co_flags & 0x04)

def takes_arguments(function, *named_arguments):
    """Returns the arguments that a function takes from a list of requested arguments"""
    return set(named_arguments).intersection(arguments(function))

def takes_all_arguments(function, *named_arguments):
    """Returns True if all supplied arguments are found in the function"""
    return bool(takes_arguments(function, *named_arguments) == set(named_arguments))

def generate_accepted_kwargs(function, *named_arguments):
    """Dynamically creates a function that when called with dictionary of arguments will produce a kwarg that's
       compatible with the supplied function
    """
    if hasattr(function, '__code__') and takes_kwargs(function):
        function_takes_kwargs = True
        function_takes_arguments = []
    else:
        function_takes_kwargs = False
        function_takes_arguments = takes_arguments(function, *named_arguments)

    def accepted_kwargs(kwargs):
        if function_takes_kwargs:
            return kwargs
        elif function_takes_arguments:
            return {key: value for key, value in kwargs.items() if key in function_takes_arguments}
        else:
            return {}
    return accepted_kwargs
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file as well as the whole tsfresh package are licenced under the MIT licence (see the LICENCE.txt)
# Maximilian Christ (maximilianchrist.com), Blue Yonder Gmbh, 2016
"""
This module contains the main function to interact with tsfresh: extract features
"""

from __future__ import absolute_import, division

import itertools
import logging
import warnings
from functools import partial
from multiprocessing import Pool

import numpy as np
import pandas as pd
from tqdm import tqdm

from tsfresh import defaults
from tsfresh.feature_extraction import feature_calculators
from tsfresh.feature_extraction.settings import ComprehensiveFCParameters
from tsfresh.utilities import dataframe_functions, profiling
from tsfresh.utilities.string_manipulation import convert_to_output_format

_logger = logging.getLogger(__name__)

def extract_features(timeseries_container, default_fc_parameters=None,
                     kind_to_fc_parameters=None,
                     column_id=None, column_sort=None, column_kind=None, column_value=None,
                     chunksize=defaults.CHUNKSIZE,
                     n_jobs=defaults.N_PROCESSES, show_warnings=defaults.SHOW_WARNINGS,
                     disable_progressbar=defaults.DISABLE_PROGRESSBAR,
                     impute_function=defaults.IMPUTE_FUNCTION,
                     profile=defaults.PROFILING,
                     profiling_filename=defaults.PROFILING_FILENAME,
                     profiling_sorting=defaults.PROFILING_SORTING):
    """
    Extract features from

    * a :class:`pandas.DataFrame` containing the different time series

    or

    * a dictionary of :class:`pandas.DataFrame` each containing one type of time series

    In both cases a :class:`pandas.DataFrame` with the calculated features will be returned.

    For a list of all the calculated time series features, please see the
    :class:`~tsfresh.feature_extraction.settings.ComprehensiveFCParameters` class,
    which is used to control which features with which parameters are calculated.

    For a detailed explanation of the different parameters and data formats please see :ref:`data-formats-label`.

    Examples
    ========

    >>> from tsfresh.examples import load_robot_execution_failures
    >>> from tsfresh import extract_features
    >>> df, _ = load_robot_execution_failures()
    >>> X = extract_features(df, column_id='id', column_sort='time')

    :param timeseries_container: The pandas.DataFrame with the time series to compute the features for, or a
            dictionary of pandas.DataFrames.
    :type timeseries_container: pandas.DataFrame or dict

    :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names
           which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for
           more information.
    :type default_fc_parameters: dict

    :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for
            default_fc_parameters. If you put a kind as a key here, the fc_parameters
            object (which is the value), will be used instead of the default_fc_parameters.
    :type kind_to_fc_parameters: dict

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_sort: The name of the sort column.
    :type column_sort: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
    :type n_jobs: int

    :param chunksize: The size of one chunk for the parallelisation
    :type chunksize: None or int

    :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
    :type show_warnings: bool

    :param disable_progressbar: Do not show a progressbar while doing the calculation.
    :type disable_progressbar: bool

    :param impute_function: None, if no imputing should happen or the function to call for imputing.
    :type impute_function: None or function

    :param profile: Turn on profiling during feature extraction
    :type profile: bool

    :param profiling_sorting: How to sort the profiling results (see the documentation of the profiling package for
           more information)
    :type profiling_sorting: basestring

    :param profiling_filename: Where to save the profiling results.
    :type profiling_filename: basestring

    :return: The (maybe imputed) DataFrame containing extracted features.
    :rtype: pandas.DataFrame
    """
    import logging
    logging.basicConfig()

    # Always use the standardized way of storing the data.
    # See the function normalize_input_to_internal_representation for more information.
    df_melt, column_id, column_kind, column_value = \
        dataframe_functions._normalize_input_to_internal_representation(timeseries_container=timeseries_container,
                                                                        column_id=column_id, column_kind=column_kind,
                                                                        column_sort=column_sort,
                                                                        column_value=column_value)

    # Use the standard setting if the user did not supply ones himself.
    if default_fc_parameters is None:
        default_fc_parameters = ComprehensiveFCParameters()

    # If requested, do profiling (advanced feature)
    if profile:
        profiler = profiling.start_profiling()

    with warnings.catch_warnings():
        if not show_warnings:
            warnings.simplefilter("ignore")
        else:
            warnings.simplefilter("default")

        result = _do_extraction(df=df_melt,
                                column_id=column_id, column_value=column_value, column_kind=column_kind,
                                n_jobs=n_jobs, chunksize=chunksize,
                                disable_progressbar=disable_progressbar,
                                default_fc_parameters=default_fc_parameters, kind_to_fc_parameters=kind_to_fc_parameters)

        # Impute the result if requested
        if impute_function is not None:
            impute_function(result)

    # Turn off profiling if it was turned on
    if profile:
        profiling.end_profiling(profiler, filename=profiling_filename,
                                sorting=profiling_sorting)

    return result

def _do_extraction(df, column_id, column_value, column_kind,
                   default_fc_parameters, kind_to_fc_parameters,
                   n_jobs, chunksize, disable_progressbar):
    """
    Wrapper around the _do_extraction_on_chunk, which calls it on all chunks in the data frame.
    A chunk is a subset of the data, with a given kind and id - so a single time series.

    The data is separated out into those single time series and the _do_extraction_on_chunk is
    called on each of them. The results are then combined into a single pandas DataFrame.

    The call is either happening in parallel or not and is showing a progress bar or not depending
    on the given flags.

    :param df: The dataframe in the normalized format which is used for extraction.
    :type df: pd.DataFrame

    :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names
           which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for
           more information.
    :type default_fc_parameters: dict

    :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for
            default_fc_parameters. If you put a kind as a key here, the fc_parameters
            object (which is the value), will be used instead of the default_fc_parameters.
    :type kind_to_fc_parameters: dict

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param chunksize: The size of one chunk for the parallelization
    :type chunksize: None or int

    :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
    :type n_jobs: int

    :param disable_progressbar: Do not show a progressbar while doing the calculation.
    :type disable_progressbar: bool

    :return: the extracted features
    :rtype: pd.DataFrame
    """
    data_in_chunks = [x + (y,) for x, y in df.groupby([column_id, column_kind])[column_value]]

    total_number_of_expected_results = len(data_in_chunks)

    if n_jobs == 0:
        map_function = map
    else:
        pool = Pool(n_jobs)

        if not chunksize:
            chunksize = _calculate_best_chunksize(data_in_chunks, n_jobs)

        map_function = partial(pool.imap_unordered, chunksize=chunksize)

    extraction_function = partial(_do_extraction_on_chunk,
                                  default_fc_parameters=default_fc_parameters,
                                  kind_to_fc_parameters=kind_to_fc_parameters)

    # Map over all those chunks and extract the features on them
    result = tqdm(map_function(extraction_function, data_in_chunks),
                  total=total_number_of_expected_results,
                  desc="Feature Extraction", disable=disable_progressbar)

    # Flatten out the lists
    result = itertools.chain.from_iterable(result)

    # Return a dataframe in the typical form (id as index and feature names as columns)
    result = pd.DataFrame(list(result), dtype=np.float)

    if len(result) != 0:
        result = result.pivot("id", "variable", "value")
        result.index = result.index.astype(df[column_id].dtype)

    return result

def _do_extraction_on_chunk(chunk, default_fc_parameters, kind_to_fc_parameters):
    """
    Main function of this module: use the feature calculators defined in the
    default_fc_parameters or kind_to_fc_parameters parameters and extract all
    features on the chunk.

    The chunk consists of the chunk id, the chunk kind and the data (as a Series),
    which is then converted to a numpy array - so a single time series.

    Returned is a list of the extracted features. Each one is a dictionary consisting of
    { "variable": the feature name in the format <kind>__<feature>__<parameters>,
      "value": the number value of the feature,
      "id": the id of the chunk }

    The <parameters> are in the form described in :mod:`~tsfresh.utilities.string_manipulation`.

    :param chunk: A tuple of sample_id, kind, data
    :param default_fc_parameters: A dictionary of feature calculators.
    :param kind_to_fc_parameters: A dictionary of fc_parameters for special kinds or None.
    :return: A list of calculated features.
    """
    sample_id, kind, data = chunk
    data = data.values

    if kind_to_fc_parameters and kind in kind_to_fc_parameters:
        fc_parameters = kind_to_fc_parameters[kind]
    else:
        fc_parameters = default_fc_parameters

    def _f():
        for function_name, parameter_list in fc_parameters.items():
            func = getattr(feature_calculators, function_name)

            if func.fctype == "combiner":
                result = func(data, param=parameter_list)
            else:
                if parameter_list:
                    result = ((convert_to_output_format(param), func(data, **param)) for param in parameter_list)
                else:
                    result = [("", func(data))]

            for key, item in result:
                feature_name = str(kind) + "__" + func.__name__
                if key:
                    feature_name += "__" + str(key)
                yield {"variable": feature_name, "value": item, "id": sample_id}

    return list(_f())

def _calculate_best_chunksize(iterable_list, n_jobs):
    """
    Helper function to calculate the best chunksize for a given number of elements to calculate.

    The formula is more or less an empirical result.
    :param iterable_list: A list which defines how many calculations there need to be.
    :param n_jobs: The number of processes that will be used in the calculation.
    :return: The chunksize which should be used.

    TODO: Investigate which is the best chunk size for different settings.
    """
    chunksize, extra = divmod(len(iterable_list), n_jobs * 5)
    if extra:
        chunksize += 1
    return chunksize
<commit_msg>Corrected a type<commit_after># -*- coding: utf-8 -*-
# This file as well as the whole tsfresh package are licenced under the MIT licence (see the LICENCE.txt)
# Maximilian Christ (maximilianchrist.com), Blue Yonder Gmbh, 2016
"""
This module contains the main function to interact with tsfresh: extract features
"""

from __future__ import absolute_import, division

import itertools
import logging
import warnings
from functools import partial
from multiprocessing import Pool

import numpy as np
import pandas as pd
from tqdm import tqdm

from tsfresh import defaults
from tsfresh.feature_extraction import feature_calculators
from tsfresh.feature_extraction.settings import ComprehensiveFCParameters
from tsfresh.utilities import dataframe_functions, profiling
from tsfresh.utilities.string_manipulation import convert_to_output_format

_logger = logging.getLogger(__name__)

def extract_features(timeseries_container, default_fc_parameters=None,
                     kind_to_fc_parameters=None,
                     column_id=None, column_sort=None, column_kind=None, column_value=None,
                     chunksize=defaults.CHUNKSIZE,
                     n_jobs=defaults.N_PROCESSES, show_warnings=defaults.SHOW_WARNINGS,
                     disable_progressbar=defaults.DISABLE_PROGRESSBAR,
                     impute_function=defaults.IMPUTE_FUNCTION,
                     profile=defaults.PROFILING,
                     profiling_filename=defaults.PROFILING_FILENAME,
                     profiling_sorting=defaults.PROFILING_SORTING):
    """
    Extract features from

    * a :class:`pandas.DataFrame` containing the different time series

    or

    * a dictionary of :class:`pandas.DataFrame` each containing one type of time series

    In both cases a :class:`pandas.DataFrame` with the calculated features will be returned.

    For a list of all the calculated time series features, please see the
    :class:`~tsfresh.feature_extraction.settings.ComprehensiveFCParameters` class,
    which is used to control which features with which parameters are calculated.

    For a detailed explanation of the different parameters and data formats please see :ref:`data-formats-label`.

    Examples
    ========

    >>> from tsfresh.examples import load_robot_execution_failures
    >>> from tsfresh import extract_features
    >>> df, _ = load_robot_execution_failures()
    >>> X = extract_features(df, column_id='id', column_sort='time')

    :param timeseries_container: The pandas.DataFrame with the time series to compute the features for, or a
            dictionary of pandas.DataFrames.
    :type timeseries_container: pandas.DataFrame or dict

    :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names
           which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for
           more information.
    :type default_fc_parameters: dict

    :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for
            default_fc_parameters. If you put a kind as a key here, the fc_parameters
            object (which is the value), will be used instead of the default_fc_parameters.
    :type kind_to_fc_parameters: dict

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_sort: The name of the sort column.
    :type column_sort: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
    :type n_jobs: int

    :param chunksize: The size of one chunk for the parallelisation
    :type chunksize: None or int

    :param: show_warnings: Show warnings during the feature extraction (needed for debugging of calculators).
    :type show_warnings: bool

    :param disable_progressbar: Do not show a progressbar while doing the calculation.
    :type disable_progressbar: bool

    :param impute_function: None, if no imputing should happen or the function to call for imputing.
    :type impute_function: None or callable

    :param profile: Turn on profiling during feature extraction
    :type profile: bool

    :param profiling_sorting: How to sort the profiling results (see the documentation of the profiling package for
           more information)
    :type profiling_sorting: basestring

    :param profiling_filename: Where to save the profiling results.
    :type profiling_filename: basestring

    :return: The (maybe imputed) DataFrame containing extracted features.
    :rtype: pandas.DataFrame
    """
    import logging
    logging.basicConfig()

    # Always use the standardized way of storing the data.
    # See the function normalize_input_to_internal_representation for more information.
    df_melt, column_id, column_kind, column_value = \
        dataframe_functions._normalize_input_to_internal_representation(timeseries_container=timeseries_container,
                                                                        column_id=column_id, column_kind=column_kind,
                                                                        column_sort=column_sort,
                                                                        column_value=column_value)

    # Use the standard setting if the user did not supply ones himself.
    if default_fc_parameters is None:
        default_fc_parameters = ComprehensiveFCParameters()

    # If requested, do profiling (advanced feature)
    if profile:
        profiler = profiling.start_profiling()

    with warnings.catch_warnings():
        if not show_warnings:
            warnings.simplefilter("ignore")
        else:
            warnings.simplefilter("default")

        result = _do_extraction(df=df_melt,
                                column_id=column_id, column_value=column_value, column_kind=column_kind,
                                n_jobs=n_jobs, chunksize=chunksize,
                                disable_progressbar=disable_progressbar,
                                default_fc_parameters=default_fc_parameters, kind_to_fc_parameters=kind_to_fc_parameters)

        # Impute the result if requested
        if impute_function is not None:
            impute_function(result)

    # Turn off profiling if it was turned on
    if profile:
        profiling.end_profiling(profiler, filename=profiling_filename,
                                sorting=profiling_sorting)

    return result

def _do_extraction(df, column_id, column_value, column_kind,
                   default_fc_parameters, kind_to_fc_parameters,
                   n_jobs, chunksize, disable_progressbar):
    """
    Wrapper around the _do_extraction_on_chunk, which calls it on all chunks in the data frame.
    A chunk is a subset of the data, with a given kind and id - so a single time series.

    The data is separated out into those single time series and the _do_extraction_on_chunk is
    called on each of them. The results are then combined into a single pandas DataFrame.

    The call is either happening in parallel or not and is showing a progress bar or not depending
    on the given flags.

    :param df: The dataframe in the normalized format which is used for extraction.
    :type df: pd.DataFrame

    :param default_fc_parameters: mapping from feature calculator names to parameters. Only those names
           which are keys in this dict will be calculated. See the class:`ComprehensiveFCParameters` for
           more information.
    :type default_fc_parameters: dict

    :param kind_to_fc_parameters: mapping from kind names to objects of the same type as the ones for
            default_fc_parameters. If you put a kind as a key here, the fc_parameters
            object (which is the value), will be used instead of the default_fc_parameters.
    :type kind_to_fc_parameters: dict

    :param column_id: The name of the id column to group by.
    :type column_id: str

    :param column_kind: The name of the column keeping record on the kind of the value.
    :type column_kind: str

    :param column_value: The name for the column keeping the value itself.
    :type column_value: str

    :param chunksize: The size of one chunk for the parallelization
    :type chunksize: None or int

    :param n_jobs: The number of processes to use for parallelization. If zero, no parallelization is used.
    :type n_jobs: int

    :param disable_progressbar: Do not show a progressbar while doing the calculation.
    :type disable_progressbar: bool

    :return: the extracted features
    :rtype: pd.DataFrame
    """
    data_in_chunks = [x + (y,) for x, y in df.groupby([column_id, column_kind])[column_value]]

    total_number_of_expected_results = len(data_in_chunks)

    if n_jobs == 0:
        map_function = map
    else:
        pool = Pool(n_jobs)

        if not chunksize:
            chunksize = _calculate_best_chunksize(data_in_chunks, n_jobs)

        map_function = partial(pool.imap_unordered, chunksize=chunksize)

    extraction_function = partial(_do_extraction_on_chunk,
                                  default_fc_parameters=default_fc_parameters,
                                  kind_to_fc_parameters=kind_to_fc_parameters)

    # Map over all those chunks and extract the features on them
    result = tqdm(map_function(extraction_function, data_in_chunks),
                  total=total_number_of_expected_results,
                  desc="Feature Extraction", disable=disable_progressbar)

    # Flatten out the lists
    result = itertools.chain.from_iterable(result)

    # Return a dataframe in the typical form (id as index and feature names as columns)
    result = pd.DataFrame(list(result), dtype=np.float)

    if len(result) != 0:
        result = result.pivot("id", "variable", "value")
        result.index = result.index.astype(df[column_id].dtype)

    return result

def _do_extraction_on_chunk(chunk, default_fc_parameters, kind_to_fc_parameters):
    """
    Main function of this module: use the feature calculators defined in the
    default_fc_parameters or kind_to_fc_parameters parameters and extract all
    features on the chunk.

    The chunk consists of the chunk id, the chunk kind and the data (as a Series),
    which is then converted to a numpy array - so a single time series.

    Returned is a list of the extracted features. Each one is a dictionary consisting of
    { "variable": the feature name in the format <kind>__<feature>__<parameters>,
      "value": the number value of the feature,
      "id": the id of the chunk }

    The <parameters> are in the form described in :mod:`~tsfresh.utilities.string_manipulation`.

    :param chunk: A tuple of sample_id, kind, data
    :param default_fc_parameters: A dictionary of feature calculators.
    :param kind_to_fc_parameters: A dictionary of fc_parameters for special kinds or None.
    :return: A list of calculated features.
    """
    sample_id, kind, data = chunk
    data = data.values

    if kind_to_fc_parameters and kind in kind_to_fc_parameters:
        fc_parameters = kind_to_fc_parameters[kind]
    else:
        fc_parameters = default_fc_parameters

    def _f():
        for function_name, parameter_list in fc_parameters.items():
            func = getattr(feature_calculators, function_name)

            if func.fctype == "combiner":
                result = func(data, param=parameter_list)
            else:
                if parameter_list:
                    result = ((convert_to_output_format(param), func(data, **param)) for param in parameter_list)
                else:
                    result = [("", func(data))]

            for key, item in result:
                feature_name = str(kind) + "__" + func.__name__
                if key:
                    feature_name += "__" + str(key)
                yield {"variable": feature_name, "value": item, "id": sample_id}

    return list(_f())

def _calculate_best_chunksize(iterable_list, n_jobs):
    """
    Helper function to calculate the best chunksize for a given number of elements to calculate.

    The formula is more or less an empirical result.
    :param iterable_list: A list which defines how many calculations there need to be.
    :param n_jobs: The number of processes that will be used in the calculation.
    :return: The chunksize which should be used.

    TODO: Investigate which is the best chunk size for different settings.
    """
    chunksize, extra = divmod(len(iterable_list), n_jobs * 5)
    if extra:
        chunksize += 1
    return chunksize
<|endoftext|>


<commit_before>import operator

from django.db import models

class Season(models.Model):
    name = models.CharField(max_length=150)
    start_date = models.DateField('Start date')
    end_date = models.DateField('End date')

    def __unicode__(self):
        return self.name

    def get_stats(self):
        player_count = 0
        results_count = 0
        total_games_count = 0.0
        current_leaders = {}
        for ladder in self.ladder_set.all():
            player_count += ladder.league_set.count()
            results_count += ladder.result_set.count() / 2
            total_games_count += (ladder.league_set.count() * (ladder.league_set.count() - 1)) / 2
            current_leaders[ladder.division] = ladder.get_leader()

        percentage_played = (results_count / total_games_count) * 100

        return {
            'percentage_played': percentage_played,
            'total_games_count': total_games_count,
            'results_count': results_count,
            'player_count': player_count,
            'current_leaders': current_leaders.items()
        }

class Player(models.Model):
    first_name = models.CharField(max_length=100)
    last_name = models.CharField(max_length=100)
    home_phone = models.CharField(max_length=100)
    mobile_phone = models.CharField(max_length=100)
    email = models.CharField(max_length=100)
    junior = models.BooleanField(default=False)

    def __unicode__(self):
        return self.first_name + ' ' + self.last_name

class Ladder(models.Model):
    season = models.ForeignKey(Season)
    division = models.IntegerField(default=0)
    ladder_type = models.CharField(max_length=100)

    def __unicode__(self):
        return self.season.name + ' division: ' + str(self.division)

    def get_leader(self):
        totals = {}
        for result in self.result_set.filter(ladder=self):
            try:
                if result.result == 9:
                    totals[result.player] += int(result.result) + 3
                else:
                    totals[result.player] += int(result.result) + 1
            except KeyError:
                if result.result == 9:
                    totals[result.player] = int(result.result) + 3
                else:
                    totals[result.player] = int(result.result) + 1

        if totals:
            player = max(totals.iteritems(), key=operator.itemgetter(1))[0]
        else:
            player = {}

        return {'player': player, 'total': totals[player]}

    def get_latest_results(self):
        results = {}
        for result in self.result_set.filter(ladder=self).order_by('-date_added')[:10]:  # [:10] to limit to 5

            opponent = self.result_set.filter(ladder=self, player=result.opponent, opponent=result.player)[0]
            player_opponent_index = ''.join(str(e) for e in sorted([result.player.id, opponent.player.id]))
            try:
                if results[player_opponent_index]:
                    continue
            except KeyError:
                results[player_opponent_index] = {'player': result.player, 'player_result': result.result,
                                                  'opponent_result': opponent.result, 'opponent': opponent.player,
                                                  'date_added': result.date_added}

        ordered_results = {}
        i = 0
        for key in sorted(results, key=lambda x: (results[x]['date_added']), reverse=True):
            ordered_results[i] = results[key]
            i += 1

        return ordered_results.items()

    def get_stats(self):
        total_matches_played = 0.00
        total_matches = self.league_set.count() * (self.result_set.count() - 1) / 2
        total_matches_played += self.result_set.count() / 2
        perc_matches_played = (total_matches_played / total_matches) * 100

        return {
            'total_matches_played': total_matches_played,
            'total_matches': total_matches,
            'perc_matches_played': perc_matches_played
        }

class League(models.Model):
    ladder = models.ForeignKey(Ladder)
    player = models.ForeignKey(Player)
    sort_order = models.IntegerField(default=0)

    def __unicode__(self):
        return self.player.first_name + ' ' +  self.player.last_name

class Result(models.Model):
    ladder = models.ForeignKey(Ladder)
    player = models.ForeignKey(Player, related_name='result_player')
    opponent = models.ForeignKey(Player, related_name='result_opponent')
    result = models.IntegerField()
    date_added = models.DateField('Date added')

    def __unicode__(self):
        return self.player.first_name + ' ' + self.player.last_name + ' vs ' + self.opponent.first_name + ' ' + self.opponent.last_name + ' score: ' + str(
            self.result)
<commit_msg>Make some player details non-required<commit_after>import operator

from django.db import models

class Season(models.Model):
    name = models.CharField(max_length=150)
    start_date = models.DateField('Start date')
    end_date = models.DateField('End date')

    def __unicode__(self):
        return self.name

    def get_stats(self):
        player_count = 0
        results_count = 0
        total_games_count = 0.0
        current_leaders = {}
        for ladder in self.ladder_set.all():
            player_count += ladder.league_set.count()
            results_count += ladder.result_set.count() / 2
            total_games_count += (ladder.league_set.count() * (ladder.league_set.count() - 1)) / 2
            current_leaders[ladder.division] = ladder.get_leader()

        percentage_played = (results_count / total_games_count) * 100

        return {
            'percentage_played': percentage_played,
            'total_games_count': total_games_count,
            'results_count': results_count,
            'player_count': player_count,
            'current_leaders': current_leaders.items()
        }

class Player(models.Model):
    first_name = models.CharField(max_length=100)
    last_name = models.CharField(max_length=100)
    home_phone = models.CharField(max_length=100, blank=True)
    mobile_phone = models.CharField(max_length=100, blank=True)
    email = models.CharField(max_length=100, blank=True)
    junior = models.BooleanField(default=False)

    def __unicode__(self):
        return self.first_name + ' ' + self.last_name

class Ladder(models.Model):
    season = models.ForeignKey(Season)
    division = models.IntegerField(default=0)
    ladder_type = models.CharField(max_length=100)

    def __unicode__(self):
        return self.season.name + ' division: ' + str(self.division)

    def get_leader(self):
        totals = {}
        for result in self.result_set.filter(ladder=self):
            try:
                if result.result == 9:
                    totals[result.player] += int(result.result) + 3
                else:
                    totals[result.player] += int(result.result) + 1
            except KeyError:
                if result.result == 9:
                    totals[result.player] = int(result.result) + 3
                else:
                    totals[result.player] = int(result.result) + 1

        if totals:
            player = max(totals.iteritems(), key=operator.itemgetter(1))[0]
        else:
            player = {}

        return {'player': player, 'total': totals[player]}

    def get_latest_results(self):
        results = {}
        for result in self.result_set.filter(ladder=self).order_by('-date_added')[:10]:  # [:10] to limit to 5

            opponent = self.result_set.filter(ladder=self, player=result.opponent, opponent=result.player)[0]
            player_opponent_index = ''.join(str(e) for e in sorted([result.player.id, opponent.player.id]))
            try:
                if results[player_opponent_index]:
                    continue
            except KeyError:
                results[player_opponent_index] = {'player': result.player, 'player_result': result.result,
                                                  'opponent_result': opponent.result, 'opponent': opponent.player,
                                                  'date_added': result.date_added}

        ordered_results = {}
        i = 0
        for key in sorted(results, key=lambda x: (results[x]['date_added']), reverse=True):
            ordered_results[i] = results[key]
            i += 1

        return ordered_results.items()

    def get_stats(self):
        total_matches_played = 0.00
        total_matches = self.league_set.count() * (self.result_set.count() - 1) / 2
        total_matches_played += self.result_set.count() / 2
        perc_matches_played = (total_matches_played / total_matches) * 100

        return {
            'total_matches_played': total_matches_played,
            'total_matches': total_matches,
            'perc_matches_played': perc_matches_played
        }

class League(models.Model):
    ladder = models.ForeignKey(Ladder)
    player = models.ForeignKey(Player)
    sort_order = models.IntegerField(default=0)

    def __unicode__(self):
        return self.player.first_name + ' ' +  self.player.last_name

class Result(models.Model):
    ladder = models.ForeignKey(Ladder)
    player = models.ForeignKey(Player, related_name='result_player')
    opponent = models.ForeignKey(Player, related_name='result_opponent')
    result = models.IntegerField()
    date_added = models.DateField('Date added')

    def __unicode__(self):
        return self.player.first_name + ' ' + self.player.last_name + ' vs ' + self.opponent.first_name + ' ' + self.opponent.last_name + ' score: ' + str(
            self.result)
<|endoftext|>


<commit_before># Pansharpened Image Process using Rasterio
# Landsat Util
# License: CC0 1.0 Universal

import warnings
import sys
from os.path import join
import tarfile
import glob
import subprocess

import numpy
import rasterio
from rasterio.warp import reproject, RESAMPLING, transform

from skimage import transform as sktransform
from skimage.util import img_as_ubyte
from skimage.exposure import rescale_intensity

import settings
from mixins import VerbosityMixin
from utils import get_file, timer, check_create_folder, exit

class FileDoesNotExist(Exception):
    """ Exception to be used when the file does not exist. """
    pass

class Process(VerbosityMixin):
    """
    Image procssing class

    To initiate the following parameters must be passed:

    :param path:
        Path of the image.
    :type path:
        String
    :param bands:
        The band sequence for the final image. Must be a python list. (optional)
    :type bands:
        List
    :param dst_path:
        Path to the folder where the image should be stored. (optional)
    :type dst_path:
        String
    :param verbose:
        Whether the output should be verbose. Default is False.
    :type verbose:
        boolean

    """

    def __init__(self, path, bands=None, dst_path=None, verbose=False):

        self.projection = {'init': 'epsg:3857'}
        self.dst_crs = {'init': u'epsg:3857'}
        self.scene = get_file(path).split('.')[0]
        self.bands = bands if isinstance(bands, list) else [4, 3, 2]

        # Landsat source path
        self.src_path = path.replace(get_file(path), '')

        # Build destination folder if doesn't exits
        self.dst_path = dst_path if dst_path else settings.PROCESSED_IMAGE
        self.dst_path = check_create_folder(join(self.dst_path, self.scene))
        self.verbose = verbose

        # Path to the unzipped folder
        self.scene_path = join(self.src_path, self.scene)

        if self._check_if_zipped(path):
            self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene)

        self.bands_path = []
        for band in self.bands:
            self.bands_path.append(join(self.scene_path, self._get_full_filename(band)))

    def run(self, pansharpen=True):
        """ Executes the image processing.

        :param pansharpen:
            Whether the process should also run pansharpenning. Default is True
        :type pansharpen:
            boolean

        :returns:
            (String) the path to the processed image
        """

        self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True)

        # Read cloud coverage from mtl file
        cloud_cover = 0
        try:
            with open(self.scene_path + '/' + self.scene + '_MTL.txt', 'rU') as mtl:
                lines = mtl.readlines()
                for line in lines:
                    if 'CLOUD_COVER' in line:
                        cloud_cover = float(line.replace('CLOUD_COVER = ', ''))
                        break
        except IOError:
            pass

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            with rasterio.drivers():
                bands = []

                # Add band 8 for pansharpenning
                if pansharpen:
                    self.bands.append(8)

                bands_path = []

                for band in self.bands:
                    bands_path.append(join(self.scene_path, self._get_full_filename(band)))

                try:
                    for i, band in enumerate(self.bands):
                        bands.append(self._read_band(bands_path[i]))
                except IOError as e:
                    exit(e.message, 1)

                src = rasterio.open(bands_path[-1])

                # Get pixel size from source
                self.pixel = src.affine[0]

                # Only collect src data that is needed and delete the rest
                src_data = {
                    'transform': src.transform,
                    'crs': src.crs,
                    'affine': src.affine,
                    'shape': src.shape
                }
                del src

                crn = self._get_boundaries(src_data)

                dst_shape = src_data['shape']
                dst_corner_ys = [crn[k]['y'][1][0] for k in crn.keys()]
                dst_corner_xs = [crn[k]['x'][1][0] for k in crn.keys()]
                y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / dst_shape[0]
                x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / dst_shape[1]

                dst_transform = (min(dst_corner_xs),
                                 x_pixel,
                                 0.0,
                                 max(dst_corner_ys),
                                 0.0,
                                 -y_pixel)
                # Delete crn since no longer needed
                del crn

                new_bands = []
                for i in range(0, 3):
                    new_bands.append(numpy.empty(dst_shape, dtype=numpy.uint16))

                if pansharpen:
                    bands[:3] = self._rescale(bands[:3])
                    new_bands.append(numpy.empty(dst_shape, dtype=numpy.uint16))

                self.output("Projecting", normal=True, arrow=True)
                for i, band in enumerate(bands):
                    self.output("band %s" % self.bands[i], normal=True, color='green', indent=1)
                    reproject(band, new_bands[i], src_transform=src_data['transform'], src_crs=src_data['crs'],
                              dst_transform=dst_transform, dst_crs=self.dst_crs, resampling=RESAMPLING.nearest)

                # Bands are no longer needed
                del bands

                if pansharpen:
                    new_bands = self._pansharpenning(new_bands)
                    del self.bands[3]

                self.output("Final Steps", normal=True, arrow=True)

                output_file = '%s_bands_%s' % (self.scene, "".join(map(str, self.bands)))

                if pansharpen:
                    output_file += '_pan'

                output_file += '.TIF'
                output_file = join(self.dst_path, output_file)

                output = rasterio.open(output_file, 'w', driver='GTiff',
                                       width=dst_shape[1], height=dst_shape[0],
                                       count=3, dtype=numpy.uint8,
                                       nodata=0, transform=dst_transform, photometric='RGB',
                                       crs=self.dst_crs)

                for i, band in enumerate(new_bands):
                    # Color Correction
                    band = self._color_correction(band, self.bands[i], 0, cloud_cover)

                    output.write_band(i+1, img_as_ubyte(band))

                    new_bands[i] = None
                self.output("Writing to file", normal=True, color='green', indent=1)
                return output_file

    def _pansharpenning(self, bands):

        self.output("Pansharpening", normal=True, arrow=True)
        # Pan sharpening
        m = sum(bands[:3])
        m = m + 0.1

        self.output("calculating pan ratio", normal=True, color='green', indent=1)
        pan = 1/m * bands[-1]

        del m
        del bands[3]
        self.output("computing bands", normal=True, color='green', indent=1)

        for i, band in enumerate(bands):
            bands[i] = band * pan

        del pan

        return bands

    def _color_correction(self, band, band_id, low, cloud_cover):
        band = band.astype(numpy.uint16)

        self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1)
        p_low, cloud_cut_low = self._percent_cut(band, low, 100 - (cloud_cover * 3 / 4))
        temp = numpy.zeros(numpy.shape(band), dtype=numpy.uint16)
        cloud_divide = 65000 - cloud_cover * 100
        mask = numpy.logical_and(band < cloud_cut_low, band > 0)
        temp[mask] = rescale_intensity(band[mask], in_range=(p_low, cloud_cut_low), out_range=(256, cloud_divide))
        temp[band >= cloud_cut_low] = rescale_intensity(band[band >= cloud_cut_low], out_range=(cloud_divide, 65535))
        return temp

    def _read_band(self, band_path):
        """ Reads a band with rasterio """
        return rasterio.open(band_path).read_band(1)

    def _rescale(self, bands):
        """ Rescale bands """
        self.output("Rescaling", normal=True, arrow=True)

        for key, band in enumerate(bands):
            self.output("band %s" % self.bands[key], normal=True, color='green', indent=1)
            bands[key] = sktransform.rescale(band, 2)
            bands[key] = (bands[key] * 65535).astype('uint16')

        return bands

    def _get_boundaries(self, src):

        self.output("Getting boundaries", normal=True, arrow=True)
        output = {'ul': {'x': [0, 0], 'y': [0, 0]},  # ul: upper left
                  'ur': {'x': [0, 0], 'y': [0, 0]},  # ur: upper right
                  'll': {'x': [0, 0], 'y': [0, 0]},  # ll: lower left
                  'lr': {'x': [0, 0], 'y': [0, 0]}}  # lr: lower right

        output['ul']['x'][0] = src['affine'][2]
        output['ul']['y'][0] = src['affine'][5]
        output['ur']['x'][0] = output['ul']['x'][0] + self.pixel * src['shape'][1]
        output['ur']['y'][0] = output['ul']['y'][0]
        output['ll']['x'][0] = output['ul']['x'][0]
        output['ll']['y'][0] = output['ul']['y'][0] - self.pixel * src['shape'][0]
        output['lr']['x'][0] = output['ul']['x'][0] + self.pixel * src['shape'][1]
        output['lr']['y'][0] = output['ul']['y'][0] - self.pixel * src['shape'][0]

        output['ul']['x'][1], output['ul']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['ul']['x'][0]],
                                                               [output['ul']['y'][0]])

        output['ur']['x'][1], output['ur']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['ur']['x'][0]],
                                                               [output['ur']['y'][0]])

        output['ll']['x'][1], output['ll']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['ll']['x'][0]],
                                                               [output['ll']['y'][0]])

        output['lr']['x'][1], output['lr']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['lr']['x'][0]],
                                                               [output['lr']['y'][0]])

        return output

    def _percent_cut(self, color, low, high):
        return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high))

    def _unzip(self, src, dst, scene):
        """ Unzip tar files """
        self.output("Unzipping %s - It might take some time" % scene, normal=True, arrow=True)

        try:
            tar = tarfile.open(src, 'r')
            tar.extractall(path=dst)
            tar.close()
        except tarfile.ReadError:
            check_create_folder(dst)
            subprocess.check_call(['tar', '-xf', src, '-C', dst])

    def _get_full_filename(self, band):

        base_file = '%s_B%s.*' % (self.scene, band)

        try:
            return glob.glob(join(self.scene_path, base_file))[0].split('/')[-1]
        except IndexError:
            raise FileDoesNotExist('%s does not exist' % '%s_B%s.*' % (self.scene, band))

    def _check_if_zipped(self, path):
        """ Checks if the filename shows a tar/zip file """
        filename = get_file(path).split('.')

        if filename[-1] in ['bz', 'bz2']:
            return True

        return False

if __name__ == '__main__':

    with timer():
        p = Process(sys.argv[1])

        print p.run(sys.argv[2] == 't')
<commit_msg>if unzipped file exist do not unzip unless forced to<commit_after># Pansharpened Image Process using Rasterio
# Landsat Util
# License: CC0 1.0 Universal

import warnings
import sys
from os.path import join, isdir
import tarfile
import glob
import subprocess

import numpy
import rasterio
from rasterio.warp import reproject, RESAMPLING, transform

from skimage import transform as sktransform
from skimage.util import img_as_ubyte
from skimage.exposure import rescale_intensity

import settings
from mixins import VerbosityMixin
from utils import get_file, timer, check_create_folder, exit

class FileDoesNotExist(Exception):
    """ Exception to be used when the file does not exist. """
    pass

class Process(VerbosityMixin):
    """
    Image procssing class

    To initiate the following parameters must be passed:

    :param path:
        Path of the image.
    :type path:
        String
    :param bands:
        The band sequence for the final image. Must be a python list. (optional)
    :type bands:
        List
    :param dst_path:
        Path to the folder where the image should be stored. (optional)
    :type dst_path:
        String
    :param verbose:
        Whether the output should be verbose. Default is False.
    :type verbose:
        boolean
    :param force_unzip:
        Whether to force unzip the tar file. Default is False
    :type force_unzip:
        boolean

    """

    def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=False):

        self.projection = {'init': 'epsg:3857'}
        self.dst_crs = {'init': u'epsg:3857'}
        self.scene = get_file(path).split('.')[0]
        self.bands = bands if isinstance(bands, list) else [4, 3, 2]

        # Landsat source path
        self.src_path = path.replace(get_file(path), '')

        # Build destination folder if doesn't exits
        self.dst_path = dst_path if dst_path else settings.PROCESSED_IMAGE
        self.dst_path = check_create_folder(join(self.dst_path, self.scene))
        self.verbose = verbose

        # Path to the unzipped folder
        self.scene_path = join(self.src_path, self.scene)

        if self._check_if_zipped(path):
            self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene, force_unzip)

        self.bands_path = []
        for band in self.bands:
            self.bands_path.append(join(self.scene_path, self._get_full_filename(band)))

    def run(self, pansharpen=True):
        """ Executes the image processing.

        :param pansharpen:
            Whether the process should also run pansharpenning. Default is True
        :type pansharpen:
            boolean

        :returns:
            (String) the path to the processed image
        """

        self.output("* Image processing started for bands %s" % "-".join(map(str, self.bands)), normal=True)

        # Read cloud coverage from mtl file
        cloud_cover = 0
        try:
            with open(self.scene_path + '/' + self.scene + '_MTL.txt', 'rU') as mtl:
                lines = mtl.readlines()
                for line in lines:
                    if 'CLOUD_COVER' in line:
                        cloud_cover = float(line.replace('CLOUD_COVER = ', ''))
                        break
        except IOError:
            pass

        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            with rasterio.drivers():
                bands = []

                # Add band 8 for pansharpenning
                if pansharpen:
                    self.bands.append(8)

                bands_path = []

                for band in self.bands:
                    bands_path.append(join(self.scene_path, self._get_full_filename(band)))

                try:
                    for i, band in enumerate(self.bands):
                        bands.append(self._read_band(bands_path[i]))
                except IOError as e:
                    exit(e.message, 1)

                src = rasterio.open(bands_path[-1])

                # Get pixel size from source
                self.pixel = src.affine[0]

                # Only collect src data that is needed and delete the rest
                src_data = {
                    'transform': src.transform,
                    'crs': src.crs,
                    'affine': src.affine,
                    'shape': src.shape
                }
                del src

                crn = self._get_boundaries(src_data)

                dst_shape = src_data['shape']
                dst_corner_ys = [crn[k]['y'][1][0] for k in crn.keys()]
                dst_corner_xs = [crn[k]['x'][1][0] for k in crn.keys()]
                y_pixel = abs(max(dst_corner_ys) - min(dst_corner_ys)) / dst_shape[0]
                x_pixel = abs(max(dst_corner_xs) - min(dst_corner_xs)) / dst_shape[1]

                dst_transform = (min(dst_corner_xs),
                                 x_pixel,
                                 0.0,
                                 max(dst_corner_ys),
                                 0.0,
                                 -y_pixel)
                # Delete crn since no longer needed
                del crn

                new_bands = []
                for i in range(0, 3):
                    new_bands.append(numpy.empty(dst_shape, dtype=numpy.uint16))

                if pansharpen:
                    bands[:3] = self._rescale(bands[:3])
                    new_bands.append(numpy.empty(dst_shape, dtype=numpy.uint16))

                self.output("Projecting", normal=True, arrow=True)
                for i, band in enumerate(bands):
                    self.output("band %s" % self.bands[i], normal=True, color='green', indent=1)
                    reproject(band, new_bands[i], src_transform=src_data['transform'], src_crs=src_data['crs'],
                              dst_transform=dst_transform, dst_crs=self.dst_crs, resampling=RESAMPLING.nearest)

                # Bands are no longer needed
                del bands

                if pansharpen:
                    new_bands = self._pansharpenning(new_bands)
                    del self.bands[3]

                self.output("Final Steps", normal=True, arrow=True)

                output_file = '%s_bands_%s' % (self.scene, "".join(map(str, self.bands)))

                if pansharpen:
                    output_file += '_pan'

                output_file += '.TIF'
                output_file = join(self.dst_path, output_file)

                output = rasterio.open(output_file, 'w', driver='GTiff',
                                       width=dst_shape[1], height=dst_shape[0],
                                       count=3, dtype=numpy.uint8,
                                       nodata=0, transform=dst_transform, photometric='RGB',
                                       crs=self.dst_crs)

                for i, band in enumerate(new_bands):
                    # Color Correction
                    band = self._color_correction(band, self.bands[i], 0, cloud_cover)

                    output.write_band(i+1, img_as_ubyte(band))

                    new_bands[i] = None
                self.output("Writing to file", normal=True, color='green', indent=1)
                return output_file

    def _pansharpenning(self, bands):

        self.output("Pansharpening", normal=True, arrow=True)
        # Pan sharpening
        m = sum(bands[:3])
        m = m + 0.1

        self.output("calculating pan ratio", normal=True, color='green', indent=1)
        pan = 1/m * bands[-1]

        del m
        del bands[3]
        self.output("computing bands", normal=True, color='green', indent=1)

        for i, band in enumerate(bands):
            bands[i] = band * pan

        del pan

        return bands

    def _color_correction(self, band, band_id, low, cloud_cover):
        band = band.astype(numpy.uint16)

        self.output("Color correcting band %s" % band_id, normal=True, color='green', indent=1)
        p_low, cloud_cut_low = self._percent_cut(band, low, 100 - (cloud_cover * 3 / 4))
        temp = numpy.zeros(numpy.shape(band), dtype=numpy.uint16)
        cloud_divide = 65000 - cloud_cover * 100
        mask = numpy.logical_and(band < cloud_cut_low, band > 0)
        temp[mask] = rescale_intensity(band[mask], in_range=(p_low, cloud_cut_low), out_range=(256, cloud_divide))
        temp[band >= cloud_cut_low] = rescale_intensity(band[band >= cloud_cut_low], out_range=(cloud_divide, 65535))
        return temp

    def _read_band(self, band_path):
        """ Reads a band with rasterio """
        return rasterio.open(band_path).read_band(1)

    def _rescale(self, bands):
        """ Rescale bands """
        self.output("Rescaling", normal=True, arrow=True)

        for key, band in enumerate(bands):
            self.output("band %s" % self.bands[key], normal=True, color='green', indent=1)
            bands[key] = sktransform.rescale(band, 2)
            bands[key] = (bands[key] * 65535).astype('uint16')

        return bands

    def _get_boundaries(self, src):

        self.output("Getting boundaries", normal=True, arrow=True)
        output = {'ul': {'x': [0, 0], 'y': [0, 0]},  # ul: upper left
                  'ur': {'x': [0, 0], 'y': [0, 0]},  # ur: upper right
                  'll': {'x': [0, 0], 'y': [0, 0]},  # ll: lower left
                  'lr': {'x': [0, 0], 'y': [0, 0]}}  # lr: lower right

        output['ul']['x'][0] = src['affine'][2]
        output['ul']['y'][0] = src['affine'][5]
        output['ur']['x'][0] = output['ul']['x'][0] + self.pixel * src['shape'][1]
        output['ur']['y'][0] = output['ul']['y'][0]
        output['ll']['x'][0] = output['ul']['x'][0]
        output['ll']['y'][0] = output['ul']['y'][0] - self.pixel * src['shape'][0]
        output['lr']['x'][0] = output['ul']['x'][0] + self.pixel * src['shape'][1]
        output['lr']['y'][0] = output['ul']['y'][0] - self.pixel * src['shape'][0]

        output['ul']['x'][1], output['ul']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['ul']['x'][0]],
                                                               [output['ul']['y'][0]])

        output['ur']['x'][1], output['ur']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['ur']['x'][0]],
                                                               [output['ur']['y'][0]])

        output['ll']['x'][1], output['ll']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['ll']['x'][0]],
                                                               [output['ll']['y'][0]])

        output['lr']['x'][1], output['lr']['y'][1] = transform(src['crs'], self.projection,
                                                               [output['lr']['x'][0]],
                                                               [output['lr']['y'][0]])

        return output

    def _percent_cut(self, color, low, high):
        return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high))

    def _unzip(self, src, dst, scene, force_unzip=False):
        """ Unzip tar files """
        self.output("Unzipping %s - It might take some time" % scene, normal=True, arrow=True)

        try:
            # check if file is already unzipped, skip
            if isdir(dst) and not force_unzip:
                self.output("%s is already unzipped." % scene, normal=True, arrow=True)
                return
            else:
                tar = tarfile.open(src, 'r')
                tar.extractall(path=dst)
                tar.close()
        except tarfile.ReadError:
            check_create_folder(dst)
            subprocess.check_call(['tar', '-xf', src, '-C', dst])

    def _get_full_filename(self, band):

        base_file = '%s_B%s.*' % (self.scene, band)

        try:
            return glob.glob(join(self.scene_path, base_file))[0].split('/')[-1]
        except IndexError:
            raise FileDoesNotExist('%s does not exist' % '%s_B%s.*' % (self.scene, band))

    def _check_if_zipped(self, path):
        """ Checks if the filename shows a tar/zip file """
        filename = get_file(path).split('.')

        if filename[-1] in ['bz', 'bz2']:
            return True

        return False

if __name__ == '__main__':

    with timer():
        p = Process(sys.argv[1])

        print p.run(sys.argv[2] == 't')
<|endoftext|>