<commit_before># -*- mode: python; indent-tabs-mode: nil; c-basic-offset: 4; tab-width: 4; -*-
# vim: set shiftwidth=4 softtabstop=4 expandtab:

"""Support for handling a set of files with with time-stamps in their names.

2014, Copyright University Corporation for Atmospheric Research

This file is part of the "django-ncharts" package.
The license and distribution terms for this file may be found in the
file LICENSE in this package.
"""

import os, glob, stat, re, sys, threading, logging
# from stat import *
from datetime import datetime
from pytz import utc
import sre_constants

_logger = logging.getLogger(__name__)   # pylint: disable=invalid-name

def globify_time_descriptors(path):
    """Convert strftime time descriptors to a glob-compatible expression.

    For example, convert,'%Y' to '[12][0-9][0-9][0-9]'. Then the
    glob expression can be used to match path names containing
    a four digit year.
    """

    path = path.replace('%Y', '[12][0-9][0-9][0-9]')
    path = path.replace('%y', '[0-9][0-9]')
    path = path.replace('%m', '[01][0-9]')
    path = path.replace('%d', '[0-3][0-9]')
    path = path.replace('%H', '[0-2][0-9]')
    path = path.replace('%M', '[0-5][0-9]')
    path = path.replace('%S', '[0-5][0-9]')
    return path

def pathsplit(path):
    """Like os.path.split, but the first value returned is the leading
    portion of the path.

    For 'a/b/c/d.txt' pathsplit returns ('a','b/c/d.txt"), whereas
    os.path.split() returns ('a/b/c','d.txt').  For 'a',
    pathsplit returns ('a','')
    """

    # python.org says do not use os.sep to split path names, use
    # os.path.split()
    tail = ''
    rem = ''
    while len(path) > 0 and not path == os.sep:
        # print('path=',path)
        if len(rem) > 0:
            rem = os.path.join(tail, rem)
        else:
            rem = tail
        (path, tail) = os.path.split(path)
    return (tail, rem)

# pylint: disable=too-few-public-methods
class File(object):
    """A file path, the path expression containing time descriptors that
    was used to find the file, and the associated time parsed from the
    path.

    Attribute:
        path: A str containing the actual path to the file.
        pathdesc: A str containing the path, with possible datetime
            strftime descriptors, such as '%Y'.
        time: datetime.datetime parsed from the path using pathdesc.

    """

    def __init__(self, path, pathdesc):
        """Construct a File, and parse its associated time from path
            using pathdesc.

        Args:
            path: The path to the file.
            pathdesc: A str containing the path to the file, with possible
                datetime strftime descriptors, such as '%Y'.
        """

        self.path = path
        self.pathdesc = pathdesc
        # print('path=', path, ', pathdesc=', pathdesc)
        try:
            self.time = utc.localize(datetime.strptime(path, pathdesc))
            return
        except ValueError as exc:
            print(exc.args)
            raise
        except sre_constants.error as exc:
            print(exc.args)
        except:
            print('unexpected error:', sys.exc_info()[0])
            raise

        # strptime chokes when there are two time descriptors for
        # the same time quantity, e.g. two %Y in 'dir/data_%Y/file_%Y%m%d.dat'
        # convert the first %Y, %m etc to '' in pathdesc, and use
        # reg expressions to convert those characters to '' in path.

        #   pathdesc = 'dir/data_%Y/file_%Y%m%d.dat'
        #   path = 'dir/data_2013/file_20130715.dat'
        #
        #   want, for purposes of parsing:
        #   pathdesc = 'dir/data_/file_%Y%m%d.dat'
        #   path = 'dir/data_/file_20130715.dat'
        #
        #   convert
        #   pathdesc = 'dir/data_%Y/file_%Y%m%d.dat'
        #   to
        #   pathdesc = 'dir/data_[12][0-9][0-9][0-9]/file_%Y%m%d.dat'
        #   pat = '^(dir/data_)[12][0-9][0-9][0-9](.*)$'
        #   i is index of '%Y' in pathdesc
        #   pat is '^(' + pathdesc up to i + ')[12][0-9][0-9][0-9](.*)$'
        #   path = re.sub(pat,'\1\2',path)
        #   'dir/data_/file_20130715.dat'
        #   try strptime,
        #   cycle through replace dict {'%Y: '[12][0-9][0-9][0-9]',
        #       trying strptime, and if you get sre_constants.error
        #       keey trying
        #   pathdesc is common to all files, is there a way
        #   so that this process happens only once?
        #   but path must be changed at the same time. Difficult
        #   need a re that is applied to remove unneeded fields
        #
        # this still needs work...

        pref = os.path.commonprefix([path, pathdesc])
        path = path.replace(pref, '')
        pathdesc = pathdesc.replace(pref, '')
        while pathdesc.count('%Y') > 1:
            pattern = pathdesc.replace('%Y', '[12][0-9][0-9][0-9]', 1)
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%Y', '')
        if pathdesc.count('%y') > 1:
            pattern = pathdesc.replace('%y', '[0-9][0-9]', 1)
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%y', '')
        if pathdesc.count('%m') > 1:
            pattern = pathdesc.replace("%m", "[01][0-9]")
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%m', '')
        if pathdesc.count('%d') > 1:
            pattern = pathdesc.replace("%d", "[0-3][0-9]")
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%d', '')
        try:
            print('try again, path=', path, ', pathdesc=', pathdesc)
            self.time = datetime.strptime(path, pathdesc)
        except ValueError as exc:
            print(exc.args)
            raise
        except sre_constants.error as exc:
            print(exc.args)
        except:
            print('unexpected error:', sys.exc_info()[0])
            raise

class Dir(object):
    """A directory that can be scanned for files matching a
    path containing possible datetime strftime descriptors.

    Example, initial call from Fileset:
        dir = Dir.get(path='/data/acme',pathdesc='/data/acme',
            pathrem='%Y/xxx_%Y%m%d.nc')

        When dir.scan(t1,t2) is called, then for each 4 digit subdirectory
            of /data/acme that is found, then a recursive call for
            files matching 'xxx_%Y%m%d.nc' is performed.

        For example, if '/data/acme' contains a directory '2014':

            subdir = Dir.get(path='/data/acme/2014',pathdesc='/data/acme/%Y'
                pathrem='xxx_%Y%m%d.nc')

        Scan is recursively called on subdir:
            res = subdir.scan(t1,t2)

        Assuming that one or more files in subdir match 'xxx_%Y%m%d.nc',
        for the time period, the list of File objects is returned.
        For example:
        [
            File('/data/acme/2014/xxx_20140414.nc',
                '/data/acme/%Y/xxx_%Y%m%d.nc'),

            File('/data/acme/2014/xxx_20140415.nc',
                '/data/acme/%Y/xxx_%Y%m%d.nc')
        ]

    Attributes:

        path: A str, path to a directory.
        pathdesc: Same as path attribute, but with possible datetime
            descriptors that were resolved to create the actual path.
        pathrem: Remainder of path to be scanned, with possible descriptors.
        modtime: Modification time of the directory at the time it was
            last scanned. If modtime has not changed from the last scan
            then the cached values can be used.
        cached_subdirs: List of Dir objects scanned in this directory
            which match the head portion of pathrem.
        cached_files: List of File objects in this directory which match
            the head portion of pathrem.
        lock: Mutex for modtime, cached_subdirs, cached_files
    """

    __cached_filesets = {}
    __cached_dirs = {}
    __cache_lock = threading.Lock()

    def __init__(self, path, pathdesc, pathrem):
        """Create a Dir.

        Args:
            path: A str, path to a directory.
            pathdesc: A str, same as path, but with possible datetime
                descriptors, such as %Y, %m, %d that were resolved
                to create path.
            pathrem: The portion of pathdesc after path to be scanned.
        """

        self.path = path
        self.pathdesc = pathdesc
        self.pathrem = pathrem
        self.modtime = datetime.min
        self.cached_subdirs = []
        self.cached_files = []
        self.lock = threading.Lock()

    @staticmethod
    def get(path, pathdesc, pathrem):
        """Return a Dir, which may be from a cache in order to
        avoid repeated scans.
        """

        Dir.__cache_lock.acquire()

        hashval = hash(path) + hash(pathrem)
        if hashval in Dir.__cached_dirs:
            ddir = Dir.__cached_dirs[hashval]
        else:
            ddir = Dir(path, pathdesc, pathrem)
            Dir.__cached_dirs[hash(path) + hash(pathrem)] = ddir

        Dir.__cache_lock.release()
        return ddir

    def scan(self, start_time=datetime.min, end_time=datetime.max):
        """Scan this Dir for files which match by name and time.

        The current directory is scanned for files which match pathrem.
        Also any subdirectories that match the head of pathrem are
        scanned. A list of files is returns whose associated times
        are within the period [start_time, end_time).

        If the directory has not been modified since the
        previous scan, then the previous list of files in the
        directory, if any is returned.  Scans are performed
        on any subdirectories which match the head of pathrem.

        Args:
            start_time: A datetime.datetime, start of the time period.
            end_time: A datetime.datetime, end of the time period.

        Returns:
            A list of matching File objects, sorted by their associated
            path time, whose times fall within [start_time, end_time).

        Raises:
            FileNotFoundError, PermissionError
        """

        files = []

        try:
            pstat = os.stat(self.path)
        except FileNotFoundError as exc:
            _logger.error(exc)
            raise
        except PermissionError as exc:
            _logger.error(exc)
            raise

        dirmodtime = datetime.utcfromtimestamp(pstat.st_mtime)

        # get previous snapshot of this directory
        self.lock.acquire()
        mymodtime = self.modtime
        cached_files = self.cached_files.copy()
        cached_subdirs = self.cached_subdirs.copy()
        self.lock.release()

        # modification time of directory is newer than last scan
        if dirmodtime > mymodtime:
            cached_files = []
            cached_subdirs = []
            mymodtime = dirmodtime

            (nextpath, pathrem) = pathsplit(self.pathrem)

            # next portion of path
            globpath = globify_time_descriptors(nextpath)

            # search for directories
            for subpath in glob.iglob(os.path.join(self.path, globpath)):
                # print('subpath=', subpath)
                try:
                    pstat = os.stat(subpath)
                except FileNotFoundError as exc:
                    _logger.error(exc)
                    continue    # maybe it was (very) recently deleted
                except PermissionError as exc:
                    _logger.error(exc)
                    continue
                if stat.S_ISDIR(pstat.st_mode):
                    pdir = Dir.get(
                        subpath, os.path.join(self.pathdesc, nextpath), pathrem)
                else:
                    pfile = File(subpath, os.path.join(self.pathdesc, nextpath))
                    cached_files.append(pfile)

            # save snapshot
            self.lock.acquire()
            self.modtime = mymodtime
            self.cached_files = sorted(cached_files, key=lambda x: x.time)
            self.cached_subdirs = cached_subdirs
            self.lock.release()

        for pdir in cached_subdirs:
            # recursive listing.
            files.extend(pdir.scan(start_time, end_time))

        files.extend(cached_files)

        # exclude files whose time is equal to or after end_time, then sort
        files = sorted(
            list(
                [x for x in files if x.time < end_time]),
            key=lambda x: x.time)

        i = 0
        for i in range(len(files)):
            if files[i].time > start_time:
                break

        # print("i=",i)
        # we want to include the previous file
        files = files[max(i-1, 0):]
        # print("len(files)=", len(files))
        return files

class Fileset(object):
    """A set of files defined by a path, which usually contains
    datetime descriptors, such as %Y, %m and %d.

    Attributes:
        path: A str, path with possible strptime descriptors to the dataset.
        pdir: Dir corresponding to initial non-descriptor part of path.
    """

    __cached_filesets = {}

    __cached_dirs = {}

    __cache_lock = threading.Lock()

    def __init__(self, path):
        """Construct a Fileset from a path, which may contain
        datetime strptime descriptors, such as %Y

        Args:
            path: A str, containing the path to the dataset, with possible
                datetime descriptors, such as '/data/acme/xxx_%Y%m%d_%H%M.nc'.
        """
        self.path = path
        pathrem = ''

        # Look for datetime descriptors in path
        while '%' in path:
            # returns a empty string for path if there is only
            # a last pathname component: os.path.split('a') returns ('', 'a')
            (path, tail) = os.path.split(path)
            if len(pathrem) > 0:
                pathrem = os.path.join(tail, pathrem)
            else:
                pathrem = tail

        self.pdir = Dir.get(path, path, pathrem)

    @staticmethod
    def get(path):
        """Fetch a Fileset by path, which may be cached.

        Args:
            path: A str, path describing the Fileset, which may
                contain datetime strftime descriptors.
        """
        Fileset.__cache_lock.acquire()
        if path in Fileset.__cached_filesets:
            fset = Fileset.__cached_filesets[path]
        else:
            fset = Fileset(path)
            Fileset.__cached_filesets[path] = fset

        Fileset.__cache_lock.release()
        return fset

    def scan(
            self, start_time=utc.localize(datetime.min),
            end_time=utc.localize(datetime.max)):
        """Scan this Fileset for files matching a time period.

        Args:
            start_time: A datetime.datetime, start of the time period.
            end_time: A datetime.datetime, end of the time period.

        Returns:
            A list of matching File objects, sorted by their associated
            path time, whose times fall within [start_time, end_time).

        Raises:
            FileNotFoundError, PermissionError

        """
        return self.pdir.scan(start_time, end_time)

<commit_msg>Added __str__ method<commit_after># -*- mode: python; indent-tabs-mode: nil; c-basic-offset: 4; tab-width: 4; -*-
# vim: set shiftwidth=4 softtabstop=4 expandtab:

"""Support for handling a set of files with with time-stamps in their names.

2014, Copyright University Corporation for Atmospheric Research

This file is part of the "django-ncharts" package.
The license and distribution terms for this file may be found in the
file LICENSE in this package.
"""

import os, glob, stat, re, sys, threading, logging
# from stat import *
from datetime import datetime
from pytz import utc
import sre_constants

_logger = logging.getLogger(__name__)   # pylint: disable=invalid-name

def globify_time_descriptors(path):
    """Convert strftime time descriptors to a glob-compatible expression.

    For example, convert,'%Y' to '[12][0-9][0-9][0-9]'. Then the
    glob expression can be used to match path names containing
    a four digit year.
    """

    path = path.replace('%Y', '[12][0-9][0-9][0-9]')
    path = path.replace('%y', '[0-9][0-9]')
    path = path.replace('%m', '[01][0-9]')
    path = path.replace('%d', '[0-3][0-9]')
    path = path.replace('%H', '[0-2][0-9]')
    path = path.replace('%M', '[0-5][0-9]')
    path = path.replace('%S', '[0-5][0-9]')
    return path

def pathsplit(path):
    """Like os.path.split, but the first value returned is the leading
    portion of the path.

    For 'a/b/c/d.txt' pathsplit returns ('a','b/c/d.txt"), whereas
    os.path.split() returns ('a/b/c','d.txt').  For 'a',
    pathsplit returns ('a','')
    """

    # python.org says do not use os.sep to split path names, use
    # os.path.split()
    tail = ''
    rem = ''
    while len(path) > 0 and not path == os.sep:
        # print('path=',path)
        if len(rem) > 0:
            rem = os.path.join(tail, rem)
        else:
            rem = tail
        (path, tail) = os.path.split(path)
    return (tail, rem)

# pylint: disable=too-few-public-methods
class File(object):
    """A file path, the path expression containing time descriptors that
    was used to find the file, and the associated time parsed from the
    path.

    Attribute:
        path: A str containing the actual path to the file.
        pathdesc: A str containing the path, with possible datetime
            strftime descriptors, such as '%Y'.
        time: datetime.datetime parsed from the path using pathdesc.

    """

    def __init__(self, path, pathdesc):
        """Construct a File, and parse its associated time from path
            using pathdesc.

        Args:
            path: The path to the file.
            pathdesc: A str containing the path to the file, with possible
                datetime strftime descriptors, such as '%Y'.
        """

        self.path = path
        self.pathdesc = pathdesc
        # print('path=', path, ', pathdesc=', pathdesc)
        try:
            self.time = utc.localize(datetime.strptime(path, pathdesc))
            return
        except ValueError as exc:
            print(exc.args)
            raise
        except sre_constants.error as exc:
            print(exc.args)
        except:
            print('unexpected error:', sys.exc_info()[0])
            raise

        # strptime chokes when there are two time descriptors for
        # the same time quantity, e.g. two %Y in 'dir/data_%Y/file_%Y%m%d.dat'
        # convert the first %Y, %m etc to '' in pathdesc, and use
        # reg expressions to convert those characters to '' in path.

        #   pathdesc = 'dir/data_%Y/file_%Y%m%d.dat'
        #   path = 'dir/data_2013/file_20130715.dat'
        #
        #   want, for purposes of parsing:
        #   pathdesc = 'dir/data_/file_%Y%m%d.dat'
        #   path = 'dir/data_/file_20130715.dat'
        #
        #   convert
        #   pathdesc = 'dir/data_%Y/file_%Y%m%d.dat'
        #   to
        #   pathdesc = 'dir/data_[12][0-9][0-9][0-9]/file_%Y%m%d.dat'
        #   pat = '^(dir/data_)[12][0-9][0-9][0-9](.*)$'
        #   i is index of '%Y' in pathdesc
        #   pat is '^(' + pathdesc up to i + ')[12][0-9][0-9][0-9](.*)$'
        #   path = re.sub(pat,'\1\2',path)
        #   'dir/data_/file_20130715.dat'
        #   try strptime,
        #   cycle through replace dict {'%Y: '[12][0-9][0-9][0-9]',
        #       trying strptime, and if you get sre_constants.error
        #       keey trying
        #   pathdesc is common to all files, is there a way
        #   so that this process happens only once?
        #   but path must be changed at the same time. Difficult
        #   need a re that is applied to remove unneeded fields
        #
        # this still needs work...

        pref = os.path.commonprefix([path, pathdesc])
        path = path.replace(pref, '')
        pathdesc = pathdesc.replace(pref, '')
        while pathdesc.count('%Y') > 1:
            pattern = pathdesc.replace('%Y', '[12][0-9][0-9][0-9]', 1)
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%Y', '')
        if pathdesc.count('%y') > 1:
            pattern = pathdesc.replace('%y', '[0-9][0-9]', 1)
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%y', '')
        if pathdesc.count('%m') > 1:
            pattern = pathdesc.replace("%m", "[01][0-9]")
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%m', '')
        if pathdesc.count('%d') > 1:
            pattern = pathdesc.replace("%d", "[0-3][0-9]")
            path = re.sub(pattern, '', path, count=1)
            pathdesc = pathdesc.replace('%d', '')
        try:
            print('try again, path=', path, ', pathdesc=', pathdesc)
            self.time = datetime.strptime(path, pathdesc)
        except ValueError as exc:
            print(exc.args)
            raise
        except sre_constants.error as exc:
            print(exc.args)
        except:
            print('unexpected error:', sys.exc_info()[0])
            raise

class Dir(object):
    """A directory that can be scanned for files matching a
    path containing possible datetime strftime descriptors.

    Example, initial call from Fileset:
        dir = Dir.get(path='/data/acme',pathdesc='/data/acme',
            pathrem='%Y/xxx_%Y%m%d.nc')

        When dir.scan(t1,t2) is called, then for each 4 digit subdirectory
            of /data/acme that is found, then a recursive call for
            files matching 'xxx_%Y%m%d.nc' is performed.

        For example, if '/data/acme' contains a directory '2014':

            subdir = Dir.get(path='/data/acme/2014',pathdesc='/data/acme/%Y'
                pathrem='xxx_%Y%m%d.nc')

        Scan is recursively called on subdir:
            res = subdir.scan(t1,t2)

        Assuming that one or more files in subdir match 'xxx_%Y%m%d.nc',
        for the time period, the list of File objects is returned.
        For example:
        [
            File('/data/acme/2014/xxx_20140414.nc',
                '/data/acme/%Y/xxx_%Y%m%d.nc'),

            File('/data/acme/2014/xxx_20140415.nc',
                '/data/acme/%Y/xxx_%Y%m%d.nc')
        ]

    Attributes:

        path: A str, path to a directory.
        pathdesc: Same as path attribute, but with possible datetime
            descriptors that were resolved to create the actual path.
        pathrem: Remainder of path to be scanned, with possible descriptors.
        modtime: Modification time of the directory at the time it was
            last scanned. If modtime has not changed from the last scan
            then the cached values can be used.
        cached_subdirs: List of Dir objects scanned in this directory
            which match the head portion of pathrem.
        cached_files: List of File objects in this directory which match
            the head portion of pathrem.
        lock: Mutex for modtime, cached_subdirs, cached_files
    """

    __cached_filesets = {}
    __cached_dirs = {}
    __cache_lock = threading.Lock()

    def __init__(self, path, pathdesc, pathrem):
        """Create a Dir.

        Args:
            path: A str, path to a directory.
            pathdesc: A str, same as path, but with possible datetime
                descriptors, such as %Y, %m, %d that were resolved
                to create path.
            pathrem: The portion of pathdesc after path to be scanned.
        """

        self.path = path
        self.pathdesc = pathdesc
        self.pathrem = pathrem
        self.modtime = datetime.min
        self.cached_subdirs = []
        self.cached_files = []
        self.lock = threading.Lock()

    @staticmethod
    def get(path, pathdesc, pathrem):
        """Return a Dir, which may be from a cache in order to
        avoid repeated scans.
        """

        Dir.__cache_lock.acquire()

        hashval = hash(path) + hash(pathrem)
        if hashval in Dir.__cached_dirs:
            ddir = Dir.__cached_dirs[hashval]
        else:
            ddir = Dir(path, pathdesc, pathrem)
            Dir.__cached_dirs[hash(path) + hash(pathrem)] = ddir

        Dir.__cache_lock.release()
        return ddir

    def scan(self, start_time=datetime.min, end_time=datetime.max):
        """Scan this Dir for files which match by name and time.

        The current directory is scanned for files which match pathrem.
        Also any subdirectories that match the head of pathrem are
        scanned. A list of files is returns whose associated times
        are within the period [start_time, end_time).

        If the directory has not been modified since the
        previous scan, then the previous list of files in the
        directory, if any is returned.  Scans are performed
        on any subdirectories which match the head of pathrem.

        Args:
            start_time: A datetime.datetime, start of the time period.
            end_time: A datetime.datetime, end of the time period.

        Returns:
            A list of matching File objects, sorted by their associated
            path time, whose times fall within [start_time, end_time).

        Raises:
            FileNotFoundError, PermissionError
        """

        files = []

        try:
            pstat = os.stat(self.path)
        except FileNotFoundError as exc:
            _logger.error(exc)
            raise
        except PermissionError as exc:
            _logger.error(exc)
            raise

        dirmodtime = datetime.utcfromtimestamp(pstat.st_mtime)

        # get previous snapshot of this directory
        self.lock.acquire()
        mymodtime = self.modtime
        cached_files = self.cached_files.copy()
        cached_subdirs = self.cached_subdirs.copy()
        self.lock.release()

        # modification time of directory is newer than last scan
        if dirmodtime > mymodtime:
            cached_files = []
            cached_subdirs = []
            mymodtime = dirmodtime

            (nextpath, pathrem) = pathsplit(self.pathrem)

            # next portion of path
            globpath = globify_time_descriptors(nextpath)

            # search for directories
            for subpath in glob.iglob(os.path.join(self.path, globpath)):
                # print('subpath=', subpath)
                try:
                    pstat = os.stat(subpath)
                except FileNotFoundError as exc:
                    _logger.error(exc)
                    continue    # maybe it was (very) recently deleted
                except PermissionError as exc:
                    _logger.error(exc)
                    continue
                if stat.S_ISDIR(pstat.st_mode):
                    pdir = Dir.get(
                        subpath, os.path.join(self.pathdesc, nextpath), pathrem)
                else:
                    pfile = File(subpath, os.path.join(self.pathdesc, nextpath))
                    cached_files.append(pfile)

            # save snapshot
            self.lock.acquire()
            self.modtime = mymodtime
            self.cached_files = sorted(cached_files, key=lambda x: x.time)
            self.cached_subdirs = cached_subdirs
            self.lock.release()

        for pdir in cached_subdirs:
            # recursive listing.
            files.extend(pdir.scan(start_time, end_time))

        files.extend(cached_files)

        # exclude files whose time is equal to or after end_time, then sort
        files = sorted(
            list(
                [x for x in files if x.time < end_time]),
            key=lambda x: x.time)

        i = 0
        for i in range(len(files)):
            if files[i].time > start_time:
                break

        # print("i=",i)
        # we want to include the previous file
        files = files[max(i-1, 0):]
        # print("len(files)=", len(files))
        return files

class Fileset(object):
    """A set of files defined by a path, which usually contains
    datetime descriptors, such as %Y, %m and %d.

    Attributes:
        path: A str, path with possible strptime descriptors.
        pdir: Dir corresponding to initial portion directory path up to
            a directory or file name with strptime descriptors.
    """

    __cached_filesets = {}

    __cached_dirs = {}

    __cache_lock = threading.Lock()

    def __init__(self, path):
        """Construct a Fileset from a path, which may contain
        datetime strptime descriptors, such as %Y

        Args:
            path: A str, containing the path to the dataset, with possible
                datetime descriptors, such as '/data/acme/xxx_%Y%m%d_%H%M.nc'.
        """
        self.path = path
        pathrem = ''

        # Look for datetime descriptors in path
        while '%' in path:
            # returns a empty string for path if there is only
            # a last pathname component: os.path.split('a') returns ('', 'a')
            (path, tail) = os.path.split(path)
            if len(pathrem) > 0:
                pathrem = os.path.join(tail, pathrem)
            else:
                pathrem = tail

        self.pdir = Dir.get(path, path, pathrem)

    def __str__(self):
        return self.path

    @staticmethod
    def get(path):
        """Fetch a Fileset by path, which may be cached.

        Args:
            path: A str, path describing the Fileset, which may
                contain datetime strftime descriptors.
        """
        Fileset.__cache_lock.acquire()
        if path in Fileset.__cached_filesets:
            fset = Fileset.__cached_filesets[path]
        else:
            fset = Fileset(path)
            Fileset.__cached_filesets[path] = fset

        Fileset.__cache_lock.release()
        return fset

    def scan(
            self, start_time=utc.localize(datetime.min),
            end_time=utc.localize(datetime.max)):
        """Scan this Fileset for files matching a time period.

        Args:
            start_time: A datetime.datetime, start of the time period.
            end_time: A datetime.datetime, end of the time period.

        Returns:
            A list of matching File objects, sorted by their associated
            path time, whose times fall within [start_time, end_time).

        Raises:
            FileNotFoundError, PermissionError

        """
        return self.pdir.scan(start_time, end_time)

<|endoftext|>


<commit_before>import unittest
import numpy.testing as testing
import numpy as np
import fitsio

import redmapper

class BackgroundTestCase(unittest.TestCase):

	def test_io(self):
		# test that we fail if we try a non-existent file
        self.assertRaises(IOError, redmapper.background.Background,
        											'nonexistent.fit')
        # test that we fail if we read a non-fits file
        self.assertRaises(IOError, redmapper.background.Background,
        							'%s/testconfig.yaml' % (self.file_path))
        # test that we fail if we try a file without the right header info
        self.assertRaises(ValueError, redmapper.background.Background,
        							'%s/test_dr8_pars.fit' % (self.file_path))

    def test_sigma_g(self):
    	np.random.seed(0)
    	idl_sigma_g_outputs = [np.array([0, 0, 0]),
        			   		   np.array([0, 0, 0]),
        			   		   np.array([0, 0, 0]),
        			   		   np.array([0, 0, 0]),
        			   		   np.array([0, 0, 0])]
        idl_sigma_lng_outputs = [np.array([0, 0, 0]),
        			   		   	 np.array([0, 0, 0]),
        			   		   	 np.array([0, 0, 0]),
        			   		   	 np.array([0, 0, 0]),
        			   		   	 np.array([0, 0, 0])]
        for (a, b) in zip(idl_sigma_g_outputs, idl_sigma_lng_outputs):
        	idx = tuple(np.random.randint(i) for i in self.bkg.sigma_g.shape)
        	self.assert_almost_equal(self.bkg.sigma_g[idx], a, decimal=5)
        	self.assert_almost_equal(self.bkg.sigma_lng[idx], b, decimal=5)

    def test_lookup(self): pass

	def runTest(self):
		self.file_name, self.file_path = 'test_bkg.fit', 'data'
		self.test_io()
        # read in the parameters
        self.bkg = redmapper.background.Background('%s/%s' % (file_path,
        													  file_name))
        self.test_sigma_g()
        self.test_lookup()<commit_msg>Added idl data.<commit_after>import unittest
import numpy.testing as testing
import numpy as np
import fitsio

import redmapper

class BackgroundTestCase(unittest.TestCase):

    def test_io(self):
        # test that we fail if we try a non-existent file
        self.assertRaises(IOError, redmapper.background.Background,
                          'nonexistent.fit')
        # test that we fail if we read a non-fits file
        self.assertRaises(IOError, redmapper.background.Background,
                          '%s/testconfig.yaml' % (self.file_path))
        # test that we fail if we try a file without the right header info
        self.assertRaises(AttributeError, redmapper.background.Background,
                          '%s/test_dr8_pars.fit' % (self.file_path))

    def test_sigma_g(self):
        np.random.seed(0)
        idl_outputs = [0.32197464, 6.4165196, 0.0032830855, 1.4605126,
                       0.0098356586, 0.79848081, 0.011284498, 9.3293247]
        for out in idl_outputs:
            idx = tuple(np.random.randint(i) for i in self.bkg.sigma_g.shape)
            testing.assert_almost_equal(self.bkg.sigma_g[idx], out, decimal=5)

    def test_lookup(self): pass

    def setUp(self):
        self.file_name, self.file_path = 'test_bkg.fit', 'data'
        self.bkg = redmapper.background.Background('%s/%s' % (self.file_path,
                                                              self.file_name))

if __name__=='__main__':
    unittest.main()<|endoftext|>


<commit_before>"""
    config
    ~~~~~
    This module contains configuration keys for the application.
    See http://flask.pocoo.org/docs/config/ for more information.
"""
# Statement for enabling the development environment
DEBUG = True

# Define the application directory
import os
_basedir = os.path.abspath(os.path.dirname(__file__))

# Define the secret key for signing session cookies
SECRET_KEY = '\x94\xb2\xf2</6\xd7+Op\xc5\x97v)\x83\xda\xff)\xd0UD\xd4NB'

# Define the database
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(_basedir, 'data.db')

# Application threads. A common general assumption is
# using 2 per available processor cores - to handle
# incoming requests using one and performing background
# operations using the other.
THREADS_PER_PAGE = 2

# Enable protection agains *Cross-site Request Forgery (CSRF)*
CSRF_ENABLED = True

# Use a secure, unique and absolutely secret key for
# signing the data.
CSRF_SESSION_KEY = "secret"

# Staging options
STAGING_DIR = '/tmp/staging/'

# SMTP Configs
MAIL_SERVER = 'iwmcas.iwm.fraunhofer.de'

# Notification flag. If true will try to send notifications (emails) when job
# status changes.
NOTIFICATION = True

# Mail server settings
SMTP_HOST = 'localhost'
SMTP_PORT = 1025

# Default web server address. Set this to whatever address the server will run
# This will be used to generate urls outside of a request context for example
# for notifications which contain links to certain pages such as job details.
# SERVER_NAME = 'sqmpy.iwm.fraunhofer.de:3000'

# Admin email to send job status notifications to, in case login is disabled
ADMIN_EMAIL = 'sade@iwm.fraunhofer.de'

# For first release there is no login-procedure
LOGIN_DISABLED = False

# Enables LDAP login, this will cause LDAP credentials to be
#   used for SSH as well
USE_LDAP_LOGIN = False
LDAP_SERVER = 'iwmnds0.iwm.fraunhofer.de'
LDAP_BASEDN = 'ou=People,ou=IWM,o=Fraunhofer,c=DE'

# User user login information for SSH
SSH_WITH_LOGIN_INFO = True
<commit_msg>Change defaults to avoid confusion<commit_after>"""
    config
    ~~~~~
    This module contains configuration keys for the application.
    See http://flask.pocoo.org/docs/config/ for more information.
"""
# Statement for enabling the development environment
DEBUG = True

# Define the application directory
import os
_basedir = os.path.abspath(os.path.dirname(__file__))

# Define the secret key for signing session cookies
SECRET_KEY = '\x94\xb2\xf2</6\xd7+Op\xc5\x97v)\x83\xda\xff)\xd0UD\xd4NB'

# Define the database
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(_basedir, 'data.db')

# Application threads. A common general assumption is
# using 2 per available processor cores - to handle
# incoming requests using one and performing background
# operations using the other.
THREADS_PER_PAGE = 2

# Enable protection agains *Cross-site Request Forgery (CSRF)*
CSRF_ENABLED = True

# Use a secure, unique and absolutely secret key for
# signing the data.
CSRF_SESSION_KEY = "secret"

# Sqmpy uses this directory as file storage. All scripts, input and output
# files will be stored at subfolders at this directory, separated by
# usernames and job IDs.
# STAGING_DIR = '/var/sqmpy/'
STAGING_DIR = '/tmp/staging/'

# SMTP Configs
MAIL_SERVER = 'iwmcas.iwm.fraunhofer.de'

# Notification flag. If true will try to send notifications (emails) when job
# status changes.
NOTIFICATION = True

# Mail server settings
SMTP_HOST = 'localhost'
SMTP_PORT = 1025

# Default web server address. Set this to whatever address the server will run
# This will be used to generate urls outside of a request context for example
# for notifications which contain links to certain pages such as job details.
# SERVER_NAME = 'sqmpy.iwm.fraunhofer.de:3000'

# Admin email to send job status notifications to, in case login is disabled
ADMIN_EMAIL = 'sade@iwm.fraunhofer.de'

# For first release there is no login-procedure
LOGIN_DISABLED = False

# Enables LDAP login, this will cause LDAP credentials to be
#   used for SSH as well
USE_LDAP_LOGIN = False
LDAP_SERVER = 'iwmnds0.iwm.fraunhofer.de'
LDAP_BASEDN = 'ou=People,ou=IWM,o=Fraunhofer,c=DE'

# User user login information for SSH
SSH_WITH_LOGIN_INFO = False
<|endoftext|>


<commit_before>from time import sleep

__author__ = 'David Moser <david.moser@bitmovin.com>'

import unittest
from bitcodin import create_job, get_job
from bitcodin import create_input
from bitcodin import create_encoding_profile
from bitcodin import delete_input
from bitcodin import delete_encoding_profile
from bitcodin import Job
from bitcodin import Input
from bitcodin import AudioStreamConfig
from bitcodin import EncodingProfile
from bitcodin.test.bitcodin_test_case import BitcodinTestCase

class CreateJobTestCase(BitcodinTestCase):
    def setUp(self):
        super(CreateJobTestCase, self).setUp()
        input_url = "http://bitbucketireland.s3.amazonaws.com/audiosample.mp3"
        input = Input(input_url)
        self.input = create_input(input)
        audio_stream_config = AudioStreamConfig(default_stream_id=0, bitrate=192000)
        encoding_profile = EncodingProfile('Audio Only Test Profile', [], [audio_stream_config])
        self.encoding_profile = create_encoding_profile(encoding_profile)
        self.manifests = ['m3u8', 'mpd']

    def runTest(self):
        job = Job(
            input_id=self.input.input_id,
            encoding_profile_id=self.encoding_profile.encoding_profile_id,
            manifest_types=self.manifests,
            speed='standard'
        )
        self.job = create_job(job)
        self.assertEquals(self.job.input.input_id, job.inputId)
        self.assertEquals(self.job.input.url, self.input.url)
        self.assertEquals(self.job.encoding_profiles[0].encoding_profile_id, job.encodingProfileId)
        self.wait_until_job_finished(self.job.job_id)

    def tearDown(self):
        delete_input(self.input.input_id)
        delete_encoding_profile(self.encoding_profile.encoding_profile_id)
        super(CreateJobTestCase, self).tearDown()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Renamed test<commit_after>from time import sleep

__author__ = 'David Moser <david.moser@bitmovin.com>'

import unittest
from bitcodin import create_job, get_job
from bitcodin import create_input
from bitcodin import create_encoding_profile
from bitcodin import delete_input
from bitcodin import delete_encoding_profile
from bitcodin import Job
from bitcodin import Input
from bitcodin import AudioStreamConfig
from bitcodin import EncodingProfile
from bitcodin.test.bitcodin_test_case import BitcodinTestCase

class CreateJobAudioOnlyTestCase(BitcodinTestCase):
    def setUp(self):
        super(CreateJobAudioOnlyTestCase, self).setUp()
        input_url = "http://bitbucketireland.s3.amazonaws.com/audiosample.mp3"
        input = Input(input_url)
        self.input = create_input(input)
        audio_stream_config = AudioStreamConfig(default_stream_id=0, bitrate=192000)
        encoding_profile = EncodingProfile('Audio Only Test Profile', [], [audio_stream_config])
        self.encoding_profile = create_encoding_profile(encoding_profile)
        self.manifests = ['m3u8', 'mpd']

    def runTest(self):
        job = Job(
            input_id=self.input.input_id,
            encoding_profile_id=self.encoding_profile.encoding_profile_id,
            manifest_types=self.manifests,
            speed='standard'
        )
        self.job = create_job(job)
        self.assertEquals(self.job.input.input_id, job.inputId)
        self.assertEquals(self.job.input.url, self.input.url)
        self.assertEquals(self.job.encoding_profiles[0].encoding_profile_id, job.encodingProfileId)
        self.wait_until_job_finished(self.job.job_id)

    def tearDown(self):
        delete_input(self.input.input_id)
        delete_encoding_profile(self.encoding_profile.encoding_profile_id)
        super(CreateJobTestCase, self).tearDown()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# graph_qc.py
# Created by Greg Kiar on 2016-05-11.
# Email: gkiar@jhu.edu

import numpy as np
import networkx as nx
import matplotlib
matplotlib.use('Agg') #very important this is above pyplot import

import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
import time
import json
import sys
import os

font = {'weight' : 'bold',
        'size'   : 14}
matplotlib.rc('font', **font)

cols = '#000000'

class plot_metrics():

    def __init__(self, nnz, deg, ew, ccoefs, ss1, eigs, centrality, outf):
        """
        Visualizes the computed summary statistics for each atlas

        Required Parameters:
            nnz:
                - dictionary of number of non-zero edges per graph
            deg:
                - dictionary of degree distributions per graph
            ew:
                - dictionary of edge weight distributions per graph
            ccoefs:
                - dictionary of clustering coefficients per graph
            ss1:
                - dictionary of scan statistic-1 distributions per graph
            eigs:
                - dictionary of eigen sequence per graph
            centrality:
                - dictionary of betweenness centrality coefficients per graph
            outf:
                - string for outgoing figure file name
        """
        self.nnz = nnz
        self.deg = deg
        self.ew = ew
        self.ccoefs = ccoefs
        self.ss1 = ss1
        self.eigs = eigs
        self.centrality = centrality
        self.color = cols
        self.outf = outf
        self.plotting()

    def plotting(self):
        fig = plt.figure(figsize=(14, 6))

        i = 0
        metric_list = [self.nnz, self.deg, self.ew, self.ccoefs, self.ss1,
                       self.centrality, self.eigs]
        name_list = ['NNZ', 'Degree', 'Edge Weight', 'Clustering Coefficient',
                     'Scan Statistic-1', 'Centrality', 'Eigen']
        type_list = ['sc', 'hi', 'hi', 'hi', 'hi', 'hi', 'se']
        for idx, val in enumerate(metric_list):
            i += 1
            ax = plt.subplot(2, 4, i)
            ax.yaxis.set_major_formatter(mtick.FormatStrFormatter('%2.1e'))
            plt.hold(True)
            self.plot_helper(val, name_list[idx], typ=type_list[idx])

        fig.tight_layout()

        plt.savefig(self.outf, bbox_inches='tight')
        # plt.show()

        metadata = {"subjects" : self.nnz.keys(),
                    "date": time.asctime(time.localtime())}
        with open(os.path.splitext(self.outf)[0]+'_info.json', 'w') as fp:
            json.dump(metadata, fp)

    def plot_helper(self, data, tit, typ='hi'):
        maxy = float('-inf')
        if typ == 'hi':
            for subj in data['pdfs']:
                x = data['xs'][subj]
                y = data['pdfs'][subj]
                ty = np.max(y)
                maxy = ty if ty > maxy else maxy
                plt.plot(x, y, color=self.color, alpha=0.1)
        elif typ == 'se':
            for subj in data:
                x = np.linspace(1, len(data[subj]), len(data[subj]))
                y = data[subj]
                ty = np.max(y)
                maxy = ty if ty > maxy else maxy
                plt.plot(x, y, color=self.color, alpha=0.1)
        elif typ == 'sc':
            x = self.rand_jitter([0]*len(data.values()))
            y = data.values()
            plt.scatter(x, y, alpha=0.1, color=self.color)

        if typ == 'sc':
            plt.xlim([np.mean(x)-0.2, np.mean(x)+0.2])
            plt.ylim([0, np.max(data.values())*1.1])
            plt.xticks([])
            plt.yticks([0, (np.max(y))/2, np.max(y)])
        else:
            plt.xlim([np.min(x), np.max(x)])
            plt.ylim([np.min(y), maxy])
            # eps = 1e-10
            # minx = 0 if abs(np.min(x)) < eps else np.min(x)
            # miny = 0 if abs(np.min(y)) < eps else np.min(y)
            plt.xticks([np.min(x),  np.max(x)])
            plt.yticks([np.min(y), (np.max(y) - np.min(y))/2, np.max(y)])

        # plt.ticklabel_format(style='sci', axis='y', scilimits=(0,0))
        plt.title(tit, y=1.04)

    def rand_jitter(self, arr):
        stdev = .03*(max(arr)-min(arr)+2)
        return arr + np.random.randn(len(arr)) * stdev

    def factors(self, N):
        return set([item for subitem in
                    [(i, N//i)
                    for i in range(1, int(N**0.5) + 1)
                    if N % i == 0 and i > 1]
                    for item in subitem])
<commit_msg>simplified logic in factor module<commit_after>#!/usr/bin/env python

# Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# graph_qc.py
# Created by Greg Kiar on 2016-05-11.
# Email: gkiar@jhu.edu

import numpy as np
import networkx as nx
import matplotlib
matplotlib.use('Agg') #very important this is above pyplot import

import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
import time
import json
import sys
import os

font = {'weight' : 'bold',
        'size'   : 14}
matplotlib.rc('font', **font)

cols = '#000000'

class plot_metrics():

    def __init__(self, nnz, deg, ew, ccoefs, ss1, eigs, centrality, outf):
        """
        Visualizes the computed summary statistics for each atlas

        Required Parameters:
            nnz:
                - dictionary of number of non-zero edges per graph
            deg:
                - dictionary of degree distributions per graph
            ew:
                - dictionary of edge weight distributions per graph
            ccoefs:
                - dictionary of clustering coefficients per graph
            ss1:
                - dictionary of scan statistic-1 distributions per graph
            eigs:
                - dictionary of eigen sequence per graph
            centrality:
                - dictionary of betweenness centrality coefficients per graph
            outf:
                - string for outgoing figure file name
        """
        self.nnz = nnz
        self.deg = deg
        self.ew = ew
        self.ccoefs = ccoefs
        self.ss1 = ss1
        self.eigs = eigs
        self.centrality = centrality
        self.color = cols
        self.outf = outf
        self.plotting()

    def plotting(self):
        fig = plt.figure(figsize=(14, 6))

        i = 0
        metric_list = [self.nnz, self.deg, self.ew, self.ccoefs, self.ss1,
                       self.centrality, self.eigs]
        name_list = ['NNZ', 'Degree', 'Edge Weight', 'Clustering Coefficient',
                     'Scan Statistic-1', 'Centrality', 'Eigen']
        type_list = ['sc', 'hi', 'hi', 'hi', 'hi', 'hi', 'se']
        for idx, val in enumerate(metric_list):
            i += 1
            ax = plt.subplot(2, 4, i)
            ax.yaxis.set_major_formatter(mtick.FormatStrFormatter('%2.1e'))
            plt.hold(True)
            self.plot_helper(val, name_list[idx], typ=type_list[idx])

        fig.tight_layout()

        plt.savefig(self.outf, bbox_inches='tight')
        # plt.show()

        metadata = {"subjects" : self.nnz.keys(),
                    "date": time.asctime(time.localtime())}
        with open(os.path.splitext(self.outf)[0]+'_info.json', 'w') as fp:
            json.dump(metadata, fp)

    def plot_helper(self, data, tit, typ='hi'):
        maxy = float('-inf')
        if typ == 'hi':
            for subj in data['pdfs']:
                x = data['xs'][subj]
                y = data['pdfs'][subj]
                ty = np.max(y)
                maxy = ty if ty > maxy else maxy
                plt.plot(x, y, color=self.color, alpha=0.1)
        elif typ == 'se':
            for subj in data:
                x = np.linspace(1, len(data[subj]), len(data[subj]))
                y = data[subj]
                ty = np.max(y)
                maxy = ty if ty > maxy else maxy
                plt.plot(x, y, color=self.color, alpha=0.1)
        elif typ == 'sc':
            x = self.rand_jitter([0]*len(data.values()))
            y = data.values()
            plt.scatter(x, y, alpha=0.1, color=self.color)

        if typ == 'sc':
            plt.xlim([np.mean(x)-0.2, np.mean(x)+0.2])
            plt.ylim([0, np.max(data.values())*1.1])
            plt.xticks([])
            plt.yticks([0, (np.max(y))/2, np.max(y)])
        else:
            plt.xlim([np.min(x), np.max(x)])
            plt.ylim([np.min(y), maxy])
            plt.xticks([np.min(x),  np.max(x)])
            plt.yticks([np.min(y), (np.max(y) - np.min(y))/2, np.max(y)])

        plt.title(tit, y=1.04)

    def rand_jitter(self, arr):
        stdev = .03*(max(arr)-min(arr)+2)
        return arr + np.random.randn(len(arr)) * stdev

    def factors(self, N):
        factors = [subitem for subitem in [(i, N//i)
                   for i in range(1, int(N**0.5) + 1)
                   if N % i == 0 and i > 1]]
        return set([fact for item in factors for fact in item])
<|endoftext|>


<commit_before># Downloads french "Code Civil" from https://github.com/steeve/france.code-civil and remove article names from each file

import os

# Clone from github
os.system('git clone https://github.com/steeve/france.code-civil.git')

base = "france.code-civil-master"

# Search recursively through each sub-directory and remove two first lines from each md/txt file
def rec(dirname):
	for d in os.listdir(dirname):
		filename = dirname+"/"+d
		if os.path.isdir(filename):
			rec(filename)
		elif os.path.isfile(filename):
			if filename.endswith(".md") or filename.endswith(".txt"):
				with open(filename, "r") as txtfile:
					text=txtfile.readlines()
				with open(filename, "w") as txtfile:
					for k in range(2,len(text)):
						txtfile.write(text[k])

# Apply function to root directory
rec(base)

<commit_msg>Update code_civil_dl.py<commit_after># Downloads french "Code Civil" from https://github.com/steeve/france.code-civil and remove article names from each file

import os

# Clone from github
os.system('git clone https://github.com/steeve/france.code-civil.git')

base = "france.code-civil"

# Search recursively through each sub-directory and remove two first lines from each md/txt file
def rec(dirname):
	for d in os.listdir(dirname):
		filename = dirname+"/"+d
		if os.path.isdir(filename):
			rec(filename)
		elif os.path.isfile(filename):
			if filename.endswith(".md") or filename.endswith(".txt"):
				with open(filename, "r") as txtfile:
					text=txtfile.readlines()
				with open(filename, "w") as txtfile:
					for k in range(2,len(text)):
						txtfile.write(text[k])

# Apply function to root directory
rec(base)

<|endoftext|>


<commit_before>import logging
import json
from flask import Flask, current_app
from flask import jsonify
from flask import request
from flask import session
from flask import redirect
from flask import url_for
from bson import json_util
from bson.objectid import ObjectId

from database import Database

def create_rest_app():
    with open('config/config.json') as conf_file:
        conf_data = json.load(conf_file)
        rest = Flask(__name__)
        rest.config['LOG_DIR'] = conf_data['LOG_DIR']
        rest.config['PORT'] = conf_data['PORT']
        rest.config['MONGO_HOST'] = conf_data['MONGO_HOST']
        rest.config['MONGO_DBNAME'] = conf_data['MONGO_DBNAME']
        rest.config['MONGO_PORT'] = conf_data['MONGO_PORT']
    return rest

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

# Console handler
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
# format
formatter = logging.Formatter('%(filename)s ## [%(asctime)s] %(levelname)s == "%(message)s"', datefmt='%Y/%b/%d %H:%M:%S')
stream_handler.setFormatter(formatter)
# add handler
logger.addHandler(stream_handler)

# FILE HANDLER LOGGER (not implemented again)
# from logging.handlers import RotatingFileHandler
# # création d'un handler qui va rediriger une écriture du log vers
# # un fichier en mode 'append', avec 1 backup et une taille max de 1Mo
# file_handler = RotatingFileHandler('activity.log', 'a', 1000000, 1)
# # on lui met le niveau sur DEBUG, on lui dit qu'il doit utiliser le formateur
# # créé précédement et on ajoute ce handler au logger
# file_handler.setLevel(logging.DEBUG)
# # création d'un formateur qui va ajouter le temps, le niveau
# # de chaque message quand on écrira un message dans le log
# formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')
# file_handler.setFormatter(formatter)
# logger.addHandler(file_handler)

try:
    rest = create_rest_app()
    logger.debug('app is initied')
    mongo_curs = Database().init_mongo(rest)
    logger.debug('mongo_curs is initiated')
    #log_dir = rest.config['LOG_DIR']
except BaseException as error:
    logger.debug('An exception occurred : {}'.format(error))

##########################################################################
# REST view - all
##########################################################################
# all queries
@rest.route('/queries/', methods=['GET'])
def all_queries_list():
    result = mongo_curs.db.query.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

# all videos
@rest.route('/queries/videos/', methods=['GET'])
def all_videos_list():
    result = mongo_curs.db.videos.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

# all comments
@rest.route('/queries/comments/', methods=['GET'])
def all_comments_list():
    result = mongo_curs.db.comments.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

# all captions
@rest.route('/queries/captions/', methods=['GET'])
def all_captions_list():
    result = mongo_curs.db.captions.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

##########################################################################
# REST view by users
##########################################################################
# all queries by user
@rest.route('/<user_id>/queries/', methods=['GET'])
def queries_list(user_id):
    result = mongo_curs.db.query.find({'author_id': user_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# one query by user
@rest.route('/<user_id>/queries/<query_id>', methods=['GET'])
def query_search(user_id, query_id):
    result = mongo_curs.db.query.find_one_or_404({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# list of videos by queries
@rest.route('/<user_id>/queries/<query_id>/videos/', methods=['GET'])
def videos_list_by_query(user_id, query_id):
    result = mongo_curs.db.videos.find({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# list of comments by queries
@rest.route('/<user_id>/queries/<query_id>/comments/', methods=['GET'])
def comments_list_by_query(user_id, query_id):
    result = mongo_curs.db.comments.find({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# list of captions by queries
@rest.route('/<user_id>/queries/<query_id>/captions/', methods=['GET'])
def captions_list_by_query(user_id, query_id):
    result = mongo_curs.db.captions.find({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

## by VIDEOS
@rest.route('/<user_id>/videos/<video_id>', methods=['GET'])
def video_search(user_id, video_id):
    result = mongo_curs.db.videos.find({'id.videoId': video_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

@rest.route('/<user_id>/videos/<video_id>/comments/', methods=['GET'])
def comments_list_by_video(user_id, video_id):
    result = mongo_curs.db.comments.find({'videoId': video_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

## by comments
@rest.route('/<user_id>/comments/<comment_id>', methods=['GET'])
def comment_search(user_id, comment_id):
    result = mongo_curs.db.comments.find_one_or_404(
        {'_id': ObjectId(comment_id)})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

## by captions
@rest.route('/<user_id>/captions/<caption_id>', methods=['GET'])
def caption_search(user_id, caption_id):
    result = mongo_curs.db.captions.find_one_or_404(
        {'_id': ObjectId(caption_id)})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

##########################################################################
# REST DEL
##########################################################################
# all queries by user
# @rest.route('/<user_id>/queries/', methods=['GET'])
# def queries_list(user_id):
#     result = mongo_curs.db.query.find({'author_id': user_id})
#     json_res = json_util.dumps(
#         result, sort_keys=True, indent=2, separators=(',', ': '))
#     logger.info(
#         'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
#     return jsonify(json.loads(json_res))

##########################################################################
# REST POST
##########################################################################

if __name__ == '__main__':
    rest.run(debug=True, host='0.0.0.0', port=rest.config['PORT'], threaded=True )<commit_msg>integration log file<commit_after>import logging
import json
from logging.handlers import RotatingFileHandler
from flask import Flask, current_app
from flask import jsonify
from flask import request
from flask import session
from flask import redirect
from flask import url_for
from bson import json_util
from bson.objectid import ObjectId
from database import Database

# config app
def create_rest_app():
    with open('config/config.json') as conf_file:
        conf_data = json.load(conf_file)
        rest = Flask(__name__)
        rest.config['LOG_DIR'] = conf_data['LOG_DIR']
        rest.config['PORT'] = conf_data['PORT']
        rest.config['MONGO_HOST'] = conf_data['MONGO_HOST']
        rest.config['MONGO_DBNAME'] = conf_data['MONGO_DBNAME']
        rest.config['MONGO_PORT'] = conf_data['MONGO_PORT']
    return rest

#init app
try:
    rest = create_rest_app()
    rest.logger.debug('app is initied')
    mongo_curs = Database().init_mongo(rest)
    rest.logger.debug('mongo_curs is initiated')
    #log_dir = rest.config['LOG_DIR']
except BaseException as error:
    logger.debug('An exception occurred : {}'.format(error))

##########################################################################
# REST view - all
##########################################################################
# all queries
@rest.route('/queries/', methods=['GET'])
def all_queries_list():
    result = mongo_curs.db.query.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

# all videos
@rest.route('/queries/videos/', methods=['GET'])
def all_videos_list():
    result = mongo_curs.db.videos.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

# all comments
@rest.route('/queries/comments/', methods=['GET'])
def all_comments_list():
    result = mongo_curs.db.comments.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

# all captions
@rest.route('/queries/captions/', methods=['GET'])
def all_captions_list():
    result = mongo_curs.db.captions.find({})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for *wildcard'.format(url=request.endpoint))
    return jsonify(json.loads(json_res))

##########################################################################
# REST view by users
##########################################################################
# all queries by user
@rest.route('/<user_id>/queries/', methods=['GET'])
def queries_list(user_id):
    result = mongo_curs.db.query.find({'author_id': user_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# one query by user
@rest.route('/<user_id>/queries/<query_id>', methods=['GET'])
def query_search(user_id, query_id):
    result = mongo_curs.db.query.find_one_or_404({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# list of videos by queries
@rest.route('/<user_id>/queries/<query_id>/videos/', methods=['GET'])
def videos_list_by_query(user_id, query_id):
    result = mongo_curs.db.videos.find({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# list of comments by queries
@rest.route('/<user_id>/queries/<query_id>/comments/', methods=['GET'])
def comments_list_by_query(user_id, query_id):
    result = mongo_curs.db.comments.find({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

# list of captions by queries
@rest.route('/<user_id>/queries/<query_id>/captions/', methods=['GET'])
def captions_list_by_query(user_id, query_id):
    result = mongo_curs.db.captions.find({'query_id': query_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

## by VIDEOS
@rest.route('/<user_id>/videos/<video_id>', methods=['GET'])
def video_search(user_id, video_id):
    result = mongo_curs.db.videos.find({'id.videoId': video_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

@rest.route('/<user_id>/videos/<video_id>/comments/', methods=['GET'])
def comments_list_by_video(user_id, video_id):
    result = mongo_curs.db.comments.find({'videoId': video_id})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

## by comments
@rest.route('/<user_id>/comments/<comment_id>', methods=['GET'])
def comment_search(user_id, comment_id):
    result = mongo_curs.db.comments.find_one_or_404(
        {'_id': ObjectId(comment_id)})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

## by captions
@rest.route('/<user_id>/captions/<caption_id>', methods=['GET'])
def caption_search(user_id, caption_id):
    result = mongo_curs.db.captions.find_one_or_404(
        {'_id': ObjectId(caption_id)})
    json_res = json_util.dumps(
        result, sort_keys=True, indent=2, separators=(',', ': '))
    rest.logger.info(
        'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
    return jsonify(json.loads(json_res))

##########################################################################
# REST DEL
##########################################################################
# all queries by user
# @rest.route('/<user_id>/queries/', methods=['GET'])
# def queries_list(user_id):
#     result = mongo_curs.db.query.find({'author_id': user_id})
#     json_res = json_util.dumps(
#         result, sort_keys=True, indent=2, separators=(',', ': '))
#     logger.info(
#         'try_request success on {url} for {user_id}'.format(url=request.endpoint, user_id=user_id))
#     return jsonify(json.loads(json_res))

##########################################################################
# REST POST
##########################################################################

# run app
if __name__ == '__main__':
    # config logger (prefering builtin flask logger)
    formatter = logging.Formatter('%(filename)s ## [%(asctime)s] %(levelname)s == "%(message)s"', datefmt='%Y/%b/%d %H:%M:%S')
    handler = RotatingFileHandler('activity.log', maxBytes=10000, backupCount=1)
    handler.setFormatter(formatter)
    #logger = logging.getLogger(__name__)
    rest.logger.setLevel(logging.DEBUG)
    rest.logger.addHandler(handler)
    rest.run(debug=True, host='0.0.0.0', port=rest.config['PORT'], threaded=True )
    <|endoftext|>


<commit_before>import unittest

from tests.rpc_mock import BitcoinProxyMock

from bitcoingraph.blockchain import BlockChain

BH1 = "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506"
BH1_HEIGHT = 100000

TX1 = "8c14f0db3df150123e6f3dbbf30f8b955a8249b62ac1d1ff16284aefa3d06d87"

class TestBlockchain(unittest.TestCase):

    def setUp(self):
        self.bitcoin_proxy = BitcoinProxyMock()
        self.blockchain = BlockChain(self.bitcoin_proxy)

    def test_init(self):
        self.assertIsNotNone(self.blockchain)

    ## Test Blockchain accessors

    def test_get_block_by_hash(self):
        block = self.blockchain.get_block_by_hash(BH1)
        self.assertEqual(block.hash, BH1)

    def test_get_block_by_height(self):
        block = self.blockchain.get_block_by_height(BH1_HEIGHT)
        self.assertEqual(block.height, BH1_HEIGHT)

    def test_get_block_range(self):
        blocks = [block for block in self.blockchain.get_block_range(
                  99999, 100001)]
        self.assertEqual(len(blocks), 3)
        self.assertEqual(blocks[0].height, 99999)
        self.assertEqual(blocks[1].height, 100000)
        self.assertEqual(blocks[2].height, 100001)

    def test_get_transaction(self):
        tx = self.blockchain.get_transaction(TX1)
        self.assertEqual(tx.id, TX1)
<commit_msg>finished blockchain tests<commit_after>import unittest

from tests.rpc_mock import BitcoinProxyMock

from bitcoingraph.blockchain import BlockChain

BH1 = "000000000002d01c1fccc21636b607dfd930d31d01c3a62104612a1719011250"
BH1_HEIGHT = 99999
BH2 = "000000000003ba27aa200b1cecaad478d2b00432346c3f1f3986da1afd33e506"
BH2_HEIGHT = 100000
BH3 = "00000000000080b66c911bd5ba14a74260057311eaeb1982802f7010f1a9f090"
BH3_HEIGHT = 100001

TX1 = "8c14f0db3df150123e6f3dbbf30f8b955a8249b62ac1d1ff16284aefa3d06d87"

class TestBlock(unittest.TestCase):

    def setUp(self):
        self.bitcoin_proxy = BitcoinProxyMock()
        self.blockchain = BlockChain(self.bitcoin_proxy)

    def test_height(self):
        block = self.blockchain.get_block_by_hash(BH1)
        self.assertEqual(block.height, BH1_HEIGHT)

    def test_hash(self):
        block = self.blockchain.get_block_by_hash(BH1)
        self.assertEqual(block.hash, BH1)

    def test_nextblockhash(self):
        block = self.blockchain.get_block_by_hash(BH1)
        self.assertTrue(block.hasnextblock)
        block = self.blockchain.get_block_by_hash(BH3)
        self.assertFalse(block.hasnextblock)

    def test_nextblock(self):
        block = self.blockchain.get_block_by_hash(BH1)
        self.assertEqual(block.nextblock.height, BH2_HEIGHT)

class TestBlockchain(unittest.TestCase):

    def setUp(self):
        self.bitcoin_proxy = BitcoinProxyMock()
        self.blockchain = BlockChain(self.bitcoin_proxy)

    def test_init(self):
        self.assertIsNotNone(self.blockchain)

    ## Test Blockchain accessors

    def test_get_block_by_hash(self):
        block = self.blockchain.get_block_by_hash(BH1)
        self.assertEqual(block.hash, BH1)

    def test_get_block_by_height(self):
        block = self.blockchain.get_block_by_height(BH1_HEIGHT)
        self.assertEqual(block.height, BH1_HEIGHT)

    def test_get_block_range(self):
        blocks = [block for block in self.blockchain.get_block_range(
                  99999, 100001)]
        self.assertEqual(len(blocks), 3)
        self.assertEqual(blocks[0].height, 99999)
        self.assertEqual(blocks[1].height, 100000)
        self.assertEqual(blocks[2].height, 100001)

    def test_get_transaction(self):
        tx = self.blockchain.get_transaction(TX1)
        self.assertEqual(tx.id, TX1)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os

'''
    Used for finding environment variables through configuration
    if a default is not given, the site will raise an exception
'''
def get_env_variable(var_name, default=-1):
    try:
        return os.environ[var_name]
    except KeyError:
        if default != -1:
            return default
        error_msg = "Set the %s os.environment variable" % var_name
        raise Exception(error_msg)

''' Default debugging to False '''
DEBUG = not get_env_variable("DATAUSA_PRODUCTION", False)
<commit_msg>trimmed config.py<commit_after># -*- coding: utf-8 -*-
import os

''' Default debugging to False '''
DEBUG = not os.environ.get("DATAUSA_PRODUCTION", True)
<|endoftext|>


<commit_before>import sys
sys.path.insert(1, "../../")
import h2o

#def impute2(ip,port):
    # Connect to a pre-existing cluster
#    h2o.init(ip,port)

h2o.init()

prostate = h2o.upload_file(h2o.locate("smalldata/logreg/prostate_missing.csv"))
methods = ["mean","median","mode"]
combine_methods=["interpolate", "average", "low", "high"]
inplace = [False, True]

for inpl in inplace:
    for method in methods:
        for combine_method in combine_methods:
            h2o.impute(prostate, "DPROS", method = method, combine_method = combine_method, inplace = inpl)

air = h2o.upload_file(h2o.locate("smalldata/airlines/allyears2k_headers.zip"))
for inpl in inplace:
    for method in methods:
        for combine_method in combine_methods:
            if method == "mode":
                h2o.impute(air, "TailNum", method = method, combine_method = combine_method, inplace = inpl)
            else:
                try:
                    h2o.impute(air, "TailNum", method = method, combine_method = combine_method, inplace = inpl)
                    assert False, "only \"mode\" method allowed for categorical column, but {0} was allowed here".\
                        format(method)
                except ValueError:
                    assert True

data = [[None, 2,    3,    1,    'a',  1,    9],
        [1,    None, 4,    2,    'a',  1,    9],
        [2,    3,    None, None, 'b',  1,    9],
        [3,    4,    None, None, 'b',  3,    8],
        [4,    5,    9,    5,    None, 2,    8],
        [5,    None, 10,   7,    'b',  None, 8]]
h2o_data = h2o.H2OFrame(python_obj=data)

# mean check
h2o.impute(h2o_data, column="C1", method="mean")
c1_imputed = h2o_data[0,0]
assert c1_imputed == 3, "Wrong value imputed. Expected imputed value of 3, but got {0}".format(c1_imputed)

# inplace check
h2o_data = h2o.H2OFrame(python_obj=data)
h2o.impute(h2o_data, column="C1", method="mean", inplace=False)
assert h2o_data["C1"].isna().sum() == 1, "Expected imputation to be done in place."

# median-interpolate
h2o_data = h2o.H2OFrame(python_obj=data)
h2o.impute(h2o_data, column="C1", method="median", combine_method="interpolate")

# median-average
h2o_data = h2o.H2OFrame(python_obj=data)
h2o.impute(h2o_data, column="C2", method="median", combine_method="average")
c2_imputed = h2o_data[1,1]
assert c2_imputed == 3.5, "Wrong value imputed. Expected imputed value of 3.5, but got {0}".format(c2_imputed)

# # median-low
# h2o_data = h2o.H2OFrame(python_obj=data)
# h2o.impute(h2o_data, column="C3", method="median", combine_method="low")
# c3_imputed = h2o_data[2,2]
# assert c3_imputed == 4, "Wrong value imputed. Expected imputed value of 4, but got {0}".format(c3_imputed)

# # median-high
# h2o_data = h2o.H2OFrame(python_obj=data)
# h2o.impute(h2o_data, column="C4", method="median", combine_method="high")
# c4_imputed = h2o_data[2,3]
# assert c4_imputed == 5, "Wrong value imputed. Expected imputed value of 5, but got {0}".format(c4_imputed)

# mode-categorical
h2o_data = h2o.H2OFrame(python_obj=data)
h2o.impute(h2o_data, column="C5", method="mode")
c5_imputed = h2o_data[4,4]
assert c5_imputed == 'b', "Wrong value imputed. Expected imputed value of b, but got {0}".format(c5_imputed)

# mode-numeric
h2o_data = h2o.H2OFrame(python_obj=data)
h2o.impute(h2o_data, column="C6", method="mode")
c6_imputed = h2o_data[5,5]
assert c6_imputed == 1, "Wrong value imputed. Expected imputed value of 1, but got {0}".format(c6_imputed)

# mean-group by C7
h2o_data = h2o.H2OFrame(python_obj=data)
h2o.impute(h2o_data, column="C3", method="mean", by=["C7"])
imputed1 = h2o_data[2,2]
imputed2 = h2o_data[3,2]
assert imputed1 == 3.5, "Wrong value imputed. Expected imputed value of 3.5, but got {0}".format(imputed1)
assert imputed2 == 9.5, "Wrong value imputed. Expected imputed value of 9.5, but got {0}".format(imputed2)

#if __name__ == "__main__":
#    h2o.run_test(sys.argv, impute2)<commit_msg>impute NOPASS. PUBDEV-1055<commit_after>import sys
sys.path.insert(1, "../../")
import h2o

def impute2(ip,port):
    # Connect to a pre-existing cluster
    h2o.init(ip,port)

    prostate = h2o.upload_file(h2o.locate("smalldata/logreg/prostate_missing.csv"))
    methods = ["mean","median","mode"]
    combine_methods=["interpolate", "average", "low", "high"]
    inplace = [False, True]

    for inpl in inplace:
        for method in methods:
            for combine_method in combine_methods:
                h2o.impute(prostate, "DPROS", method = method, combine_method = combine_method, inplace = inpl)

    air = h2o.upload_file(h2o.locate("smalldata/airlines/allyears2k_headers.zip"))
    for inpl in inplace:
        for method in methods:
            for combine_method in combine_methods:
                if method == "mode":
                    h2o.impute(air, "TailNum", method = method, combine_method = combine_method, inplace = inpl)
                else:
                    try:
                        h2o.impute(air, "TailNum", method = method, combine_method = combine_method, inplace = inpl)
                        assert False, "only \"mode\" method allowed for categorical column, but {0} was allowed here".\
                            format(method)
                    except ValueError:
                        assert True

    data = [[None, 2,    3,    1,    'a',  1,    9],
            [1,    None, 4,    2,    'a',  1,    9],
            [2,    3,    None, None, 'b',  1,    9],
            [3,    4,    None, None, 'b',  3,    8],
            [4,    5,    9,    5,    None, 2,    8],
            [5,    None, 10,   7,    'b',  None, 8]]
    h2o_data = h2o.H2OFrame(python_obj=data)

    # mean check
    h2o.impute(h2o_data, column="C1", method="mean")
    c1_imputed = h2o_data[0,0]
    assert c1_imputed == 3, "Wrong value imputed. Expected imputed value of 3, but got {0}".format(c1_imputed)

    # inplace check
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C1", method="mean", inplace=False)
    assert h2o_data["C1"].isna().sum() == 1, "Expected imputation to be done in place."

    # median-average
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C2", method="median", combine_method="average")
    c2_imputed = h2o_data[1,1]
    assert c2_imputed == 3.5, "Wrong value imputed. Expected imputed value of 3.5, but got {0}".format(c2_imputed)

    # median-low
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C3", method="median", combine_method="low")
    c3_imputed = h2o_data[2,2]
    assert c3_imputed == 4, "Wrong value imputed. Expected imputed value of 4, but got {0}".format(c3_imputed)

    # median-high
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C4", method="median", combine_method="high")
    c4_imputed = h2o_data[2,3]
    assert c4_imputed == 5, "Wrong value imputed. Expected imputed value of 5, but got {0}".format(c4_imputed)

    # mode-categorical
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C5", method="mode")
    c5_imputed = h2o_data[4,4]
    assert c5_imputed == 'b', "Wrong value imputed. Expected imputed value of b, but got {0}".format(c5_imputed)

    # mode-numeric
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C6", method="mode")
    c6_imputed = h2o_data[5,5]
    assert c6_imputed == 1, "Wrong value imputed. Expected imputed value of 1, but got {0}".format(c6_imputed)

    # mean-group by C7
    h2o_data = h2o.H2OFrame(python_obj=data)
    h2o.impute(h2o_data, column="C3", method="mean", by=["C7"])
    imputed1 = h2o_data[2,2]
    imputed2 = h2o_data[3,2]
    assert imputed1 == 3.5, "Wrong value imputed. Expected imputed value of 3.5, but got {0}".format(imputed1)
    assert imputed2 == 9.5, "Wrong value imputed. Expected imputed value of 9.5, but got {0}".format(imputed2)

if __name__ == "__main__":
    h2o.run_test(sys.argv, impute2)<|endoftext|>


<commit_before>"""Standard utilties for a theano model."""
import numbers
import numpy as np
import pickle
import random
import sys
import theano
import time
from Tkinter import TclError

import __init__ as ntu
from .. import log

class TheanoModel(object):
  """A generic theano model.

  This class handles some standard boilerplate.
  Current features include:
    Basic training loop
    Saving and reloading of model

  An implementing subclass must override the following methods:
    self.__init__(*args, **kwargs)
    self.init_params()
    self.setup_theano_funcs()
    self.get_objective(example)
    self.train_one(example, lr)
    self.evaluate(dataset)

  A basic self.__init__() routine is provided here, just as an example.
  Most users should override __init__() to perform additional functionality.

  See these methods for more details.
  """
  def __init__(self):
    """A minimal example of what functions must be called during initialization.

    Implementing subclasses should override this method,
    but maintain the basic functionality presented here.
    """
    # self.params and self.params_list are required by self.create_matrix()
    self.params = {}
    self.param_list = []

    # Initialize parameters
    self.init_params()

    # If desired, set up grad norm caches for momentum, AdaGrad, etc. here.
    # It must be after params are initialized but before theano functionss are created.
    # self.velocities = nt.create_grad_cache(self.param_list)

    # Set up theano functions
    self.theano_funcs = {}
    self.setup_theano_funcs()

  def init_params(self):
    """Initialize parameters with repeated calls to self.create_matrix()."""
    raise NotImplementedError

  def setup_theano_funcs(self):
    """Create theano.function objects for this model in self.theano_funcs."""
    raise NotImplementedError

  def get_objective(self, example):
    """Get the loss on a single example."""
    raise NotImplementedError

  def train_one(self, example, lr):
    """Run training on a single example."""
    raise NotImplementedError

  def create_matrix(self, name, shape, weight_scale):
    """A helper method that creates a parameter matrix."""
    if shape:
      value = weight_scale * np.random.uniform(-1.0, 1.0, shape).astype(
          theano.config.floatX)
    else:
      # None means it's a scalar
      dtype = np.dtype(theano.config.floatX)
      value = dtype.type(weight_scale * np.random.uniform(-1.0, 1.0))
    mat = theano.shared(name=name, value=value)
    self.params[name] = mat
    self.param_list.append(mat)

  def train(self, train_data, lr_init, epochs, dev_data=None, rng_seed=0,
            do_plot=False, plot_outfile=None):
    """Train the model.

    Args:
      train_data: A list of training examples
      lr_init: Initial learning rate
      epochs: An integer number of epochs to train, or a list of integers,
          where we halve the learning rate after each period.
      dev_data: A list of dev examples, evaluate loss on this each epoch.
      rng_seed: Random seed for shuffling the dataset at each epoch.
      do_plot: If True, plot a learning curve.
      plot_outfile: If provided, save learning curve to file.
    """
    random.seed(rng_seed)
    train_data = list(train_data)
    lr = lr_init
    if isinstance(epochs, numbers.Number):
      lr_changes = []
      num_epochs = epochs
    else:
      lr_changes = set([sum(epochs[:i]) for i in range(1, len(epochs))])
      num_epochs = sum(epochs)
    num_epochs_digits = len(str(num_epochs))
    train_obj_list = []
    dev_obj_list = []
    len_train_obj = 0
    len_dev_obj = 0
    len_time = 0
    for epoch in range(num_epochs):
      t0 = time.time()
      random.shuffle(train_data)
      if epoch in lr_changes:
        lr *= 0.5
      train_obj = 0.0
      for ex in train_data:
        cur_obj = self.train_one(ex, lr)
        train_obj += cur_obj
      if dev_data:
        dev_obj = sum(self.get_objective(ex) for ex in dev_data)
        dev_obj_list.append(dev_obj)
      else:
        dev_obj = 0.0
      train_obj_list.append(train_obj)
      t1 = time.time()

      # Some formatting to make things align in columns
      train_obj_str = '%.2f' % train_obj
      dev_obj_str = '%.2f' % dev_obj
      time_str = '%.2f' % (t1 - t0)
      len_train_obj = max(len(train_obj_str), len_train_obj)
      len_dev_obj = max(len(dev_obj_str), len_dev_obj)
      len_time = max(len(time_str), len_time)
      dev_str = ', dev loss = %s' % dev_obj_str.rjust(len_dev_obj) if dev_data else ''
      log('Epoch %s: train loss = %s%s (lr = %.1e) (time = %s s)' % (
          str(epoch+1).rjust(num_epochs_digits),
          train_obj_str.rjust(len_train_obj),
          dev_str, lr, time_str.rjust(len_time)))

    if do_plot:
      plot_data = [('Train Objective', train_obj_list)]
      if dev_data:
        plot_data.append(('Dev Objective', dev_obj_list))
      try:
        ntu.plot_learning_curve(plot_data, outfile=plot_outfile)
      except TclError:
        print >> sys.stderr, 'Encoutered error while plotting learning curve'

  def evaluate(self, dataset):
    """Evaluate the model."""
    raise NotImplementedError

  def save(self, filename):
    tf = self.theano_funcs  # save
    self.theano_funcs = None  # Don't pickle theano functions
    with open(filename, 'wb') as f:
      pickle.dump(self, f)
    self.theano_funcs = tf  # restore

  @classmethod
  def load(cls, filename):
    with open(filename, 'rb') as f:
      model = pickle.load(f)
    # Recompile theano functions
    model.theano_funcs = {}
    model.setup_theano_funcs()
    return model
<commit_msg>Minor edit to theanoutil.model<commit_after>"""Standard utilties for a theano model."""
import numbers
import numpy as np
import pickle
import random
import sys
import theano
import time
from Tkinter import TclError

import __init__ as ntu
from .. import log

class TheanoModel(object):
  """A generic theano model.

  This class handles some standard boilerplate.
  Current features include:
    Basic training loop
    Saving and reloading of model

  An implementing subclass must override the following methods:
    self.__init__(*args, **kwargs)
    self.init_params()
    self.setup_theano_funcs()
    self.get_objective(example)
    self.train_one(example, lr)
    self.evaluate(dataset)

  A basic self.__init__() routine is provided here, just as an example.
  Most users should override __init__() to perform additional functionality.

  See these methods for more details.
  """
  def __init__(self):
    """A minimal example of what functions must be called during initialization.

    Implementing subclasses should override this method,
    but maintain the basic functionality presented here.
    """
    # self.params and self.params_list are required by self.create_matrix()
    self.params = {}
    self.param_list = []

    # Initialize parameters
    self.init_params()

    # If desired, set up grad norm caches for momentum, AdaGrad, etc. here.
    # It must be after params are initialized but before theano functionss are created.
    # self.velocities = nt.create_grad_cache(self.param_list)

    # Set up theano functions
    self.theano_funcs = {}
    self.setup_theano_funcs()

  def init_params(self):
    """Initialize parameters with repeated calls to self.create_matrix()."""
    raise NotImplementedError

  def setup_theano_funcs(self):
    """Create theano.function objects for this model in self.theano_funcs."""
    raise NotImplementedError

  def get_objective(self, example):
    """Get the loss on a single example."""
    raise NotImplementedError

  def train_one(self, example, lr):
    """Run training on a single example."""
    raise NotImplementedError

  def create_matrix(self, name, shape, weight_scale, value=None):
    """A helper method that creates a parameter matrix."""
    if value:
      pass
    if shape:
      value = weight_scale * np.random.uniform(-1.0, 1.0, shape).astype(
          theano.config.floatX)
    else:
      # None means it's a scalar
      dtype = np.dtype(theano.config.floatX)
      value = dtype.type(weight_scale * np.random.uniform(-1.0, 1.0))
    mat = theano.shared(name=name, value=value)
    self.params[name] = mat
    self.param_list.append(mat)

  def train(self, train_data, lr_init, epochs, dev_data=None, rng_seed=0,
            do_plot=False, plot_outfile=None):
    """Train the model.

    Args:
      train_data: A list of training examples
      lr_init: Initial learning rate
      epochs: An integer number of epochs to train, or a list of integers,
          where we halve the learning rate after each period.
      dev_data: A list of dev examples, evaluate loss on this each epoch.
      rng_seed: Random seed for shuffling the dataset at each epoch.
      do_plot: If True, plot a learning curve.
      plot_outfile: If provided, save learning curve to file.
    """
    random.seed(rng_seed)
    train_data = list(train_data)
    lr = lr_init
    if isinstance(epochs, numbers.Number):
      lr_changes = []
      num_epochs = epochs
    else:
      lr_changes = set([sum(epochs[:i]) for i in range(1, len(epochs))])
      num_epochs = sum(epochs)
    num_epochs_digits = len(str(num_epochs))
    train_obj_list = []
    dev_obj_list = []
    len_train_obj = 0
    len_dev_obj = 0
    len_time = 0
    for epoch in range(num_epochs):
      t0 = time.time()
      random.shuffle(train_data)
      if epoch in lr_changes:
        lr *= 0.5
      train_obj = 0.0
      for ex in train_data:
        cur_obj = self.train_one(ex, lr)
        train_obj += cur_obj
      if dev_data:
        dev_obj = sum(self.get_objective(ex) for ex in dev_data)
        dev_obj_list.append(dev_obj)
      else:
        dev_obj = 0.0
      train_obj_list.append(train_obj)
      t1 = time.time()

      # Some formatting to make things align in columns
      train_obj_str = '%.2f' % train_obj
      dev_obj_str = '%.2f' % dev_obj
      time_str = '%.2f' % (t1 - t0)
      len_train_obj = max(len(train_obj_str), len_train_obj)
      len_dev_obj = max(len(dev_obj_str), len_dev_obj)
      len_time = max(len(time_str), len_time)
      dev_str = ', dev loss = %s' % dev_obj_str.rjust(len_dev_obj) if dev_data else ''
      log('Epoch %s: train loss = %s%s (lr = %.1e) (time = %s s)' % (
          str(epoch+1).rjust(num_epochs_digits),
          train_obj_str.rjust(len_train_obj),
          dev_str, lr, time_str.rjust(len_time)))

    if do_plot:
      plot_data = [('Train Objective', train_obj_list)]
      if dev_data:
        plot_data.append(('Dev Objective', dev_obj_list))
      try:
        ntu.plot_learning_curve(plot_data, outfile=plot_outfile)
      except TclError:
        print >> sys.stderr, 'Encoutered error while plotting learning curve'

  def evaluate(self, dataset):
    """Evaluate the model."""
    raise NotImplementedError

  def save(self, filename):
    tf = self.theano_funcs  # save
    self.theano_funcs = None  # Don't pickle theano functions
    with open(filename, 'wb') as f:
      pickle.dump(self, f)
    self.theano_funcs = tf  # restore

  @classmethod
  def load(cls, filename):
    with open(filename, 'rb') as f:
      model = pickle.load(f)
    # Recompile theano functions
    model.theano_funcs = {}
    model.setup_theano_funcs()
    return model
<|endoftext|>