<commit_before>#!/usr/bin/env python2

"""Standardize images."""

from __future__ import division
import argparse

import numpy as np

import dwi.dataset
import dwi.mask
import dwi.plot
import dwi.util

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='increase verbosity')
    p.add_argument('--samplelist',
            help='sample list file')
    p.add_argument('--subregiondir',
            help='subregion bounding box directory')
    p.add_argument('--pmapdir', default='dicoms_Mono_combinedDICOM',
            help='input parametric map directory')
    p.add_argument('--param', default='ADCm',
            help='image parameter to use')
    p.add_argument('--cases', metavar='I', nargs='*', type=int, default=[],
            help='case numbers')
    p.add_argument('--scans', metavar='S', nargs='*', default=[],
            help='scan identifiers')
    args = p.parse_args()
    return args

def select_ioi(data, pc1, pc2):
    """Select intensity of interest (IOI) parts of images."""
    for d in data:
        img = d['image'][...,0]
        s1 = np.percentile(img, pc1)
        s2 = np.percentile(img, pc2)
        img = img[img>=s1]
        img = img[img<=s2]
        d['ioi'] = img

def plot_hist(hist, bin_edges):
    import pylab as pl
    pl.plot(bin_edges[1:], hist)
    pl.show()
    pl.close()

args = parse_args()
if args.verbose:
    print 'Reading data...'
data = dwi.dataset.dataset_read_samplelist(args.samplelist, args.cases,
        args.scans)
dwi.dataset.dataset_read_patientinfo(data, args.samplelist)
if args.subregiondir:
    dwi.dataset.dataset_read_subregions(data, args.subregiondir)
dwi.dataset.dataset_read_pmaps(data, args.pmapdir, [args.param])

for d in data:
    img = d['image'][...,0]
    if args.verbose:
        print d['case'], d['scan'], img.shape, dwi.util.fivenum(img)

select_ioi(data, 0, 99.8)

for d in data:
    img = d['ioi']
    if args.verbose:
        print d['case'], d['scan'], img.shape, dwi.util.fivenum(img)

import pylab as pl
for d in data:
    img = d['ioi']
    hist, bin_edges = np.histogram(img, bins=1000, density=True)
    pl.plot(bin_edges[:-1], hist)
pl.show()
pl.close()
<commit_msg>deciles<commit_after>#!/usr/bin/env python2

"""Standardize images."""

from __future__ import division
import argparse

import numpy as np
import scipy as sp

import dwi.dataset
import dwi.mask
import dwi.plot
import dwi.util

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='increase verbosity')
    p.add_argument('--samplelist',
            help='sample list file')
    p.add_argument('--subregiondir',
            help='subregion bounding box directory')
    p.add_argument('--pmapdir', default='dicoms_Mono_combinedDICOM',
            help='input parametric map directory')
    p.add_argument('--param', default='ADCm',
            help='image parameter to use')
    p.add_argument('--cases', metavar='I', nargs='*', type=int, default=[],
            help='case numbers')
    p.add_argument('--scans', metavar='S', nargs='*', default=[],
            help='scan identifiers')
    args = p.parse_args()
    return args

def select_ioi(data, pc1, pc2):
    """Select intensity of interest (IOI) parts of images."""
    for d in data:
        img = d['image'][...,0]
        s1 = np.percentile(img, pc1)
        s2 = np.percentile(img, pc2)
        img = img[img>=s1]
        img = img[img<=s2]
        d['ioi'] = img

def plot(data):
    import pylab as pl
    for d in data:
        img = d['ioi']
        hist, bin_edges = np.histogram(img, bins=1000, density=True)
        pl.plot(bin_edges[:-1], hist)
    pl.show()
    pl.close()
    for d in data:
        y = d['deciles']
        x = range(len(y))
        pl.plot(x, y)
    pl.show()
    pl.close()

args = parse_args()
if args.verbose:
    print 'Reading data...'
data = dwi.dataset.dataset_read_samplelist(args.samplelist, args.cases,
        args.scans)
dwi.dataset.dataset_read_patientinfo(data, args.samplelist)
if args.subregiondir:
    dwi.dataset.dataset_read_subregions(data, args.subregiondir)
dwi.dataset.dataset_read_pmaps(data, args.pmapdir, [args.param])

print
for d in data:
    img = d['image'][...,0]
    if args.verbose:
        print d['case'], d['scan'], img.shape, dwi.util.fivenum(img)

select_ioi(data, 0, 99.8)

print
for d in data:
    img = d['ioi']
    if args.verbose:
        print d['case'], d['scan'], img.shape, dwi.util.fivenum(img)
    d['deciles'] = [sp.stats.scoreatpercentile(img, i*10) for i in range(11)]

plot(data)
<|endoftext|>


<commit_before># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import logging
import subprocess
import shlex
import sys
import os
import tarfile
import zipfile
import tempfile
import glob

from cerbero.config import Platform
from cerbero.utils import _, system_info
from cerbero.errors import FatalError

PATCH = 'patch'
TAR = 'tar'

PLATFORM = system_info()[0]
LOGFILE = None  # open('/tmp/cerbero.log', 'w+')

class StdOut:

    def __init__(self, stream=sys.stdout):
        self.stream = stream

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()

    def __getattr__(self, attr):
        return getattr(self.stream, attr)

def _fix_mingw_cmd(path):
    reserved = ['/', ' ', '\\', ')', '(']
    l_path = list(path)
    for i in range(len(path)):
        if path[i] == '\\':
            if i+1 == len(path) or path [i+1] not in reserved:
                l_path[i] = '/'
    return ''.join(l_path)

def call(cmd, cmd_dir='.', fail=True):
    '''
    Run a shell command

    @param cmd: the command to run
    @type cmd: str
    @param cmd_dir: directory where the command will be run
    @param cmd_dir: str
    @param fail: wheter to raise an exception if the command failed or not
    @type fail: bool
    '''
    try:
        logging.info("Running command '%s'" % cmd)
        shell = True
        if PLATFORM == Platform.WINDOWS:
            # windows do not understand ./
            if cmd.startswith('./'):
                cmd = cmd[2:]
            # run all processes through sh.exe to get scripts working
            cmd = '%s "%s"' % ('sh -c', cmd)
            # fix paths with backslashes
            cmd = _fix_mingw_cmd(cmd)
            # Disable shell which uses cmd.exe
            shell = False
        stream = LOGFILE or sys.stdout
        ret = subprocess.check_call(cmd, cwd=cmd_dir,
                stderr=subprocess.STDOUT, stdout=StdOut(stream),
                env=os.environ.copy(), shell=shell)
    except subprocess.CalledProcessError:
        if fail:
            raise FatalError(_("Error running command: %s") % cmd)
        else:
            ret = 0
    return ret

def check_call(cmd, cmd_dir=None, shell=False, split=True, fail=False):
    try:
        if split:
            cmd = shlex.split(cmd)
        process = subprocess.Popen(cmd, cwd=cmd_dir,
            stdout=subprocess.PIPE, stderr=open(os.devnull), shell=shell)
        output, unused_err = process.communicate()
        if process.poll() and fail:
            raise Exception()
    except Exception:
        raise FatalError(_("Error running command: %s") % cmd)
    return output

def apply_patch(patch, directory, strip=1):
    '''
    Apply a patch

    @param patch: path of the patch file
    @type patch: str
    @param directory: directory to apply the apply
    @type: directory: str
    @param strip: strip
    @type strip: int
    '''
    logging.info("Applying patch %s" % (patch))
    call('%s -p%s -f -i %s', (PATCH, strip, patch))

def unpack(filepath, output_dir):
    '''
    Extracts a tarball

    @param filepath: path of the tarball
    @type filepath: str
    @param output_dir: output directory
    @type output_dir: str
    '''
    logging.info("Unpacking %s in %s" % (filepath, output_dir))
    if filepath.endswith('tar.gz') or filepath.endswith('tar.bz2'):
        tf = tarfile.open(filepath, mode='r:*')
        tf.extractall(path=output_dir)
    if filepath.endswith('tar.xz'):
        call("%s -Jxf %s" % (TAR, filepath), output_dir)
    if filepath.endswith('.zip'):
        zf = zipfile.ZipFile(filepath, "r")
        zf.extractall(path=output_dir)

def download(url, destination=None, recursive=False):
    '''
    Downloads a file with wget

    @param url: url to download
    @type: str
    @param destination: destination where the file will be saved
    @type destination: str
    '''
    cmd = "wget %s " % url
    path = None
    if recursive:
        cmd += "-r "
        path = destination
    else:
        if destination is not None:
            cmd += "-O %s " % destination

    if not recursive and os.path.exists(destination):
        logging.info("File %s already downloaded." % destination)
    else:
        logging.info("Downloading %s", url)
        try:
            call(cmd, path)
        except FatalError, e:
            os.remove(tarfile)
            raise e

def _splitter(string, base_url):
    lines = string.split('\n')
    for line in lines:
        try:
            yield "%s/%s" % (base_url, line.split(' ')[2])
        except:
            continue

def recursive_download(url, destination):
    '''
    Recursive download for servers that don't return a list a url's but only
    the index.html file
    '''
    raw_list = check_call('curl %s' % url)

    with tempfile.NamedTemporaryFile() as f:
        for path in _splitter(raw_list, url):
            f.file.write(path + '\n')
        if not os.path.exists(destination):
            os.makedirs(destination)
        call("wget -i %s %s" % (f.name, url), destination)

def ls_files(files, prefix):
    if files == []:
        return files
    sfiles = check_call('ls %s' % ' '.join(files),
        prefix, True, False, False).split('\n')
    sfiles.remove('')
    return list(set(sfiles))

def find_newer_files(prefix, compfile):
    sfiles = check_call('find -L * -type f -newer %s' % compfile,
        prefix, True, False, False).split('\n')
    sfiles.remove('')
    return sfiles

def replace(filepath, replacements):
    ''' Replaces keys in the 'replacements' dict with their values in file '''
    with open(filepath, 'r') as f:
        content = f.read()
    for k, v in replacements.iteritems():
        content = content.replace(k, v)
    with open(filepath, 'w+') as f:
        f.write(content)

def find_files(pattern, prefix):
    return glob.glob(os.path.join(prefix, pattern))

def prompt(message, options=[]):
    ''' Prompts the user for input with the message and options '''
    if len(options) != 0:
        message = "%s [%s]" % (message, '/'.join(options))
    res = raw_input(message)
    while res not in [str(x) for x in options]:
        res = raw_input(message)
    return res
<commit_msg>shell: fix function to replace paths<commit_after># cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import logging
import subprocess
import shlex
import sys
import os
import tarfile
import zipfile
import tempfile
import glob

from cerbero.config import Platform
from cerbero.utils import _, system_info
from cerbero.errors import FatalError

PATCH = 'patch'
TAR = 'tar'

PLATFORM = system_info()[0]
LOGFILE = None  # open('/tmp/cerbero.log', 'w+')

class StdOut:

    def __init__(self, stream=sys.stdout):
        self.stream = stream

    def write(self, data):
        self.stream.write(data)
        self.stream.flush()

    def __getattr__(self, attr):
        return getattr(self.stream, attr)

def _fix_mingw_cmd(path):
    reserved = ['/', ' ', '\\', ')', '(', '"']
    l_path = list(path)
    for i in range(len(path)):
        if path[i] == '\\':
            if i+1 == len(path) or path [i+1] not in reserved:
                l_path[i] = '/'
    return ''.join(l_path)

def call(cmd, cmd_dir='.', fail=True):
    '''
    Run a shell command

    @param cmd: the command to run
    @type cmd: str
    @param cmd_dir: directory where the command will be run
    @param cmd_dir: str
    @param fail: wheter to raise an exception if the command failed or not
    @type fail: bool
    '''
    try:
        logging.info("Running command '%s'" % cmd)
        shell = True
        if PLATFORM == Platform.WINDOWS:
            # windows do not understand ./
            if cmd.startswith('./'):
                cmd = cmd[2:]
            # run all processes through sh.exe to get scripts working
            cmd = '%s "%s"' % ('sh -c', cmd)
            # fix paths with backslashes
            cmd = _fix_mingw_cmd(cmd)
            # Disable shell which uses cmd.exe
            shell = False
        stream = LOGFILE or sys.stdout
        ret = subprocess.check_call(cmd, cwd=cmd_dir,
                stderr=subprocess.STDOUT, stdout=StdOut(stream),
                env=os.environ.copy(), shell=shell)
    except subprocess.CalledProcessError:
        if fail:
            raise FatalError(_("Error running command: %s") % cmd)
        else:
            ret = 0
    return ret

def check_call(cmd, cmd_dir=None, shell=False, split=True, fail=False):
    try:
        if split:
            cmd = shlex.split(cmd)
        process = subprocess.Popen(cmd, cwd=cmd_dir,
            stdout=subprocess.PIPE, stderr=open(os.devnull), shell=shell)
        output, unused_err = process.communicate()
        if process.poll() and fail:
            raise Exception()
    except Exception:
        raise FatalError(_("Error running command: %s") % cmd)
    return output

def apply_patch(patch, directory, strip=1):
    '''
    Apply a patch

    @param patch: path of the patch file
    @type patch: str
    @param directory: directory to apply the apply
    @type: directory: str
    @param strip: strip
    @type strip: int
    '''
    logging.info("Applying patch %s" % (patch))
    call('%s -p%s -f -i %s', (PATCH, strip, patch))

def unpack(filepath, output_dir):
    '''
    Extracts a tarball

    @param filepath: path of the tarball
    @type filepath: str
    @param output_dir: output directory
    @type output_dir: str
    '''
    logging.info("Unpacking %s in %s" % (filepath, output_dir))
    if filepath.endswith('tar.gz') or filepath.endswith('tar.bz2'):
        tf = tarfile.open(filepath, mode='r:*')
        tf.extractall(path=output_dir)
    if filepath.endswith('tar.xz'):
        call("%s -Jxf %s" % (TAR, filepath), output_dir)
    if filepath.endswith('.zip'):
        zf = zipfile.ZipFile(filepath, "r")
        zf.extractall(path=output_dir)

def download(url, destination=None, recursive=False):
    '''
    Downloads a file with wget

    @param url: url to download
    @type: str
    @param destination: destination where the file will be saved
    @type destination: str
    '''
    cmd = "wget %s " % url
    path = None
    if recursive:
        cmd += "-r "
        path = destination
    else:
        if destination is not None:
            cmd += "-O %s " % destination

    if not recursive and os.path.exists(destination):
        logging.info("File %s already downloaded." % destination)
    else:
        logging.info("Downloading %s", url)
        try:
            call(cmd, path)
        except FatalError, e:
            os.remove(tarfile)
            raise e

def _splitter(string, base_url):
    lines = string.split('\n')
    for line in lines:
        try:
            yield "%s/%s" % (base_url, line.split(' ')[2])
        except:
            continue

def recursive_download(url, destination):
    '''
    Recursive download for servers that don't return a list a url's but only
    the index.html file
    '''
    raw_list = check_call('curl %s' % url)

    with tempfile.NamedTemporaryFile() as f:
        for path in _splitter(raw_list, url):
            f.file.write(path + '\n')
        if not os.path.exists(destination):
            os.makedirs(destination)
        call("wget -i %s %s" % (f.name, url), destination)

def ls_files(files, prefix):
    if files == []:
        return files
    sfiles = check_call('ls %s' % ' '.join(files),
        prefix, True, False, False).split('\n')
    sfiles.remove('')
    return list(set(sfiles))

def find_newer_files(prefix, compfile):
    sfiles = check_call('find -L * -type f -newer %s' % compfile,
        prefix, True, False, False).split('\n')
    sfiles.remove('')
    return sfiles

def replace(filepath, replacements):
    ''' Replaces keys in the 'replacements' dict with their values in file '''
    with open(filepath, 'r') as f:
        content = f.read()
    for k, v in replacements.iteritems():
        content = content.replace(k, v)
    with open(filepath, 'w+') as f:
        f.write(content)

def find_files(pattern, prefix):
    return glob.glob(os.path.join(prefix, pattern))

def prompt(message, options=[]):
    ''' Prompts the user for input with the message and options '''
    if len(options) != 0:
        message = "%s [%s]" % (message, '/'.join(options))
    res = raw_input(message)
    while res not in [str(x) for x in options]:
        res = raw_input(message)
    return res
<|endoftext|>


<commit_before>#!/bin/env python3
"""
Manage tasks interactively

Usage:
    task_manage.py add
    task_manage.py search [-t|-p]
    task_manage.py dep [-r] [<id>]
    task_manage.py review [<filter>]
    task_manage.py review [-a | -n <count>]

Options:
    -h, --help  Show this screen and exit.

Commands:
    add             Add new task
    search          Fuzzy search for task(+annotations) and retrieve id
    review          Review tasks, by default will review the top 20 tasks
    dep             Add dependencies to supplied id. Blocking it until selected
                        tasks are done

dep:
    -r, --reverse  Reverse dependency. Supplied task id will be parent of all
                    selected tasks
search:
    -p, --projects  Search for projects
    -t, --tags  Search for tags

review:
    -n <count>, --number <count>  Number of top tasks to review[default: 20]
    -a, --all  Review all tasks which are flagged for review

"""

import sys
from docopt import docopt
from tasklib import TaskWarrior,Task
from prompt_toolkit.validation import Validator, ValidationError
from prompt_toolkit import prompt
from prompt_toolkit.history import InMemoryHistory
from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
from prompt_toolkit.interface import AbortAction
from iterfzf import iterfzf
import pprint
from datetime import datetime,timedelta
import random
import logging
from itertools import tee
import re

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# docopt(doc, argv=None, help=True, version=None, options_first=False))
# TODO generate singleton for taskwarrior global

history = InMemoryHistory()

valid_search_attributes = ["project", "tag"]
tw = None

def main():
    opt = docopt(__doc__, sys.argv[1:])
    #print(opt)

    command = generate_command(opt,  **opt)
    if not command:
        logger.info("Command not legal.")
        sys.exit(1)

    command()

def generate_command(opts, *args, **kwargs):
    global tw
    tw = TaskWarrior()

    def generate_review():
        task_filter = kwargs.get("<filter>", "")
        review_all = kwargs.get("--all")
        top_count = kwargs.get("--number")

        if review_all:
            review_task("")
        elif task_filter:
            review_task(task_filter, force=True)
        else:
            for task_id in get_top_tasks(top_count):
                review_task(task_id, force=True)

    def generate_dependency():
        task_filter = kwargs.get("<id>", "")
        reverse_dependency = kwargs.get("--reverse")
        add_dependencies(task_filter, reverse_dependency)

    def generate_search():
        tasks = tw.tasks.pending()

        if kwargs.get("--projects"):
            search_attribute(tasks, "project")
        elif kwargs.get("--tags"):
            search_attribute(tasks, "tag")
        else:
            search_task(tasks)

    commands = {
            "add": add_task,
            "review": generate_review,
            "dep": generate_dependency,
            "search": generate_search,
            }

    command = ""
    for cmd in commands.keys():
        if opts.get(cmd):
            command = cmd

    return commands.get(command, "")

def get_top_tasks(count):
    import subprocess
    cmd = ["task", "limit:{}".format(count), "top"]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    valid_pattern = "^([0-9]+$)"
    for line in p.stdout:
        s = canonize_string(str(line,"UTF-8"))
        if string_match(valid_pattern, s):
            yield s

def search_task(tasks):
    tasks_mapping = ( to_string_task_simple(x, annotations=True) for x in tasks )
    task = iterfzf(tasks_mapping)
    if not task:
        return
    task_id = task.split()[0]
    task = tasks.get(id=task_id)
    print(to_string_task_full(task))
    return task

def search_attribute(tasks, attribute):
    """
    search for specific attribute - either
        'tag'
        'project'
    """
    global valid_search_attributes
    if attribute not in valid_search_attributes:
        logger.info("Supplied attribute {} is not valid(or implemented) for search".format(attribute))
        sys.exit(1)

    import subprocess
    cmd = [ "task", "{}s".format(attribute) ]
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)

    items = []
    invalid_pattern = "^([0-9- ()]|Project|Tag)"
    for line in process.stdout:
        line = str(line.strip(), "utf-8")
        if string_match(invalid_pattern, line) or line == "":
            continue
        items.append(line)
    items = iterfzf(items, multi=True)
    items = [ item.strip().split()[0] for item in items ]

    if attribute == "project":
        filter_string = [ "project:{}".format(x) for x in items ]
    elif attribute == "tag":
        filter_string = [ "+{}".format(x) for x in items ]

    if len(items) > 1:
        tasks = tasks.filter(" or ".join(filter_string))
    else:
        tasks = tasks.filter("".join(filter_string))

    for task in tasks:
        print(to_string_task_simple(task))

def string_match(pattern, string):
    match = re.match(pattern, string)
    if match:
        return True
    else:
        return False

def add_dependencies(child_task, reverse_dependency=False, status="pending"):
    """
    add dependencies to a parent task
    """
    from copy import deepcopy

    global tw
    tasks = tw.tasks.pending()

    if not child_task:
        child_task = search_task(tasks)
    else:
        try:
            child_task = tasks.get(id=child_task)
        except Task.DoesNotExist as e:
            logger.error("Could not find parent task to apply dependencies for.", exc_info=True)
            sys.exit(1)

    if not child_task:
        logger.error("Task not valid. Abort.")
        sys.exit(1)

    available_tasks = filter_task(child_task, tasks)

    dependencies = []
    while True:
        available_tasks, taskgencopy = tee(available_tasks)
        taskgencopy = ( to_string_task_simple(x) for x in taskgencopy )
        parent_dependency = iterfzf(taskgencopy)
        if not parent_dependency:
            break
        try:
            parent_dependency = parent_dependency.split()[0]
            parent_dependency = tasks.get(id=parent_dependency)
        except Task.DoesNotExist as e:
            logger.error(e)
            logger.error("Task with id {} could not be found.".format(parent_dependency))

        dependencies.append(parent_dependency)
        available_tasks = filter_task(parent_dependency, available_tasks)

    task_dependencies = child_task["depends"]
    for dependency in dependencies:
        # add dependency to child - blocking it until dependency is done
        if not reverse_dependency:
            task_dependencies.add(dependency)
        # add dependency to selected task blocking them until 'child_task'(the
        #   parent now) is done
        else:
            dependency["depends"].add(child_task)
            dependency.save()

    child_task["depends"] = task_dependencies
    child_task.save()

def filter_task(task, tasks):
    for t in tasks:
        if t["id"] == task["id"]:
            continue
        yield t

def to_string_task_simple(task, annotations=False):
    description = canonize_string(task["description"])
    if annotations:
        description += " - {}".format(task["annotations"])
    task_id = task["id"]
    return "{}  {}".format(task_id, description)

def to_string_task_full(task):
    output = "[{}]  {}".format(task["id"], task["description"])
    projects = task["projects"]
    output += "\nProjects: "
    if projects:
        output += "{}".format(", ".join(projects))
    else:
        output += "{}".format(projects)

    tags = task["tags"]
    output += "\nTags: "
    if tags:
        output += "{}".format(", ".join(tags))
    else:
        output += "{}".format(tags)

    output += "\nDependencies: {}".format(task["depends"])

    annotations = task["annotations"]
    output += "\nAnnotations: "
    if tags:
        for ann in annotations:
            output += "\n - {}".format(ann)
    else:
        output += "{}".format(annotations)

    return output

def canonize_string(string):
    return string.strip().replace("\n"," ")

def add_task():
    global tw
    config = tw.config.items()

    udas_new_map = dict()

    description = prompt_value("Task description", exit_if_empty=True)
    udas_new_map.update({"description":description})

    project = prompt_value("Project")
    udas_new_map.update({"project":project})

    tags = []
    for tag in prompt_items("Add a tag."):
        tags.append(tag)
    udas_new_map.update({"tags": tags})

    annotations = []
    for annotation in prompt_items("Add an annotation."):
        annotations.append(annotation)

    config_udas = parse_udas(config)
    config_udas, udas = tee(config_udas)
    for uda in udas:
        uda, defaults = canonize_uda(uda)
        menu = create_interactive_menu(uda, defaults)
        validator = SelectMenuValidator(uda, defaults)

        while True:
            print(menu)
            value = prompt_value(validator=validator)
            if value:
                break
        udas_new_map.update({uda: value})

    print("The new task:")
    pp = pprint.PrettyPrinter(indent=4)
    pp.pprint(udas_new_map)
    if prompt_confirm():
        new_task = Task(tw, **udas_new_map)
        new_task.save()
        if new_task.saved:
            for annotation in annotations:
                new_task.add_annotation(annotation)
            print(new_task["id"])
        else:
            logger.info("Task has not been saved.")
    else:
        logger.info("Did not add task.")

def prompt_items(string):
    while True:
        tag = prompt_value(string)
        if not tag:
            break
        yield tag

def review_task(taskfilter, status="pending", force=False):
    global tw
    config = tw.config.items()
    config_udas = parse_udas(config)

    if taskfilter:
        pending_tasks = tw.tasks.filter(taskfilter, status=status)
    else:
        pending_tasks = tw.tasks.pending()

    if not pending_tasks:
        logger.info("No tasks found with status '{}'.".format(status))
        sys.exit(2)

    for task in pending_tasks:
        if not (force or review_required(task)):
            continue

        udas_new_map = dict()
        print("""
################################################################################
Updating task:
 id: {}
 description: {}
 annotations:
    {}
 project: {}
 tags: {}
""".format(
    task["id"],
    task["description"],
    task["annotations"],
    task["project"],
    task["tags"],
    ))

        config_udas, udas = tee(config_udas)
        for uda in udas:
            uda, defaults = canonize_uda(uda)
            menu = create_interactive_menu(uda, defaults)
            validator = SelectMenuValidator(uda, defaults)

            while True:
                print(menu)
                value = prompt_value(validator=validator,
                    default=task[uda])
                if value:
                    break
            udas_new_map.update({uda: value})

        print("The new task:")
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(udas_new_map)
        if prompt_confirm():
            update_task(task, **udas_new_map)
        else:
            logger.info("Did not continue updating task.")

def update_task(task, **kwargs):
    for k, v in kwargs.items():
        task[k] = v

    review_day = timedelta(days=random.randint(21,28))
    date = datetime.now() + review_day
    task["review"] = format_datetime_iso(date)
    task.save()

    if task.saved:
        logger.info(task["id"])
    else:
        logger.info("Task has not been updated.")

def prompt_value(string="", validator=None, default="", exit_if_empty=False):
    try:
        user_input = prompt("{}> ".format(string),
                history=history,
                validator=validator,
                default=default,
                on_abort=AbortAction.RETRY,
                erase_when_done=True,
                auto_suggest=AutoSuggestFromHistory())
    except InputError:
        user_input = "?"
    except (KeyboardInterrupt, EOFError):
        print("bye")
        sys.exit(0)
    if not user_input and exit_if_empty:
        logger.error("No input. Exiting now.")
        sys.exit(1)
    return user_input

def review_required(task):
    now = datetime.now().date()
    day90old = now - timedelta(days=90)

    modified = task["modified"]
    review = task["review"]

    review_task = False

    if not review:
        return True

    if modified:
        modified = modified.date()
        review_task = day90old > modified

    if review_task:
        try:
            review = datetime.strptime(review, "%Y%m%dT%H%M%S.%f").date()
        except ValueError:
            logger.error("Could not parse review date of task with id {}".format(task["id"]))
            sys.exit(1)

        review_task = now > review

    return review_task

def format_datetime_iso(date):
    """
    return iso string
    """
    return date.isoformat().replace("-","").replace(":","")

class SelectMenuValidator(Validator):
    def __init__(self, uda, defaults):
        self.uda = uda
        self.defaults = defaults

    def validate(self, document):
        text = document.text

        if text == "":
            raise InputError("Input is empty")
        elif text not in self.defaults:
            raise ValidationError(message="Choice is not a valid uda.")

class InputError(Exception):
    pass

def canonize_uda(uda):
    uda, defaults = uda
    # uda.name.default
    uda = uda.split(".")[1]
    # 0,1,2,3,...
    defaults = defaults.split(",")

    return uda,defaults

def parse_udas(config):
    uda_filter = build_filter("uda")
    default_filter = build_filter("default")

    config = filter(uda_filter, config)
    config = filter(default_filter, config )

    return config

def build_filter(string, negate=False):
    def filter_search(item):
        k,v = item
        if string in k and not negate:
            return True
        else:
            return False
    return filter_search

def run_menu(menu, validator,  default="?"):

    if choice == "":
        return default
    elif choice is False:
        return False

def prompt_confirm(string=""):
    print(string)
    try:
        user_input = prompt("Do you want to continue?[yn]",
                history=history,
                on_abort=AbortAction.RETRY,
                auto_suggest=AutoSuggestFromHistory())
    except (KeyboardInterrupt, EOFError):
        print("bye")
        sys.exit(0)
    if user_input.lower() in ["y","yes","ye","j","ja"]:
        return True
    else:
        return False

def create_interactive_menu(uda, values):
    output = "{:#^40}".format(" {} ".format(uda))
    output += "\nSelect from defaults: \n\t{}".format(values)
    return output

if __name__ == "__main__":
    main()
<commit_msg>Refactor logging output<commit_after>#!/bin/env python3
"""
Manage tasks interactively

Usage:
    task_manage.py add
    task_manage.py search [-t|-p]
    task_manage.py dep [-r] [<id>]
    task_manage.py review [<filter>]
    task_manage.py review [-a | -n <count>]

Options:
    -h, --help  Show this screen and exit.

Commands:
    add             Add new task
    search          Fuzzy search for task(+annotations) and retrieve id
    review          Review tasks, by default will review the top 20 tasks
    dep             Add dependencies to supplied id. Blocking it until selected
                        tasks are done

dep:
    -r, --reverse  Reverse dependency. Supplied task id will be parent of all
                    selected tasks
search:
    -p, --projects  Search for projects
    -t, --tags  Search for tags

review:
    -n <count>, --number <count>  Number of top tasks to review[default: 20]
    -a, --all  Review all tasks which are flagged for review

"""

import sys
from docopt import docopt
from tasklib import TaskWarrior,Task
from prompt_toolkit.validation import Validator, ValidationError
from prompt_toolkit import prompt
from prompt_toolkit.history import InMemoryHistory
from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
from prompt_toolkit.interface import AbortAction
from iterfzf import iterfzf
import pprint
from datetime import datetime,timedelta
import random
import logging
from itertools import tee
import re

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# docopt(doc, argv=None, help=True, version=None, options_first=False))
# TODO generate singleton for taskwarrior global

history = InMemoryHistory()

valid_search_attributes = ["project", "tag"]
tw = None

def main():
    opt = docopt(__doc__, sys.argv[1:])
    #print(opt)

    command = generate_command(opt,  **opt)
    if not command:
        logger.info("Command not legal.")
        sys.exit(1)

    command()

def generate_command(opts, *args, **kwargs):
    global tw
    tw = TaskWarrior()

    def generate_review():
        task_filter = kwargs.get("<filter>", "")
        review_all = kwargs.get("--all")
        top_count = kwargs.get("--number")

        if review_all:
            review_task("")
        elif task_filter:
            review_task(task_filter, force=True)
        else:
            for task_id in get_top_tasks(top_count):
                review_task(task_id, force=True)

    def generate_dependency():
        task_filter = kwargs.get("<id>", "")
        reverse_dependency = kwargs.get("--reverse")
        add_dependencies(task_filter, reverse_dependency)

    def generate_search():
        tasks = tw.tasks.pending()

        if kwargs.get("--projects"):
            search_attribute(tasks, "project")
        elif kwargs.get("--tags"):
            search_attribute(tasks, "tag")
        else:
            search_task(tasks)

    commands = {
            "add": add_task,
            "review": generate_review,
            "dep": generate_dependency,
            "search": generate_search,
            }

    command = ""
    for cmd in commands.keys():
        if opts.get(cmd):
            command = cmd

    return commands.get(command, "")

def get_top_tasks(count):
    import subprocess
    cmd = ["task", "limit:{}".format(count), "top"]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    valid_pattern = "^([0-9]+$)"
    for line in p.stdout:
        s = canonize_string(str(line,"UTF-8"))
        if string_match(valid_pattern, s):
            yield s

def search_task(tasks):
    tasks_mapping = ( to_string_task_simple(x, annotations=True) for x in tasks )
    task = iterfzf(tasks_mapping)
    if not task:
        return
    task_id = task.split()[0]
    task = tasks.get(id=task_id)
    print(to_string_task_full(task))
    return task

def search_attribute(tasks, attribute):
    """
    search for specific attribute - either
        'tag'
        'project'
    """
    global valid_search_attributes
    if attribute not in valid_search_attributes:
        logger.info("Supplied attribute {} is not valid(or implemented) for search".format(attribute))
        sys.exit(1)

    import subprocess
    cmd = [ "task", "{}s".format(attribute) ]
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)

    items = []
    invalid_pattern = "^([0-9- ()]|Project|Tag)"
    for line in process.stdout:
        line = str(line.strip(), "utf-8")
        if string_match(invalid_pattern, line) or line == "":
            continue
        items.append(line)
    items = iterfzf(items, multi=True)
    items = [ item.strip().split()[0] for item in items ]

    if attribute == "project":
        filter_string = [ "project:{}".format(x) for x in items ]
    elif attribute == "tag":
        filter_string = [ "+{}".format(x) for x in items ]

    if len(items) > 1:
        tasks = tasks.filter(" or ".join(filter_string))
    else:
        tasks = tasks.filter("".join(filter_string))

    for task in tasks:
        print(to_string_task_simple(task))

def string_match(pattern, string):
    match = re.match(pattern, string)
    if match:
        return True
    else:
        return False

def add_dependencies(child_task, reverse_dependency=False, status="pending"):
    """
    add dependencies to a parent task
    """
    from copy import deepcopy

    global tw
    tasks = tw.tasks.pending()

    if not child_task:
        child_task = search_task(tasks)
    else:
        try:
            child_task = tasks.get(id=child_task)
        except Task.DoesNotExist as e:
            logger.error("Could not find parent task to apply dependencies for.", exc_info=True)
            sys.exit(1)

    if not child_task:
        logger.error("Task not valid. Abort.")
        sys.exit(1)

    available_tasks = filter_task(child_task, tasks)

    dependencies = []
    while True:
        available_tasks, taskgencopy = tee(available_tasks)
        taskgencopy = ( to_string_task_simple(x) for x in taskgencopy )
        parent_dependency = iterfzf(taskgencopy)
        if not parent_dependency:
            break
        try:
            parent_dependency = parent_dependency.split()[0]
            parent_dependency = tasks.get(id=parent_dependency)
        except Task.DoesNotExist as e:
            logger.error(e)
            logger.error("Task with id {} could not be found.".format(parent_dependency))

        dependencies.append(parent_dependency)
        available_tasks = filter_task(parent_dependency, available_tasks)

    task_dependencies = child_task["depends"]
    for dependency in dependencies:
        # add dependency to child - blocking it until dependency is done
        if not reverse_dependency:
            task_dependencies.add(dependency)
        # add dependency to selected task blocking them until 'child_task'(the
        #   parent now) is done
        else:
            dependency["depends"].add(child_task)
            dependency.save()

    child_task["depends"] = task_dependencies
    child_task.save()

def filter_task(task, tasks):
    for t in tasks:
        if t["id"] == task["id"]:
            continue
        yield t

def to_string_task_simple(task, annotations=False):
    description = canonize_string(task["description"])
    if annotations:
        description += " - {}".format(task["annotations"])
    task_id = task["id"]
    return "{}  {}".format(task_id, description)

def to_string_task_full(task):
    output = "[{}]  {}".format(task["id"], task["description"])
    projects = task["projects"]
    output += "\nProjects: "
    if projects:
        output += "{}".format(", ".join(projects))
    else:
        output += "{}".format(projects)

    tags = task["tags"]
    output += "\nTags: "
    if tags:
        output += "{}".format(", ".join(tags))
    else:
        output += "{}".format(tags)

    output += "\nDependencies: {}".format(task["depends"])

    annotations = task["annotations"]
    output += "\nAnnotations: "
    if tags:
        for ann in annotations:
            output += "\n - {}".format(ann)
    else:
        output += "{}".format(annotations)

    return output

def canonize_string(string):
    return string.strip().replace("\n"," ")

def add_task():
    global tw
    config = tw.config.items()

    udas_new_map = dict()

    description = prompt_value("Task description", exit_if_empty=True)
    udas_new_map.update({"description":description})

    project = prompt_value("Project")
    udas_new_map.update({"project":project})

    tags = []
    for tag in prompt_items("Add a tag."):
        tags.append(tag)
    udas_new_map.update({"tags": tags})

    annotations = []
    for annotation in prompt_items("Add an annotation."):
        annotations.append(annotation)

    config_udas = parse_udas(config)
    config_udas, udas = tee(config_udas)
    for uda in udas:
        uda, defaults = canonize_uda(uda)
        menu = create_interactive_menu(uda, defaults)
        validator = SelectMenuValidator(uda, defaults)

        while True:
            print(menu)
            value = prompt_value(validator=validator)
            if value:
                break
        udas_new_map.update({uda: value})

    print("The new task:")
    pp = pprint.PrettyPrinter(indent=4)
    pp.pprint(udas_new_map)
    if prompt_confirm():
        new_task = Task(tw, **udas_new_map)
        new_task.save()
        if new_task.saved:
            for annotation in annotations:
                new_task.add_annotation(annotation)
            print(new_task["id"])
        else:
            logger.info("Task has not been saved.")
    else:
        logger.info("Did not add task.")

def prompt_items(string):
    while True:
        tag = prompt_value(string)
        if not tag:
            break
        yield tag

def review_task(taskfilter, status="pending", force=False):
    global tw
    config = tw.config.items()
    config_udas = parse_udas(config)

    if taskfilter:
        pending_tasks = tw.tasks.filter(taskfilter, status=status)
    else:
        pending_tasks = tw.tasks.pending()

    if not pending_tasks:
        logger.info("No tasks found with status '{}'.".format(status))
        sys.exit(2)

    for task in pending_tasks:
        if not (force or review_required(task)):
            continue

        udas_new_map = dict()
        print("""
################################################################################
Updating task:
 id: {}
 description: {}
 annotations:
    {}
 project: {}
 tags: {}
""".format(
    task["id"],
    task["description"],
    task["annotations"],
    task["project"],
    task["tags"],
    ))

        config_udas, udas = tee(config_udas)
        for uda in udas:
            uda, defaults = canonize_uda(uda)
            menu = create_interactive_menu(uda, defaults)
            validator = SelectMenuValidator(uda, defaults)

            while True:
                print(menu)
                value = prompt_value(validator=validator,
                    default=task[uda])
                if value:
                    break
            udas_new_map.update({uda: value})

        print("The new task:")
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(udas_new_map)
        if prompt_confirm():
            update_task(task, **udas_new_map)
        else:
            logger.info("Did not continue updating task.")

def update_task(task, **kwargs):
    for k, v in kwargs.items():
        task[k] = v

    review_day = timedelta(days=random.randint(21,28))
    date = datetime.now() + review_day
    task["review"] = format_datetime_iso(date)
    task.save()

    if task.saved:
        logger.info("Updated task: {}".format(task["id"]))
    else:
        logger.info("Task has not been updated.")

def prompt_value(string="", validator=None, default="", exit_if_empty=False):
    try:
        user_input = prompt("{}> ".format(string),
                history=history,
                validator=validator,
                default=default,
                on_abort=AbortAction.RETRY,
                erase_when_done=True,
                auto_suggest=AutoSuggestFromHistory())
    except InputError:
        user_input = "?"
    except (KeyboardInterrupt, EOFError):
        print("bye")
        sys.exit(0)
    if not user_input and exit_if_empty:
        logger.error("No input. Exiting now.")
        sys.exit(1)
    return user_input

def review_required(task):
    now = datetime.now().date()
    day90old = now - timedelta(days=90)

    modified = task["modified"]
    review = task["review"]

    review_task = False

    if not review:
        return True

    if modified:
        modified = modified.date()
        review_task = day90old > modified

    if review_task:
        try:
            review = datetime.strptime(review, "%Y%m%dT%H%M%S.%f").date()
        except ValueError:
            logger.error("Could not parse review date of task with id {}".format(task["id"]))
            sys.exit(1)

        review_task = now > review

    return review_task

def format_datetime_iso(date):
    """
    return iso string
    """
    return date.isoformat().replace("-","").replace(":","")

class SelectMenuValidator(Validator):
    def __init__(self, uda, defaults):
        self.uda = uda
        self.defaults = defaults

    def validate(self, document):
        text = document.text

        if text == "":
            raise InputError("Input is empty")
        elif text not in self.defaults:
            raise ValidationError(message="Choice is not a valid uda.")

class InputError(Exception):
    pass

def canonize_uda(uda):
    uda, defaults = uda
    # uda.name.default
    uda = uda.split(".")[1]
    # 0,1,2,3,...
    defaults = defaults.split(",")

    return uda,defaults

def parse_udas(config):
    uda_filter = build_filter("uda")
    default_filter = build_filter("default")

    config = filter(uda_filter, config)
    config = filter(default_filter, config )

    return config

def build_filter(string, negate=False):
    def filter_search(item):
        k,v = item
        if string in k and not negate:
            return True
        else:
            return False
    return filter_search

def run_menu(menu, validator,  default="?"):

    if choice == "":
        return default
    elif choice is False:
        return False

def prompt_confirm(string=""):
    print(string)
    try:
        user_input = prompt("Do you want to continue?[yn]",
                history=history,
                on_abort=AbortAction.RETRY,
                auto_suggest=AutoSuggestFromHistory())
    except (KeyboardInterrupt, EOFError):
        print("bye")
        sys.exit(0)
    if user_input.lower() in ["y","yes","ye","j","ja"]:
        return True
    else:
        return False

def create_interactive_menu(uda, values):
    output = "{:#^40}".format(" {} ".format(uda))
    output += "\nSelect from defaults: \n\t{}".format(values)
    return output

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import os

import json

import collections

from shapely.geometry.point import BaseGeometry
from shapely.geometry import shape, mapping
from shapely.ops import cascaded_union

import util

class Feature(object):
    """単一の feature を扱うためのクラス
    """
    def __init__(self, data):
        """feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        self.load(data)

    def load(self, data):
        """data を元に feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        if isinstance(data, Feature):
            self.geometry = data.geometry
            self.properties = data.properties
            self.attributes = data.attributes
        else:
            self.geometry = data['geometry']
            self.properties = data['properties']
            self.attributes = {
                k: v for k, v in data.items()
                if k not in set(('geometry', 'properties', 'type'))
            }
        return self

    def dump(self):
        """このインスタンスを表す、json.dumpsなどでダンプ可能なオブジェクトを返す
        """
        return dict({u'type': u'Feature',
                     u'geometry': util.rec_decode(mapping(self._geometry)),
                     u'properties': self._properties}, **self._attributes)

    @property
    def properties(self):
        """このインスタンスの properties オブジェクト自体を返す
        """
        return self._properties

    @properties.setter
    def properties(self, x):
        """このインスタンスの properties を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('properties must be an instance of Mapping')
        self._properties = util.rec_decode(x)
        return self

    @property
    def geometry(self):
        """このインスタンスの geometry オブジェクト自体を返す
        """
        return self._geometry

    @geometry.setter
    def geometry(self, x):
        """このインスタンの geometry を設定する

        :param x: shape か shape に変換可能な Mapping オブジェクト
        """
        if isinstance(x, BaseGeometry):
            self._geometry = x
        elif util.is_map(x):
            self._geometry = shape(x)
        else:
            raise Exception('geometry must be an instance of shape')
        return self

    @property
    def attributes(self):
        """このインスタンスのその他の属性の Mapping オブジェクトを返す
        """
        return self._attributes

    @attributes.setter
    def attributes(self, x):
        """このインスタンスのその他の属性を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('attributes must be an instance of Mapping')
        self._attributes = util.rec_decode(x)
        return self

    def property(self, *x):
        """properties を set/get する。
        property('id') id 属性の値を取得する
        property('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]).items():
                    self.property(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.property(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.property(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._properties:
                return self._properties[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.rec_decode(x[1])
        if v is None:
            if k in self._properties:
                del self._properties[k]
            return self
        self._properties[k] = v
        return self

    def attr(self, *x):
        """attributes を set/get する。
        attr('id') id 属性の値を取得する
        attr('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]).items():
                    self.attr(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.attr(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._attributes:
                return self._attributes[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.rec_decode(x[1])
        if v is None:
            if k in self._attributes:
                del self._attributes[k]
            return self
        self._attributes[k] = v
        return self

class FeatureCollection(collections.MutableSequence):
    def __init__(self, x):
        super(FeatureCollection, self).__init__()
        self._features = []
        self._attributes = {}
        self.load(x)

    def load(self, x):
        """Load an instance from FeatureCollection or a Mapping object
        with 'features' attribute.
        """
        if isinstance(x, FeatureCollection):
            self._features = x._features
            self._attributes = x._attributes
        elif util.is_map(x):
            self._attributes = util.rec_decode({
                k: v for k, v in x.items()
                if k not in set(('features', 'type'))})
            self._features = [Feature(y) for y in x['features']]
        return self

    def dump(self):
        """Return dict object represents this instance.
        """
        return util.rec_decode(dict({
            'type': 'FeatureCollection',
            'features': [f.dump() for f in self._features]
        }, **self._attributes))

    def attr(self, *x):
        """set/get attributes.

        attr('id'): Return value of 'id'

        attr('id', 'a123'): set value of 'id' to 'a123' then return self

        :param x: single key, list, dict, set, tuple or key-value pair
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]).items():
                    self.attr(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.attr(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._attributes:
                return self._attributes[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.rec_decode(x[1])
        if v is None:
            if k in self._attributes:
                del self._attributes[k]
            return self
        self._attributes[k] = v
        return self

    def aggregate(self, keys=None, property=None, attribute=None):
        pass

    @property
    def attributes(self):
        """Return attributes of this instance.
        """
        return self._attributes

    @attributes.setter
    def attributes(self, x):
        """Set attributes of this instance. Return self.

        :param x: mapping
        """
        if not util.is_map(x):
            raise Exception('attributes must be a mapping.')
        self._attributes = util.rec_decode(x)
        return self

    @property
    def features(self):
        """Return features
        """
        return self._features

    @features.setter
    def features(self, x):
        """Set features of this instance. Return self.

        :param x: linear iterable
        """
        if not util.is_array(x):
            raise Exception('features must be a linear iterable container.')
        self._features = [Feature(y) for y in x]
        return self

    def __iter__(self):
        return self._features.__iter__()

    def __getitem__(self, i):
        return self._features.__getitem__(i)

    def __setitem__(self, i, x):
        return self._features.__setitem__(i, Feature(x))

    def __delitem__(self, i):
        return self._features.__delitem__(i)

    def __len__(self):
        return self._features.__len__()

    def insert(self, i, x):
        return self._features.insert(i, Feature(x))
<commit_msg>add comment to FeatureCollection.aggregate<commit_after># -*- coding: utf-8 -*-

import sys
import os

import json

import collections

from shapely.geometry.point import BaseGeometry
from shapely.geometry import shape, mapping
from shapely.ops import cascaded_union

import util

class Feature(object):
    """単一の feature を扱うためのクラス
    """
    def __init__(self, data):
        """feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        self.load(data)

    def load(self, data):
        """data を元に feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        if isinstance(data, Feature):
            self.geometry = data.geometry
            self.properties = data.properties
            self.attributes = data.attributes
        else:
            self.geometry = data['geometry']
            self.properties = data['properties']
            self.attributes = {
                k: v for k, v in data.items()
                if k not in set(('geometry', 'properties', 'type'))
            }
        return self

    def dump(self):
        """このインスタンスを表す、json.dumpsなどでダンプ可能なオブジェクトを返す
        """
        return dict({u'type': u'Feature',
                     u'geometry': util.rec_decode(mapping(self._geometry)),
                     u'properties': self._properties}, **self._attributes)

    @property
    def properties(self):
        """このインスタンスの properties オブジェクト自体を返す
        """
        return self._properties

    @properties.setter
    def properties(self, x):
        """このインスタンスの properties を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('properties must be an instance of Mapping')
        self._properties = util.rec_decode(x)
        return self

    @property
    def geometry(self):
        """このインスタンスの geometry オブジェクト自体を返す
        """
        return self._geometry

    @geometry.setter
    def geometry(self, x):
        """このインスタンの geometry を設定する

        :param x: shape か shape に変換可能な Mapping オブジェクト
        """
        if isinstance(x, BaseGeometry):
            self._geometry = x
        elif util.is_map(x):
            self._geometry = shape(x)
        else:
            raise Exception('geometry must be an instance of shape')
        return self

    @property
    def attributes(self):
        """このインスタンスのその他の属性の Mapping オブジェクトを返す
        """
        return self._attributes

    @attributes.setter
    def attributes(self, x):
        """このインスタンスのその他の属性を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('attributes must be an instance of Mapping')
        self._attributes = util.rec_decode(x)
        return self

    def property(self, *x):
        """properties を set/get する。
        property('id') id 属性の値を取得する
        property('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]).items():
                    self.property(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.property(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.property(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._properties:
                return self._properties[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.rec_decode(x[1])
        if v is None:
            if k in self._properties:
                del self._properties[k]
            return self
        self._properties[k] = v
        return self

    def attr(self, *x):
        """attributes を set/get する。
        attr('id') id 属性の値を取得する
        attr('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]).items():
                    self.attr(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.attr(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._attributes:
                return self._attributes[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.rec_decode(x[1])
        if v is None:
            if k in self._attributes:
                del self._attributes[k]
            return self
        self._attributes[k] = v
        return self

class FeatureCollection(collections.MutableSequence):
    def __init__(self, x):
        super(FeatureCollection, self).__init__()
        self._features = []
        self._attributes = {}
        self.load(x)

    def load(self, x):
        """Load an instance from FeatureCollection or a Mapping object
        with 'features' attribute.
        """
        if isinstance(x, FeatureCollection):
            self._features = x._features
            self._attributes = x._attributes
        elif util.is_map(x):
            self._attributes = util.rec_decode({
                k: v for k, v in x.items()
                if k not in set(('features', 'type'))})
            self._features = [Feature(y) for y in x['features']]
        return self

    def dump(self):
        """Return dict object represents this instance.
        """
        return util.rec_decode(dict({
            'type': 'FeatureCollection',
            'features': [f.dump() for f in self._features]
        }, **self._attributes))

    def attr(self, *x):
        """set/get attributes.

        attr('id'): Return value of 'id'

        attr('id', 'a123'): set value of 'id' to 'a123' then return self

        :param x: single key, list, dict, set, tuple or key-value pair
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]).items():
                    self.attr(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.attr(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._attributes:
                return self._attributes[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.rec_decode(x[1])
        if v is None:
            if k in self._attributes:
                del self._attributes[k]
            return self
        self._attributes[k] = v
        return self

    def aggregate(self, keys=None, property=None, attribute=None):
        """Return another FeatureCollection whose features are mereged
        according to the result of the keys function. property and attribute
        are used when features are reduced.

        :param keys: a function takes properties and attributes of the feature
        as arguments.
        :param property: a function takes keys and a tuple of properties
        :param attribute: a function takes keys and a tuple of attriutes
        """
        temp = {}

    @property
    def attributes(self):
        """Return attributes of this instance.
        """
        return self._attributes

    @attributes.setter
    def attributes(self, x):
        """Set attributes of this instance. Return self.

        :param x: mapping
        """
        if not util.is_map(x):
            raise Exception('attributes must be a mapping.')
        self._attributes = util.rec_decode(x)
        return self

    @property
    def features(self):
        """Return features
        """
        return self._features

    @features.setter
    def features(self, x):
        """Set features of this instance. Return self.

        :param x: linear iterable
        """
        if not util.is_array(x):
            raise Exception('features must be a linear iterable container.')
        self._features = [Feature(y) for y in x]
        return self

    def __iter__(self):
        return self._features.__iter__()

    def __getitem__(self, i):
        return self._features.__getitem__(i)

    def __setitem__(self, i, x):
        return self._features.__setitem__(i, Feature(x))

    def __delitem__(self, i):
        return self._features.__delitem__(i)

    def __len__(self):
        return self._features.__len__()

    def insert(self, i, x):
        return self._features.insert(i, Feature(x))
<|endoftext|>


<commit_before>#!/usr/bin/python3

# Bulls and cows game
# Author: Petr Gabrlik
# Email: petrgabrlik@email.cz
# Date: 18.1.2017

import random

debugmode = False

def main():
    # The game
    dn = 4
    print("""Hi there!
Let's play a bulls and cows game.""")

    while True:
        # Game loop
        rann = rngen(dn)
        if isdebugmode():
            print("(Secret number, for debugging only: " + str(rann) + ")")
        print("I've generated a random {:}-digit number for you.\nGuess that number".format(dn))

        guess = 1
        while True:
            # Guess loop
            usrn = usrinput(dn)
            if rann == usrn:
                print("Correct, you've guessed the right number in {:} {:}!".format(guess, 'guess' if guess == 1 else 'guesses'))
                print("That's {:}.".format(evaluation(guess)))
                name = input("Enter your name: ")
                if len(name) > 7:
                    name = name[:7]
                saveresults(name, guess)
                print("\n**** RESULTS ****")
                print(open("results.txt", "r").read())
                break
            else:
                guess += 1
                bulls, cows = 0, 0
                for num in range(dn):
                    if str(usrn)[num] == str(rann)[num]: bulls += 1
                    elif str(usrn)[num] in str(rann): cows += 1
                print('{:} {:}, {:} {:}'.format(bulls, 'bull' if bulls == 1 else 'bulls', cows, 'cow' if cows == 1 else 'cows'))

        # Play again?
        again = ''
        while not (again == 'Y' or again == 'N'):
            again = input('Do you want to play again? (Y/N) ').upper()
        if again == 'N':
            print('Thank you for playing! Bye bye.')
            break

def rngen(n):
    # Generates n-digit integer, without repeating numbers
    rn = []
    while len(rn) < n:
        newnum = ( str(random.randint(0,9)) )
        if newnum not in rn and not (len(rn) == 0 and int(newnum) == 0): rn.append(newnum)
    return int(''.join(rn))

def usrinput(n):
    # Reads user's 4-digit integer, repeating numbers is not allowed
    while True:
        try:
            usrn = abs(int(input(">>> ")))
            if len(str(usrn)) > n: print('Too long. Please enter {:}-digit integer.'.format(n))
            elif len(str(usrn)) < n: print('Too short. Please enter {:}-digit integer.'.format(n))
            elif True in [True for num in str(usrn) if str(usrn).count(num) > 1] : print('Numbers must not be repeated.')
            else : break
        except:
            print('Invalid number. Please enter {:}-digit integer.'.format(n))
    return usrn

def evaluation(n):
    # Evaluates results
    return {1<=n<4: 'amazing', 4<=n<10: 'really good', 10<=n<20: 'average', 20<=n: 'not so good'}[1]

def saveresults(name, guess):
    # Saves results to file
    try:
        open("results.txt", "r")
    except:
        print("NAME", "GUESSES", sep="\t", file=open("results.txt", "a"))
    print(name, guess, sep="\t", file=open("results.txt", "a"))

def isdebugmode():
    if debugmode:
        return 1
    else:
        return 0

if __name__ == "__main__":
    main()
<commit_msg>Function saveresults rewrited<commit_after>#!/usr/bin/python3

# Bulls and cows game
# Author: Petr Gabrlik
# Email: petrgabrlik@email.cz
# Date: 18.1.2017

import random
import os.path

debugmode = False

def main():
    # The game
    dn = 4
    print("""Hi there!
Let's play a bulls and cows game.""")

    while True:
        # Game loop
        rann = rngen(dn)
        if isdebugmode():
            print("(Secret number, for debugging only: " + str(rann) + ")")
        print("I've generated a random {:}-digit number for you.\nGuess that number".format(dn))

        guess = 1
        while True:
            # Guess loop
            usrn = usrinput(dn)
            if rann == usrn:
                print("Correct, you've guessed the right number in {:} {:}!".format(guess, 'guess' if guess == 1 else 'guesses'))
                print("That's {:}.".format(evaluation(guess)))
                name = input("Enter your name: ")
                if len(name) > 7:
                    name = name[:7]
                saveresults(name, guess)
                print("\n**** RESULTS ****")
                print(open("results.txt", "r").read())
                break
            else:
                guess += 1
                bulls, cows = 0, 0
                for num in range(dn):
                    if str(usrn)[num] == str(rann)[num]: bulls += 1
                    elif str(usrn)[num] in str(rann): cows += 1
                print('{:} {:}, {:} {:}'.format(bulls, 'bull' if bulls == 1 else 'bulls', cows, 'cow' if cows == 1 else 'cows'))

        # Play again?
        again = ''
        while not (again == 'Y' or again == 'N'):
            again = input('Do you want to play again? (Y/N) ').upper()
        if again == 'N':
            print('Thank you for playing! Bye bye.')
            break

def rngen(n):
    # Generates n-digit integer, without repeating numbers
    rn = []
    while len(rn) < n:
        newnum = ( str(random.randint(0,9)) )
        if newnum not in rn and not (len(rn) == 0 and int(newnum) == 0): rn.append(newnum)
    return int(''.join(rn))

def usrinput(n):
    # Reads user's 4-digit integer, repeating numbers is not allowed
    while True:
        try:
            usrn = abs(int(input(">>> ")))
            if len(str(usrn)) > n: print('Too long. Please enter {:}-digit integer.'.format(n))
            elif len(str(usrn)) < n: print('Too short. Please enter {:}-digit integer.'.format(n))
            elif True in [True for num in str(usrn) if str(usrn).count(num) > 1] : print('Numbers must not be repeated.')
            else : break
        except:
            print('Invalid number. Please enter {:}-digit integer.'.format(n))
    return usrn

def evaluation(n):
    # Evaluates results
    return {1<=n<4: 'amazing', 4<=n<10: 'really good', 10<=n<20: 'average', 20<=n: 'not so good'}[1]

def saveresults(name, guess):
    # Saves results to file
    if not os.path.exists("results.txt"):
        resultsfile = open("results.txt", "a")
        print("NAME", "GUESSES", sep="\t", file=resultsfile)
    else:
        resultsfile = open("results.txt", "a")
    print(name, guess, sep="\t", file=resultsfile)
    resultsfile.close()

def isdebugmode():
    # Checkes if debugmode flag is set
    if debugmode:
        return 1
    else:
        return 0

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import numpy
import six

from chainer import cuda
from chainer import function
from chainer.utils import type_check

class Accuracy(function.Function):

    def __init__(self, ignore_label=None):
        self.ignore_label = ignore_label

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 2)
        x_type, t_type = in_types

        type_check.expect(
            x_type.dtype.kind == 'f',
            t_type.dtype == numpy.int32
        )

        t_ndim = t_type.ndim.eval()
        type_check.expect(
            x_type.ndim >= t_type.ndim,
            x_type.shape[0] == t_type.shape[0],
            x_type.shape[2: t_ndim + 1] == t_type.shape[1:]
        )
        for i in six.moves.range(t_ndim + 1, x_type.ndim.eval()):
            type_check.expect(x_type.shape[i] == 1)

    def forward(self, inputs):
        xp = cuda.get_array_module(*inputs)
        y, t = inputs

        if self.ignore_label is not None:
            mask = (t == self.ignore_label)
            ignore_cnt = mask.sum()

            # will always be true when the true label is ignore_label
            # TODO(henry0312)
            #   If cupy.where returns indexes, we could make the code better.
            #   Also, we would need Advanced Indexing.
            pred = xp.where(mask, self.ignore_label,
                            y.argmax(axis=1).reshape(t.shape))
            count = (pred == t).sum() - ignore_cnt
            total = t.size - ignore_cnt

            if total == 0:
                return xp.asarray(0.0, dtype=y.dtype),
            else:
                return xp.asarray(float(count) / total, dtype=y.dtype),
        else:
            pred = y.argmax(axis=1).reshape(t.shape)
            return xp.asarray((pred == t).mean(dtype=y.dtype)),

def accuracy(y, t, ignore_label=None):
    """Computes multiclass classification accuracy of the minibatch.

    Args:
        y (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Array whose (i, j, k, ...)-th element indicates the score of
            the class j at the (i, k, ...)-th sample.
            The prediction label :math:`\\hat t` is calculated by the formula
            :math:`\\hat t_(i, k, ...) = argmax_j y(i, j, k, ...)`.
        t (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray` of :class:`numpy.int32`):
            Array of ground truth labels.
        ignore_label (int or None): Skip calculating accuracy
            if the true label is ``ignore_label``.

    Returns:
        Variable: A variable holding a scalar array of the accuracy.

    .. note:: This function is non-differentiable.

    .. admonition:: Example

        We show the most common case, when``y`` is two dimensinal array.

        >>> y = np.array([[0.1, 0.7, 0.2], # prediction label is 1
        ...               [8.0, 1.0, 2.0], # prediction label is 0
        ...               [-8.0, 1.0, 2.0], # prediction label is 2
        ...               [-8.0, -1.0, -2.0]]) # prediction label is 1
        >>> t = np.array([1, 0, 2, 1], 'i')
        >>> F.accuracy(y, t).data \
# 100% accuracy because all samples are correct
        array(1.0)
        >>> t = np.array([1, 0, 0, 0], 'i')
        >>> F.accuracy(y, t).data \
# 50% accuracy because 1st and 2nd samples are correct.
        array(0.5)
        >>> F.accuracy(y, t, ignore_label=0).data \
# 100% accuracy because of ignoring the 2nd, 3rd and 4th samples.
        array(1.0)

    """
    return Accuracy(ignore_label=ignore_label)(y, t)
<commit_msg>fix type<commit_after>import numpy
import six

from chainer import cuda
from chainer import function
from chainer.utils import type_check

class Accuracy(function.Function):

    def __init__(self, ignore_label=None):
        self.ignore_label = ignore_label

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() == 2)
        x_type, t_type = in_types

        type_check.expect(
            x_type.dtype.kind == 'f',
            t_type.dtype == numpy.int32
        )

        t_ndim = t_type.ndim.eval()
        type_check.expect(
            x_type.ndim >= t_type.ndim,
            x_type.shape[0] == t_type.shape[0],
            x_type.shape[2: t_ndim + 1] == t_type.shape[1:]
        )
        for i in six.moves.range(t_ndim + 1, x_type.ndim.eval()):
            type_check.expect(x_type.shape[i] == 1)

    def forward(self, inputs):
        xp = cuda.get_array_module(*inputs)
        y, t = inputs

        if self.ignore_label is not None:
            mask = (t == self.ignore_label)
            ignore_cnt = mask.sum()

            # will always be true when the true label is ignore_label
            # TODO(henry0312)
            #   If cupy.where returns indexes, we could make the code better.
            #   Also, we would need Advanced Indexing.
            pred = xp.where(mask, self.ignore_label,
                            y.argmax(axis=1).reshape(t.shape))
            count = (pred == t).sum() - ignore_cnt
            total = t.size - ignore_cnt

            if total == 0:
                return xp.asarray(0.0, dtype=y.dtype),
            else:
                return xp.asarray(float(count) / total, dtype=y.dtype),
        else:
            pred = y.argmax(axis=1).reshape(t.shape)
            return xp.asarray((pred == t).mean(dtype=y.dtype)),

def accuracy(y, t, ignore_label=None):
    """Computes multiclass classification accuracy of the minibatch.

    Args:
        y (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray`):
            Array whose (i, j, k, ...)-th element indicates the score of
            the class j at the (i, k, ...)-th sample.
            The prediction label :math:`\\hat t` is calculated by the formula
            :math:`\\hat t_(i, k, ...) = argmax_j y(i, j, k, ...)`.
        t (:class:`~chainer.Variable` or :class:`numpy.ndarray` or \
        :class:`cupy.ndarray` of :class:`numpy.int32`):
            Array of ground truth labels.
        ignore_label (int or None): Skip calculating accuracy
            if the true label is ``ignore_label``.

    Returns:
        Variable: A variable holding a scalar array of the accuracy.

    .. note:: This function is non-differentiable.

    .. admonition:: Example

        We show the most common case, when ``y`` is two dimensinal array.

        >>> y = np.array([[0.1, 0.7, 0.2], # prediction label is 1
        ...               [8.0, 1.0, 2.0], # prediction label is 0
        ...               [-8.0, 1.0, 2.0], # prediction label is 2
        ...               [-8.0, -1.0, -2.0]]) # prediction label is 1
        >>> t = np.array([1, 0, 2, 1], 'i')
        >>> F.accuracy(y, t).data \
# 100% accuracy because all samples are correct
        array(1.0)
        >>> t = np.array([1, 0, 0, 0], 'i')
        >>> F.accuracy(y, t).data \
# 50% accuracy because 1st and 2nd samples are correct.
        array(0.5)
        >>> F.accuracy(y, t, ignore_label=0).data \
# 100% accuracy because of ignoring the 2nd, 3rd and 4th samples.
        array(1.0)

    """
    return Accuracy(ignore_label=ignore_label)(y, t)
<|endoftext|>


<commit_before>from chainer.serializers import npz
from chainer.training import extension
from chainer.training.extensions import snapshot_writers
from chainer.utils import argument

def snapshot_object(target, filename, savefun=npz.save_npz):
    """Returns a trainer extension to take snapshots of a given object.

    This extension serializes the given object and saves it to the output
    directory.

    This extension is called once per epoch by default. To take a
    snapshot at a different interval, a trigger object specifying the
    required interval can be passed along with this extension
    to the `extend()` method of the trainer.

    The default priority is -100, which is lower than that of most
    built-in extensions.

    Args:
        target: Object to serialize.
        filename (str): Name of the file into which the object is serialized.
            It can be a format string, where the trainer object is passed to
            the :meth:`str.format` method. For example,
            ``'snapshot_{.updater.iteration}'`` is converted to
            ``'snapshot_10000'`` at the 10,000th iteration.
        savefun: Function to save the object. It takes two arguments: the
            output file path and the object to serialize.

    Returns:
        Snapshot extension object.

    .. seealso::

        - :meth:`chainer.training.extensions.snapshot`
    """
    return _Snapshot(
        target=target,
        writer=snapshot_writers.SimpleWriter(savefun=savefun),
        filename=filename)

def snapshot(savefun=npz.save_npz,
             filename='snapshot_iter_{.updater.iteration}', **kwargs):
    """snapshot(savefun=npz.save_npz, \
filename='snapshot_iter_{.updater.iteration}', *, target=None, \
condition=None, writer=None)

    Returns a trainer extension to take snapshots of the trainer.

    This extension serializes the trainer object and saves it to the output
    directory. It is used to support resuming the training loop from the saved
    state.

    This extension is called once per epoch by default. To take a
    snapshot at a different interval, a trigger object specifying the
    required interval can be passed along with this extension
    to the `extend()` method of the trainer.

    The default priority is -100, which is lower than that of most
    built-in extensions.

    .. note::
       This extension first writes the serialized object to a temporary file
       and then rename it to the target file name. Thus, if the program stops
       right before the renaming, the temporary file might be left in the
       output directory.

    Args:
        savefun: Function to save the trainer. It takes two arguments: the
            output file path and the trainer object.
            If ``writer`` is specified, this argument must be ``None``.
        filename (str): Name of the file into which the trainer is serialized.
            It can be a format string, where the trainer object is passed to
            the :meth:`str.format` method.
        target: Object to serialize. If it is not specified, it will
            be the trainer object.
        condition: Condition object. It must be a callable object that returns
            boolean without any arguments. If it returns True the snapshot will
            be done. If not it will be skipped. The default is a method that
            always returns True.
        writer: Writer object.
            It must be a callable object.
            See below for the list of built-in writers.
            If ``savefun`` is other than ``None``, this argument must be
            ``None``. In that case,
            :class:`~chainer.training.extensions.snapshot_writers.SimpleWriter`.
            with specified ``savefun`` argument will be used.

    Returns:
        Snapshot extension object.

    This is the list of built-in snapshot writers.

        - :class:`chainer.training.extensions.snapshot_writers.SimpleWriter`
        - :class:`chainer.training.extensions.snapshot_writers.ThreadWriter`
        - :class:`chainer.training.extensions.snapshot_writers.ProcessWriter`
        - :class:`chainer.training.extensions.snapshot_writers.\
ThreadQueueWriter`
        - :class:`chainer.training.extensions.snapshot_writers.\
ProcessQueueWriter`

    .. TODO(niboshi): Fix the following example

    .. admonition:: Example

        The simplest use of ``Snapshot`` extension is just like
        :meth:`chainer.training.extensions.snapshot`.

        >>> from chainer.training import extensions
        >>> trainer.extend(extensions.Snapshot(), trigger=(1, 'epoch'))

        If you want to use another writer, you can explicitly specify it.

        >>> from chainer.training import extensions
        >>> writer = extensions.snapshot_writers.ProcessWriter()
        >>> trainer.extend(extensions.Snapshot(writer=writer), \
trigger=(1, 'epoch'))

        To change the format, such as npz or hdf5, you can pass a saving
        function as ``savefun`` argument of the writer.

        >>> from chainer.training import extensions
        >>> writer = extensions.snapshot_writers.ProcessWriter(
        >>>     savefun=extensions.snapshots.util.save_npz)
        >>> trainer.extend(extensions.Snapshot(writer=writer), \
trigger=(1, 'epoch'))

    .. seealso::

        - :meth:`chainer.training.extensions.snapshot_object`
    """
    target = kwargs.pop('target')
    condition = kwargs.pop('condition')
    writer = kwargs.pop('writer')
    if savefun is not None and writer is not None:
        raise TypeError(
            'savefun and writer argument cannot be specified together.')
    argument.assert_kwargs_empty(kwargs)

    if writer is None:
        writer = snapshot_writers.SimpleWriter(savefun=savefun)

    return _Snapshot(
        target=target, condition=condition, writer=writer, filename=filename)

def _always_true():
    return True

class _Snapshot(extension.Extension):
    """Trainer extension to take snapshots.

    This extension serializes the given object and saves it to the output
    directory.

    This extension is called once per epoch by default. To take a
    snapshot at a different interval, a trigger object specifying the
    required interval can be passed along with this extension
    to the `extend()` method of the trainer.

    The default priority is -100, which is lower than that of most
    built-in extensions.
    """
    trigger = 1, 'epoch'
    priority = -100

    def __init__(
            self, target=None, condition=None, writer=None,
            filename='snapshot_iter_{.updater.iteration}'):
        if condition is None:
            condition = _always_true
        if writer is None:
            writer = snapshot_writers.SimpleWriter()
        self._target = target
        self.filename = filename
        self.condition = condition
        self.writer = writer

    def __call__(self, trainer):
        if self.condition():
            target = trainer if self._target is None else self._target
            serialized_target = npz.serialize(target)
            filename = self.filename
            if callable(filename):
                filename = filename(trainer)
            else:
                filename = filename.format(trainer)
            outdir = trainer.out
            self.writer(filename, outdir, serialized_target)

    def finalize(self):
        if hasattr(self.writer, 'finalize'):
            self.writer.finalize()
<commit_msg>Set savefun=None by default to avoid conflict error with writer argument<commit_after>from chainer.serializers import npz
from chainer.training import extension
from chainer.training.extensions import snapshot_writers
from chainer.utils import argument

def snapshot_object(target, filename, savefun=npz.save_npz):
    """Returns a trainer extension to take snapshots of a given object.

    This extension serializes the given object and saves it to the output
    directory.

    This extension is called once per epoch by default. To take a
    snapshot at a different interval, a trigger object specifying the
    required interval can be passed along with this extension
    to the `extend()` method of the trainer.

    The default priority is -100, which is lower than that of most
    built-in extensions.

    Args:
        target: Object to serialize.
        filename (str): Name of the file into which the object is serialized.
            It can be a format string, where the trainer object is passed to
            the :meth:`str.format` method. For example,
            ``'snapshot_{.updater.iteration}'`` is converted to
            ``'snapshot_10000'`` at the 10,000th iteration.
        savefun: Function to save the object. It takes two arguments: the
            output file path and the object to serialize.

    Returns:
        Snapshot extension object.

    .. seealso::

        - :meth:`chainer.training.extensions.snapshot`
    """
    return _Snapshot(
        target=target,
        writer=snapshot_writers.SimpleWriter(savefun=savefun),
        filename=filename)

def snapshot(savefun=None,
             filename='snapshot_iter_{.updater.iteration}', **kwargs):
    """snapshot(savefun=None, filename='snapshot_iter_{.updater.iteration}', \
*, target=None, condition=None, writer=None)

    Returns a trainer extension to take snapshots of the trainer.

    This extension serializes the trainer object and saves it to the output
    directory. It is used to support resuming the training loop from the saved
    state.

    This extension is called once per epoch by default. To take a
    snapshot at a different interval, a trigger object specifying the
    required interval can be passed along with this extension
    to the `extend()` method of the trainer.

    The default priority is -100, which is lower than that of most
    built-in extensions.

    .. note::
       This extension first writes the serialized object to a temporary file
       and then rename it to the target file name. Thus, if the program stops
       right before the renaming, the temporary file might be left in the
       output directory.

    Args:
        savefun: Function to save the trainer. It takes two arguments: the
            output file path and the trainer object.
            It is :meth:`chainer.serializers.save_npz` by default.
            If ``writer`` is specified, this argument must be ``None``.
        filename (str): Name of the file into which the trainer is serialized.
            It can be a format string, where the trainer object is passed to
            the :meth:`str.format` method.
        target: Object to serialize. If it is not specified, it will
            be the trainer object.
        condition: Condition object. It must be a callable object that returns
            boolean without any arguments. If it returns ``True``, the snapshot
            will be done.
            If not, it will be skipped. The default is a function that always
            returns ``True``.
        writer: Writer object.
            It must be a callable object.
            See below for the list of built-in writers.
            If ``savefun`` is other than ``None``, this argument must be
            ``None``. In that case, a
            :class:`~chainer.training.extensions.snapshot_writers.SimpleWriter`
            object instantiated with specified ``savefun`` argument will be
            used.

    Returns:
        Snapshot extension object.

    This is the list of built-in snapshot writers.

        - :class:`chainer.training.extensions.snapshot_writers.SimpleWriter`
        - :class:`chainer.training.extensions.snapshot_writers.ThreadWriter`
        - :class:`chainer.training.extensions.snapshot_writers.ProcessWriter`
        - :class:`chainer.training.extensions.snapshot_writers.\
ThreadQueueWriter`
        - :class:`chainer.training.extensions.snapshot_writers.\
ProcessQueueWriter`

    .. TODO(niboshi): Fix the following example

    .. admonition:: Example

        The simplest use of ``Snapshot`` extension is just like
        :meth:`chainer.training.extensions.snapshot`.

        >>> from chainer.training import extensions
        >>> trainer.extend(extensions.Snapshot(), trigger=(1, 'epoch'))

        If you want to use another writer, you can explicitly specify it.

        >>> from chainer.training import extensions
        >>> writer = extensions.snapshot_writers.ProcessWriter()
        >>> trainer.extend(extensions.Snapshot(writer=writer), \
trigger=(1, 'epoch'))

        To change the format, such as npz or hdf5, you can pass a saving
        function as ``savefun`` argument of the writer.

        >>> from chainer.training import extensions
        >>> writer = extensions.snapshot_writers.ProcessWriter(
        >>>     savefun=extensions.snapshots.util.save_npz)
        >>> trainer.extend(extensions.Snapshot(writer=writer), \
trigger=(1, 'epoch'))

    .. seealso::

        - :meth:`chainer.training.extensions.snapshot_object`
    """
    target = kwargs.pop('target')
    condition = kwargs.pop('condition')
    writer = kwargs.pop('writer')
    if savefun is not None and writer is not None:
        raise TypeError(
            'savefun and writer argument cannot be specified together.')
    argument.assert_kwargs_empty(kwargs)

    if writer is None:
        if savefun is None:
            savefun = npz.save_npz
        writer = snapshot_writers.SimpleWriter(savefun=savefun)

    return _Snapshot(
        target=target, condition=condition, writer=writer, filename=filename)

def _always_true():
    return True

class _Snapshot(extension.Extension):
    """Trainer extension to take snapshots.

    This extension serializes the given object and saves it to the output
    directory.

    This extension is called once per epoch by default. To take a
    snapshot at a different interval, a trigger object specifying the
    required interval can be passed along with this extension
    to the `extend()` method of the trainer.

    The default priority is -100, which is lower than that of most
    built-in extensions.
    """
    trigger = 1, 'epoch'
    priority = -100

    def __init__(
            self, target=None, condition=None, writer=None,
            filename='snapshot_iter_{.updater.iteration}'):
        if condition is None:
            condition = _always_true
        if writer is None:
            writer = snapshot_writers.SimpleWriter()
        self._target = target
        self.filename = filename
        self.condition = condition
        self.writer = writer

    def __call__(self, trainer):
        if self.condition():
            target = trainer if self._target is None else self._target
            serialized_target = npz.serialize(target)
            filename = self.filename
            if callable(filename):
                filename = filename(trainer)
            else:
                filename = filename.format(trainer)
            outdir = trainer.out
            self.writer(filename, outdir, serialized_target)

    def finalize(self):
        if hasattr(self.writer, 'finalize'):
            self.writer.finalize()
<|endoftext|>