<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import sys
sys.dont_write_bytecode = True

from ruamel import yaml
from doit.task import clean_targets

from sota.utils.shell import call, rglob
from sota.constants import *

DOIT_CONFIG = {
    'verbosity': 2,
    'default_tasks': ['post'],
}

def task_version():
    '''
    write version.py and version.h
    '''
    templates = {}
    for template in rglob('sota/*.template'):
        filename = template[:-len('.template')]
        contents = open(template).read()
        templates[filename] = contents
    def render():
        for filename, contents in templates.items():
            with open(filename, 'w') as f:
                f.write(contents)
    def uptodate():
        for filename, contents in templates.items():
            if os.path.isfile(filename):
                if contents == open(filename).read():
                    continue
            return False
        return True
    return dict(
        actions=[(render,)],
        uptodate=[(uptodate,)],
        targets=list(templates.keys()),
    )

def task_submod():
    '''
    run ensure git submodules are up to date
    '''
    SYMS = [
        '+',
        '-',
    ]
    for submod, sha1hash in SUBS2SHAS.items():
        yield dict(
            name=submod,
            actions=[
                f'git submodule update --init {submod}'
            ],
            uptodate=[all(map(lambda sym: not sha1hash.startswith(sym), SYMS))],
        )

def pre_pylint():
    '''
    run pylint before the build
    '''
    return dict(
        name='pylint',
        task_dep=[
            'submod',
            'version',
        ],
        actions=[
            'pylint -j{JOBS} --rcfile {PREDIR}/pylint.rc {SOTADIR} || true',
        ]
    )

def pre_pytest():
    '''
    run pytest before the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            f'echo "{REPOROOT}"',
            f'{PYTHON} -m pytest -s -vv {PREDIR}',
        ],
    )

def pre_pycov():
    '''
    run pycov before the build
    '''
    return dict(
        name='pycov',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            f'{PYTHON} -m pytest -s -vv --cov={SOTADIR} {PREDIR}',
        ],
    )

def pre_mypy():
    '''
    run mypy before the build
    '''
    def package_wanted(directory):
        ignores = [
            'utils',
            '__pycache__',
        ]
        return directory not in ignores and os.path.isdir('sota/'+directory)
    packages = [d for d in os.listdir('sota') if package_wanted(d)]
    return dict(
        name='mypy',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            'mypy' + ''.join([f' --module sota.{package}' for package in packages]),
        ],
    )

def task_pre():
    '''
    run tasks before the build: pylint, pytest, pycov
    '''
    yield pre_pylint()
    yield pre_pytest()
    #yield pre_pycov()
    yield pre_mypy()

def task_colm():
    '''
    build colm binary for use in build
    '''
    return dict(
        task_dep=[
            f'submod:repos/colm',
        ],
        actions=[
            f'cd {COLMDIR} && autoreconf -f -i',
            f'cd {COLMDIR} && ./configure --prefix={REPOROOT}',
            f'cd {COLMDIR} && make && make install',
        ],
        uptodate=[True],
        targets=[COLM],
        clean=[clean_targets],
    )

def task_ragel():
    '''
    build ragel binary for use in build
    '''
    return dict(
        task_dep=[
            'submod:repos/ragel',
            'colm',
        ],
        actions=[
            f'cd {RAGELDIR} && autoreconf -f -i',
            f'cd {RAGELDIR} && ./configure --prefix={REPOROOT} --with-colm={REPOROOT} --disable-manual',
            f'cd {RAGELDIR} && make && make install',
        ],
        uptodate=[True],
        targets=[RAGEL],
        clean=[clean_targets],
    )

def task_liblexer():
    '''
    build so libary for use as sota's lexer
    '''
    files = [f'{LEXERDIR}/lexer.py'] + rglob(LEXERDIR + '/*.{h,rl,c}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            'ragel',
        ],
        actions=[
            f'cd {LEXERDIR} && LD_LIBRARY_PATH={LIBDIR} make --trace -j {JOBS} RAGEL={RAGEL}',
            f'install -C -D {LEXERDIR}/liblexer.so {LIBDIR}/liblexer.so',
        ],
        uptodate=[True],
        targets=[
            f'{LIBDIR}/liblexer.so',
            f'{LEXERDIR}/lexer.cpp',
            f'{LEXERDIR}/test',
        ],
        clean=[clean_targets],
    )

def task_graph():
    '''
    create Graphviz PNGs
    '''
    for machine in ('body', 'string', 'commenter'):
        yield dict(
            name=machine,
            file_dep=[
                f'{LEXERDIR}/lexer.rl',
            ],
            task_dep=[
                'liblexer',
            ],
            actions=[
                f'cd {LEXERDIR} && {RAGEL} lexer.rl -V -M {machine} -o {machine}.dot',
                f'cd {LEXERDIR} && dot -Tpng {machine}.dot -o {machine}.png',
            ],
            targets=[
                f'{LEXERDIR}/{machine}.dot',
                f'{LEXERDIR}/{machine}.png',
            ],
            clean=[clean_targets],
        )

def task_libcli():
    '''
    build so libary for use as sota's commandline interface
    '''
    files = [DODO, VERSION_H] + rglob(CLIDIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            'submod:repos/docopt',
        ],
        actions=[
            f'cd {CLIDIR} && make --trace -j {JOBS}',
            f'install -C -D {CLIDIR}/libcli.so {LIBDIR}/libcli.so',
        ],
        uptodate=[True],
        targets=[
            f'{CLIDIR}/test',
            f'{LIBDIR}/libcli.so',
        ],
        clean=[clean_targets],
    )

def task_libsha256():
    '''
    build so library for use as sota's sha256 hashing functionality
    '''
    files = [DODO] + rglob(SHA256DIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
        ],
        actions=[
            f'cd {SHA256DIR} && make --trace -j {JOBS}',
            f'install -C -D {SHA256DIR}/libsha256.so {LIBDIR}/libsha256.so',
        ],
        uptodate=[True],
        targets=[
            f'{SHA256DIR}/test',
            f'{LIBDIR}/libsha256.so',
        ],
        clean=[clean_targets],
    )

def task_sota():
    '''
    build sota binary using rpython machinery
    '''
    return dict(
        file_dep=[
            DODO,
            TARGET,
            f'{LIBDIR}/libcli.so',
            f'{LIBDIR}/liblexer.so',
            f'{LIBDIR}/libsha256.so',
        ] + rglob(f'{SOTADIR}/*.py'),
        task_dep=[
            'version',
            'pre',
            'libcli',
            'liblexer',
            'libsha256',
        ],
        actions=[
            f'mkdir -p {BINDIR}',
            f'{PYTHON} -B {RPYTHON} --no-pdb --output {BINDIR}/{BINARY} {TARGET}',
        ],
        uptodate=[True],
        targets=[f'{BINDIR}/{BINARY}'],
        clean=[clean_targets],
    )

def post_pytest():
    '''
    run pytest after the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'sota',
        ],
        actions=[
            f'{PYTHON} -m pytest -s -vv {POSTDIR}',
        ],
    )

def task_post():
    '''
    run tasks after the build: pytest
    '''
    yield post_pytest()

def task_rmcache():
    '''
    recursively delete python cache files
    '''
    pycache = dict(py2='*.pyc', py3='__pycache__')
    exec_rm = '-exec rm -r "{}" \;'
    for py, cache in pycache.items():
        for dir_ in ('sota', 'tests'):
            yield dict(
                name=py + '-' + dir_,
                actions=[
                    f'find {dir_} -depth -name {cache} -type d {exec_rm}',
                ],
            )

def task_tidy():
    '''
    clean submods and sota/lang repo
    '''
    yield dict(
        name='sota/lang',
        actions=['git clean -xfd'],
    )
    for submod in SUBS2SHAS.keys():
        yield dict(
            name=submod,
            actions=[
                f'cd {submod} && git reset --hard HEAD && git clean -xfd'
            ],
        )
<commit_msg>added missing f-string<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import sys
sys.dont_write_bytecode = True

from ruamel import yaml
from doit.task import clean_targets

from sota.utils.shell import call, rglob
from sota.constants import *

DOIT_CONFIG = {
    'verbosity': 2,
    'default_tasks': ['post'],
}

def task_version():
    '''
    write version.py and version.h
    '''
    templates = {}
    for template in rglob('sota/*.template'):
        filename = template[:-len('.template')]
        contents = open(template).read()
        templates[filename] = contents
    def render():
        for filename, contents in templates.items():
            with open(filename, 'w') as f:
                f.write(contents)
    def uptodate():
        for filename, contents in templates.items():
            if os.path.isfile(filename):
                if contents == open(filename).read():
                    continue
            return False
        return True
    return dict(
        actions=[(render,)],
        uptodate=[(uptodate,)],
        targets=list(templates.keys()),
    )

def task_submod():
    '''
    run ensure git submodules are up to date
    '''
    SYMS = [
        '+',
        '-',
    ]
    for submod, sha1hash in SUBS2SHAS.items():
        yield dict(
            name=submod,
            actions=[
                f'git submodule update --init {submod}'
            ],
            uptodate=[all(map(lambda sym: not sha1hash.startswith(sym), SYMS))],
        )

def pre_pylint():
    '''
    run pylint before the build
    '''
    return dict(
        name='pylint',
        task_dep=[
            'submod',
            'version',
        ],
        actions=[
            f'pylint -j{JOBS} --rcfile {PREDIR}/pylint.rc {SOTADIR} || true',
        ]
    )

def pre_pytest():
    '''
    run pytest before the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            f'echo "{REPOROOT}"',
            f'{PYTHON} -m pytest -s -vv {PREDIR}',
        ],
    )

def pre_pycov():
    '''
    run pycov before the build
    '''
    return dict(
        name='pycov',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            f'{PYTHON} -m pytest -s -vv --cov={SOTADIR} {PREDIR}',
        ],
    )

def pre_mypy():
    '''
    run mypy before the build
    '''
    def package_wanted(directory):
        ignores = [
            'utils',
            '__pycache__',
        ]
        return directory not in ignores and os.path.isdir('sota/'+directory)
    packages = [d for d in os.listdir('sota') if package_wanted(d)]
    return dict(
        name='mypy',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            'mypy' + ''.join([f' --module sota.{package}' for package in packages]),
        ],
    )

def task_pre():
    '''
    run tasks before the build: pylint, pytest, pycov
    '''
    yield pre_pylint()
    yield pre_pytest()
    #yield pre_pycov()
    yield pre_mypy()

def task_colm():
    '''
    build colm binary for use in build
    '''
    return dict(
        task_dep=[
            f'submod:repos/colm',
        ],
        actions=[
            f'cd {COLMDIR} && autoreconf -f -i',
            f'cd {COLMDIR} && ./configure --prefix={REPOROOT}',
            f'cd {COLMDIR} && make && make install',
        ],
        uptodate=[True],
        targets=[COLM],
        clean=[clean_targets],
    )

def task_ragel():
    '''
    build ragel binary for use in build
    '''
    return dict(
        task_dep=[
            'submod:repos/ragel',
            'colm',
        ],
        actions=[
            f'cd {RAGELDIR} && autoreconf -f -i',
            f'cd {RAGELDIR} && ./configure --prefix={REPOROOT} --with-colm={REPOROOT} --disable-manual',
            f'cd {RAGELDIR} && make && make install',
        ],
        uptodate=[True],
        targets=[RAGEL],
        clean=[clean_targets],
    )

def task_liblexer():
    '''
    build so libary for use as sota's lexer
    '''
    files = [f'{LEXERDIR}/lexer.py'] + rglob(LEXERDIR + '/*.{h,rl,c}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            'ragel',
        ],
        actions=[
            f'cd {LEXERDIR} && LD_LIBRARY_PATH={LIBDIR} make --trace -j {JOBS} RAGEL={RAGEL}',
            f'install -C -D {LEXERDIR}/liblexer.so {LIBDIR}/liblexer.so',
        ],
        uptodate=[True],
        targets=[
            f'{LIBDIR}/liblexer.so',
            f'{LEXERDIR}/lexer.cpp',
            f'{LEXERDIR}/test',
        ],
        clean=[clean_targets],
    )

def task_graph():
    '''
    create Graphviz PNGs
    '''
    for machine in ('body', 'string', 'commenter'):
        yield dict(
            name=machine,
            file_dep=[
                f'{LEXERDIR}/lexer.rl',
            ],
            task_dep=[
                'liblexer',
            ],
            actions=[
                f'cd {LEXERDIR} && {RAGEL} lexer.rl -V -M {machine} -o {machine}.dot',
                f'cd {LEXERDIR} && dot -Tpng {machine}.dot -o {machine}.png',
            ],
            targets=[
                f'{LEXERDIR}/{machine}.dot',
                f'{LEXERDIR}/{machine}.png',
            ],
            clean=[clean_targets],
        )

def task_libcli():
    '''
    build so libary for use as sota's commandline interface
    '''
    files = [DODO, VERSION_H] + rglob(CLIDIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            'submod:repos/docopt',
        ],
        actions=[
            f'cd {CLIDIR} && make --trace -j {JOBS}',
            f'install -C -D {CLIDIR}/libcli.so {LIBDIR}/libcli.so',
        ],
        uptodate=[True],
        targets=[
            f'{CLIDIR}/test',
            f'{LIBDIR}/libcli.so',
        ],
        clean=[clean_targets],
    )

def task_libsha256():
    '''
    build so library for use as sota's sha256 hashing functionality
    '''
    files = [DODO] + rglob(SHA256DIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
        ],
        actions=[
            f'cd {SHA256DIR} && make --trace -j {JOBS}',
            f'install -C -D {SHA256DIR}/libsha256.so {LIBDIR}/libsha256.so',
        ],
        uptodate=[True],
        targets=[
            f'{SHA256DIR}/test',
            f'{LIBDIR}/libsha256.so',
        ],
        clean=[clean_targets],
    )

def task_sota():
    '''
    build sota binary using rpython machinery
    '''
    return dict(
        file_dep=[
            DODO,
            TARGET,
            f'{LIBDIR}/libcli.so',
            f'{LIBDIR}/liblexer.so',
            f'{LIBDIR}/libsha256.so',
        ] + rglob(f'{SOTADIR}/*.py'),
        task_dep=[
            'version',
            'pre',
            'libcli',
            'liblexer',
            'libsha256',
        ],
        actions=[
            f'mkdir -p {BINDIR}',
            f'{PYTHON} -B {RPYTHON} --no-pdb --output {BINDIR}/{BINARY} {TARGET}',
        ],
        uptodate=[True],
        targets=[f'{BINDIR}/{BINARY}'],
        clean=[clean_targets],
    )

def post_pytest():
    '''
    run pytest after the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'sota',
        ],
        actions=[
            f'{PYTHON} -m pytest -s -vv {POSTDIR}',
        ],
    )

def task_post():
    '''
    run tasks after the build: pytest
    '''
    yield post_pytest()

def task_rmcache():
    '''
    recursively delete python cache files
    '''
    pycache = dict(py2='*.pyc', py3='__pycache__')
    exec_rm = '-exec rm -r "{}" \;'
    for py, cache in pycache.items():
        for dir_ in ('sota', 'tests'):
            yield dict(
                name=py + '-' + dir_,
                actions=[
                    f'find {dir_} -depth -name {cache} -type d {exec_rm}',
                ],
            )

def task_tidy():
    '''
    clean submods and sota/lang repo
    '''
    yield dict(
        name='sota/lang',
        actions=['git clean -xfd'],
    )
    for submod in SUBS2SHAS.keys():
        yield dict(
            name=submod,
            actions=[
                f'cd {submod} && git reset --hard HEAD && git clean -xfd'
            ],
        )
<|endoftext|>


<commit_before>"""PyDoIt file for automating tasks."""

import glob
import itertools
import os
from os.path import dirname

from doit import get_var
#from doit.tools import Interactive
from doit.tools import create_folder
#from doit.tools import result_dep

import dwi.util

"""
Backends:
dbm: (default) It uses python dbm module.
json: Plain text using a json structure, it is slow but good for debugging.
sqlite3: (experimental) very slow implementation, support concurrent access.
"""

DOIT_CONFIG = {
    'backend': 'sqlite3',
    'default_tasks': [],
    'verbosity': 2,
    }

DWILIB = '~/src/dwilib/tools'
PMAP = DWILIB+'/pmap.py'
ANON = DWILIB+'/anonymize_dicom.py'
FIND_ROI = DWILIB+'/find_roi.py'
COMPARE_MASKS = DWILIB+'/compare_masks.py'
SELECT_VOXELS = DWILIB+'/select_voxels.py'
CALC_AUC = DWILIB+'/draw_roc.py'
CORRELATION = DWILIB+'/correlation.py'

MODELS = 'Si SiN Mono MonoN Kurt KurtN Stretched StretchedN '\
        'Biexp BiexpN'.split()

SAMPLELIST = get_var('samplelist', 'all') # Sample list (train, test, etc)
SAMPLELIST_FILE = 'samples_%s.txt' % SAMPLELIST
SAMPLES = dwi.util.read_sample_list(SAMPLELIST_FILE)
SUBWINDOWS = dwi.util.read_subwindows('subwindows.txt')

FIND_ROI_PARAMS = [
        #[5, 7, 10], # ROI side min (3 was not good)
        [7, 8, 9, 10, 11], # ROI side min (3 was not good)
        #[10, 15], # ROI side max
        [7, 8, 9, 10, 11], # ROI side max
        #range(500, 6000, 500), # Number of ROIs
        #range(500, 3000, 500) + [250, 750], # Number of ROIs
        range(250, 3000, 250), # Number of ROIs
]

# Common functions

def subwindow_to_str(subwindow):
    return ' '.join(map(str, subwindow))

def fit_cmd(model, subwindow, infiles, outfile):
    d = dict(prg=PMAP,
            m=model,
            sw=subwindow_to_str(subwindow),
            i=' '.join(infiles),
            o=outfile,
            )
    s = '{prg} -m {m} -s {sw} -d {i} -o {o}'.format(**d)
    return s

# Tasks

#def task_anonymize():
#    """Anonymize imaging data."""
#    files = glob.glob('dicoms/*/DICOMDIR') + glob.glob('dicoms/*/DICOM/*')
#    files.sort()
#    for f in files:
#        cmd = '{prg} -v -i {f}'.format(prg=ANON, f=f)
#        yield {
#                'name': f,
#                'actions': [cmd],
#                'file_dep': [f],
#                }

def task_fit():
    """Fit models to imaging data."""
    for case, scan in SUBWINDOWS.keys():
        subwindow = SUBWINDOWS[(case, scan)]
        d = dict(c=case, s=scan)
        infiles = sorted(glob.glob('dicoms/{c}_*_hB_{s}*/DICOM/*'.format(**d)))
        if len(infiles) == 0:
            continue
        for model in MODELS:
            d['m'] = model
            outfile = 'pmaps/pmap_{c}_{s}_{m}.txt'.format(**d)
            cmd = fit_cmd(model, subwindow, infiles, outfile)
            yield {
                    'name': '{c}_{s}_{m}'.format(**d),
                    'actions': [(create_folder, [dirname(outfile)]),
                            cmd],
                    'file_dep': infiles,
                    'targets': [outfile],
                    'clean': True,
                    }

def get_task_find_roi(case, scan, algparams):
    d = dict(prg=FIND_ROI, sl=SAMPLELIST, slf=SAMPLELIST_FILE, c=case, s=scan,
            algparams=' '.join(algparams), algparams_='_'.join(algparams))
    maskpath = 'masks_auto_{sl}_{algparams_}/{c}_{s}_auto.mask'.format(**d)
    figpath = 'find_roi_images_{sl}/{algparams_}/{c}_{s}.png'.format(**d)
    d.update(mp=maskpath, fp=figpath)
    file_deps = [SAMPLELIST_FILE]
    file_deps += glob.glob('masks_prostate/{c}_*_{s}_*/*'.format(**d))
    file_deps += glob.glob('masks_rois/{c}_*_{s}_*/*'.format(**d))
    cmd = '{prg} --samplelist {slf} --cases {c} --scans {s} '\
            '--algparams {algparams} --outmask {mp} --outfig {fp}'.format(**d)
    return {
            'name': '{c}_{s}_{algparams_}'.format(**d),
            'actions': [(create_folder, [dirname(maskpath)]),
                        (create_folder, [dirname(figpath)]),
                    cmd],
            'file_dep': file_deps,
            'targets': [maskpath, figpath],
            'clean': True,
            }

def task_find_roi():
    """Find a cancer ROI automatically."""
    for sample in SAMPLES:
        case = sample['case']
        for scan in sample['scans']:
            for algparams in itertools.product(*FIND_ROI_PARAMS):
                if algparams[0] > algparams[1]:
                    continue
                yield get_task_find_roi(case, scan, map(str, algparams))

## Deprecated.
#def task_compare_masks():
#    """Compare ROI masks."""
#    for case, scan in SUBWINDOWS.keys():
#        subwindow = SUBWINDOWS[(case, scan)]
#        d = dict(prg=COMPARE_MASKS,
#                c=case,
#                s=scan,
#                w=subwindow_to_str(subwindow),
#                )
#        #file1 = 'masks/{c}_{s}_ca.mask'.format(**d)
#        file1 = glob.glob('masks/{c}_{s}_1_*.mask'.format(**d))[0]
#        file2 = 'masks_auto/{c}_{s}_auto.mask'.format(**d)
#        outfile = 'roi_comparison/{c}_{s}.txt'.format(**d)
#        d['f1'] = file1
#        d['f2'] = file2
#        d['o'] = outfile
#        cmd = '{prg} -s {w} {f1} {f2} > {o}'.format(**d)
#        if not os.path.exists(file1):
#            print 'Missing mask file %s' % file1
#            continue
#        yield {
#                'name': '{c}_{s}'.format(**d),
#                'actions': [(create_folder, [dirname(outfile)]),
#                        cmd],
#                'file_dep': [file1, file2],
#                'targets': [outfile],
#                'clean': True,
#                }

def get_task_select_roi_dicom_mask(case, scan, model, param, masktype,
        subwindow=None):
    """Select ROIs from the pmap DICOMs based on masks."""
    d = dict(sl=SAMPLELIST, c=case, s=scan, m=model, p=param, mt=masktype,
            sw=subwindow)
    maskpath = 'masks_rois/{c}_*_{s}_D_{mt}'.format(**d)
    outpath = 'rois_{mt}_{sl}/{c}_x_x_{s}_{m}_{p}_{mt}.txt'.format(**d)
    s = 'results_{m}_combinedDICOM/{c}_*_{s}/{c}_*_{s}_{p}'.format(**d)
    inpath = glob.glob(s)[0]
    args = [SELECT_VOXELS]
    #args += ['-v']
    if subwindow:
        args += ['-s %s' % subwindow_to_str(subwindow)]
    args += ['-m %s' % maskpath]
    args += ['-i "%s"' % inpath]
    args += ['-o "%s"' % outpath]
    cmd = ' '.join(args)
    return {
            'name': '{c}_{s}'.format(**d),
            'actions': [(create_folder, [dirname(outpath)]),
                    cmd],
            #'file_dep': [maskpath],
            'targets': [outpath],
            'clean': True,
            }

def get_task_select_roi(case, scan, model, param, masktype, algparams=[],
        subwindow=None):
    """Select ROIs from the pmap DICOMs based on masks."""
    d = dict(sl=SAMPLELIST, c=case, s=scan, m=model, p=param, mt=masktype,
            algparams_='_'.join(algparams), sw=subwindow)
    if masktype == 'auto':
        name = '{c}_{s}_{algparams_}'.format(**d)
        maskpath = 'masks_{mt}_{sl}_{algparams_}/{c}_{s}_{mt}.mask'.format(**d)
        outpath = 'rois_{mt}_{sl}/{algparams_}/{c}_x_x_{s}_{m}_{p}_{mt}.txt'.format(**d)
    else:
        name = '{c}_{s}'.format(**d)
        maskpath = 'masks_rois/{c}_*_{s}_D_{mt}'.format(**d)
        outpath = 'rois_{mt}/{c}_x_x_{s}_{m}_{p}_{mt}.txt'.format(**d)
    s = 'results_{m}_combinedDICOM/{c}_*_{s}/{c}_*_{s}_{p}'.format(**d)
    inpath = glob.glob(s)[0]
    args = [SELECT_VOXELS]
    #args += ['-v']
    if subwindow:
        args += ['-s %s' % subwindow_to_str(subwindow)]
    args += ['-m %s' % maskpath]
    args += ['-i "%s"' % inpath]
    args += ['-o "%s"' % outpath]
    cmd = ' '.join(args)
    return {
            #'name': '{c}_{s}_{algparams_}'.format(**d),
            'name': name,
            'actions': [(create_folder, [dirname(outpath)]),
                    cmd],
            'file_dep': [maskpath],
            'targets': [outpath],
            'clean': True,
            }

def task_select_roi_cancer():
    """Select cancer ROIs from the pmap DICOMs."""
    for sample in SAMPLES:
        for scan in sample['scans']:
            case = sample['case']
            masktype = 'CA'
            yield get_task_select_roi_dicom_mask(case, scan, 'Mono', 'ADCm',
                    masktype)

def task_select_roi_auto():
    """Select automatic ROIs from the pmap DICOMs."""
    for sample in SAMPLES:
        for scan in sample['scans']:
            for algparams in itertools.product(*FIND_ROI_PARAMS):
                if algparams[0] > algparams[1]:
                    continue
                case = sample['case']
                masktype = 'auto'
                yield get_task_select_roi(case, scan, 'Mono', 'ADCm', masktype,
                        algparams=map(str, algparams))

def task_evaluate_autoroi():
    """Evaluate auto-ROI prediction ability by ROC AUC and correlation with
    Gleason score."""
    outfile = 'autoroi_evaluation_%s.txt' % SAMPLELIST
    d = dict(sl=SAMPLELIST, prg_auc=CALC_AUC, prg_corr=CORRELATION, o=outfile)
    cmds = ['echo `date` > {o}'.format(**d)]
    for algparams in itertools.product(*FIND_ROI_PARAMS):
        if algparams[0] > algparams[1]:
            continue
        d['algparams_'] = '_'.join(map(str, algparams))
        d['i'] = 'rois_auto_{sl}/{algparams_}'.format(**d)
        s = 'echo -n {algparams_} >> {o}'
        cmds.append(s.format(**d))
        s = r'echo -n \\t`{prg_auc} -s patients.txt -l score -g 3+3 -m {i}/* -a --autoflip` >> {o}'
        cmds.append(s.format(**d))
        s = r'echo -n \\t`{prg_corr} -s patients.txt -l score -g 3+3 3+4 -a -m {i}/*` >> {o}'
        cmds.append(s.format(**d))
        s = r'echo \\t`{prg_corr} -s patients.txt -l score -a -m {i}/*` >> {o}'
        cmds.append(s.format(**d))
    return {
            'actions': cmds,
            'task_dep': ['select_roi_auto'],
            'targets': [outfile],
            'clean': True,
            }
<commit_msg>Masks dir structure<commit_after>"""PyDoIt file for automating tasks."""

import glob
import itertools
import os
from os.path import dirname

from doit import get_var
#from doit.tools import Interactive
from doit.tools import create_folder
#from doit.tools import result_dep

import dwi.util

"""
Backends:
dbm: (default) It uses python dbm module.
json: Plain text using a json structure, it is slow but good for debugging.
sqlite3: (experimental) very slow implementation, support concurrent access.
"""

DOIT_CONFIG = {
    'backend': 'sqlite3',
    'default_tasks': [],
    'verbosity': 2,
    }

DWILIB = '~/src/dwilib/tools'
PMAP = DWILIB+'/pmap.py'
ANON = DWILIB+'/anonymize_dicom.py'
FIND_ROI = DWILIB+'/find_roi.py'
COMPARE_MASKS = DWILIB+'/compare_masks.py'
SELECT_VOXELS = DWILIB+'/select_voxels.py'
CALC_AUC = DWILIB+'/draw_roc.py'
CORRELATION = DWILIB+'/correlation.py'

MODELS = 'Si SiN Mono MonoN Kurt KurtN Stretched StretchedN '\
        'Biexp BiexpN'.split()

SAMPLELIST = get_var('samplelist', 'all') # Sample list (train, test, etc)
SAMPLELIST_FILE = 'samples_%s.txt' % SAMPLELIST
SAMPLES = dwi.util.read_sample_list(SAMPLELIST_FILE)
SUBWINDOWS = dwi.util.read_subwindows('subwindows.txt')

FIND_ROI_PARAMS = [
        #[5, 7, 10], # ROI side min (3 was not good)
        [7, 8, 9, 10, 11], # ROI side min (3 was not good)
        #[10, 15], # ROI side max
        [7, 8, 9, 10, 11], # ROI side max
        #range(500, 6000, 500), # Number of ROIs
        #range(500, 3000, 500) + [250, 750], # Number of ROIs
        range(250, 3000, 250), # Number of ROIs
]

# Common functions

def subwindow_to_str(subwindow):
    return ' '.join(map(str, subwindow))

def fit_cmd(model, subwindow, infiles, outfile):
    d = dict(prg=PMAP,
            m=model,
            sw=subwindow_to_str(subwindow),
            i=' '.join(infiles),
            o=outfile,
            )
    s = '{prg} -m {m} -s {sw} -d {i} -o {o}'.format(**d)
    return s

# Tasks

#def task_anonymize():
#    """Anonymize imaging data."""
#    files = glob.glob('dicoms/*/DICOMDIR') + glob.glob('dicoms/*/DICOM/*')
#    files.sort()
#    for f in files:
#        cmd = '{prg} -v -i {f}'.format(prg=ANON, f=f)
#        yield {
#                'name': f,
#                'actions': [cmd],
#                'file_dep': [f],
#                }

def task_fit():
    """Fit models to imaging data."""
    for case, scan in SUBWINDOWS.keys():
        subwindow = SUBWINDOWS[(case, scan)]
        d = dict(c=case, s=scan)
        infiles = sorted(glob.glob('dicoms/{c}_*_hB_{s}*/DICOM/*'.format(**d)))
        if len(infiles) == 0:
            continue
        for model in MODELS:
            d['m'] = model
            outfile = 'pmaps/pmap_{c}_{s}_{m}.txt'.format(**d)
            cmd = fit_cmd(model, subwindow, infiles, outfile)
            yield {
                    'name': '{c}_{s}_{m}'.format(**d),
                    'actions': [(create_folder, [dirname(outfile)]),
                            cmd],
                    'file_dep': infiles,
                    'targets': [outfile],
                    'clean': True,
                    }

def get_task_find_roi(case, scan, algparams):
    d = dict(prg=FIND_ROI, sl=SAMPLELIST, slf=SAMPLELIST_FILE, c=case, s=scan,
            algparams=' '.join(algparams), algparams_='_'.join(algparams))
    maskpath = 'masks_auto_{sl}/{algparams_}/{c}_{s}_auto.mask'.format(**d)
    figpath = 'find_roi_images_{sl}/{algparams_}/{c}_{s}.png'.format(**d)
    d.update(mp=maskpath, fp=figpath)
    file_deps = [SAMPLELIST_FILE]
    file_deps += glob.glob('masks_prostate/{c}_*_{s}_*/*'.format(**d))
    file_deps += glob.glob('masks_rois/{c}_*_{s}_*/*'.format(**d))
    cmd = '{prg} --samplelist {slf} --cases {c} --scans {s} '\
            '--algparams {algparams} --outmask {mp} --outfig {fp}'.format(**d)
    return {
            'name': '{c}_{s}_{algparams_}'.format(**d),
            'actions': [(create_folder, [dirname(maskpath)]),
                        (create_folder, [dirname(figpath)]),
                    cmd],
            'file_dep': file_deps,
            'targets': [maskpath, figpath],
            'clean': True,
            }

def task_find_roi():
    """Find a cancer ROI automatically."""
    for sample in SAMPLES:
        case = sample['case']
        for scan in sample['scans']:
            for algparams in itertools.product(*FIND_ROI_PARAMS):
                if algparams[0] > algparams[1]:
                    continue
                yield get_task_find_roi(case, scan, map(str, algparams))

## Deprecated.
#def task_compare_masks():
#    """Compare ROI masks."""
#    for case, scan in SUBWINDOWS.keys():
#        subwindow = SUBWINDOWS[(case, scan)]
#        d = dict(prg=COMPARE_MASKS,
#                c=case,
#                s=scan,
#                w=subwindow_to_str(subwindow),
#                )
#        #file1 = 'masks/{c}_{s}_ca.mask'.format(**d)
#        file1 = glob.glob('masks/{c}_{s}_1_*.mask'.format(**d))[0]
#        file2 = 'masks_auto/{c}_{s}_auto.mask'.format(**d)
#        outfile = 'roi_comparison/{c}_{s}.txt'.format(**d)
#        d['f1'] = file1
#        d['f2'] = file2
#        d['o'] = outfile
#        cmd = '{prg} -s {w} {f1} {f2} > {o}'.format(**d)
#        if not os.path.exists(file1):
#            print 'Missing mask file %s' % file1
#            continue
#        yield {
#                'name': '{c}_{s}'.format(**d),
#                'actions': [(create_folder, [dirname(outfile)]),
#                        cmd],
#                'file_dep': [file1, file2],
#                'targets': [outfile],
#                'clean': True,
#                }

def get_task_select_roi_dicom_mask(case, scan, model, param, masktype,
        subwindow=None):
    """Select ROIs from the pmap DICOMs based on masks."""
    d = dict(sl=SAMPLELIST, c=case, s=scan, m=model, p=param, mt=masktype,
            sw=subwindow)
    maskpath = 'masks_rois/{c}_*_{s}_D_{mt}'.format(**d)
    outpath = 'rois_{mt}_{sl}/{c}_x_x_{s}_{m}_{p}_{mt}.txt'.format(**d)
    s = 'results_{m}_combinedDICOM/{c}_*_{s}/{c}_*_{s}_{p}'.format(**d)
    inpath = glob.glob(s)[0]
    args = [SELECT_VOXELS]
    #args += ['-v']
    if subwindow:
        args += ['-s %s' % subwindow_to_str(subwindow)]
    args += ['-m %s' % maskpath]
    args += ['-i "%s"' % inpath]
    args += ['-o "%s"' % outpath]
    cmd = ' '.join(args)
    return {
            'name': '{c}_{s}'.format(**d),
            'actions': [(create_folder, [dirname(outpath)]),
                    cmd],
            #'file_dep': [maskpath],
            'targets': [outpath],
            'clean': True,
            }

def get_task_select_roi(case, scan, model, param, masktype, algparams=[],
        subwindow=None):
    """Select ROIs from the pmap DICOMs based on masks."""
    d = dict(sl=SAMPLELIST, c=case, s=scan, m=model, p=param, mt=masktype,
            algparams_='_'.join(algparams), sw=subwindow)
    if masktype == 'auto':
        name = '{c}_{s}_{algparams_}'.format(**d)
        maskpath = 'masks_{mt}_{sl}/{algparams_}/{c}_{s}_{mt}.mask'.format(**d)
        outpath = 'rois_{mt}_{sl}/{algparams_}/{c}_x_x_{s}_{m}_{p}_{mt}.txt'.format(**d)
    else:
        name = '{c}_{s}'.format(**d)
        maskpath = 'masks_rois/{c}_*_{s}_D_{mt}'.format(**d)
        outpath = 'rois_{mt}/{c}_x_x_{s}_{m}_{p}_{mt}.txt'.format(**d)
    s = 'results_{m}_combinedDICOM/{c}_*_{s}/{c}_*_{s}_{p}'.format(**d)
    inpath = glob.glob(s)[0]
    args = [SELECT_VOXELS]
    #args += ['-v']
    if subwindow:
        args += ['-s %s' % subwindow_to_str(subwindow)]
    args += ['-m %s' % maskpath]
    args += ['-i "%s"' % inpath]
    args += ['-o "%s"' % outpath]
    cmd = ' '.join(args)
    return {
            #'name': '{c}_{s}_{algparams_}'.format(**d),
            'name': name,
            'actions': [(create_folder, [dirname(outpath)]),
                    cmd],
            'file_dep': [maskpath],
            'targets': [outpath],
            'clean': True,
            }

def task_select_roi_cancer():
    """Select cancer ROIs from the pmap DICOMs."""
    for sample in SAMPLES:
        for scan in sample['scans']:
            case = sample['case']
            masktype = 'CA'
            yield get_task_select_roi_dicom_mask(case, scan, 'Mono', 'ADCm',
                    masktype)

def task_select_roi_auto():
    """Select automatic ROIs from the pmap DICOMs."""
    for sample in SAMPLES:
        for scan in sample['scans']:
            for algparams in itertools.product(*FIND_ROI_PARAMS):
                if algparams[0] > algparams[1]:
                    continue
                case = sample['case']
                masktype = 'auto'
                yield get_task_select_roi(case, scan, 'Mono', 'ADCm', masktype,
                        algparams=map(str, algparams))

def task_evaluate_autoroi():
    """Evaluate auto-ROI prediction ability by ROC AUC and correlation with
    Gleason score."""
    outfile = 'autoroi_evaluation_%s.txt' % SAMPLELIST
    d = dict(sl=SAMPLELIST, prg_auc=CALC_AUC, prg_corr=CORRELATION, o=outfile)
    cmds = ['echo `date` > {o}'.format(**d)]
    for algparams in itertools.product(*FIND_ROI_PARAMS):
        if algparams[0] > algparams[1]:
            continue
        d['algparams_'] = '_'.join(map(str, algparams))
        d['i'] = 'rois_auto_{sl}/{algparams_}'.format(**d)
        s = 'echo -n {algparams_} >> {o}'
        cmds.append(s.format(**d))
        s = r'echo -n \\t`{prg_auc} -s patients.txt -l score -g 3+3 -m {i}/* -a --autoflip` >> {o}'
        cmds.append(s.format(**d))
        s = r'echo -n \\t`{prg_corr} -s patients.txt -l score -g 3+3 3+4 -a -m {i}/*` >> {o}'
        cmds.append(s.format(**d))
        s = r'echo \\t`{prg_corr} -s patients.txt -l score -a -m {i}/*` >> {o}'
        cmds.append(s.format(**d))
    return {
            'actions': cmds,
            'task_dep': ['select_roi_auto'],
            'targets': [outfile],
            'clean': True,
            }
<|endoftext|>


<commit_before>from os import path, listdir, makedirs
import re, codecs
import datetime as dt
import calendar as cal
import pandas as pd
import string
import numpy as np

from datamanager.envs import *
from datamanager.load import *
from datamanager.adjust import calc_adj_close, calc_booktomarket
from datamanager.utils import last_month_end
import datamanager.transforms as transf
fields = marketdata_fields()

# paths
mergein_old = MASTER_DATA_PATH
mergein_new = CONVERT_PATH

closepath = path.join(MERGED_PATH, "Close.csv")
divpath = path.join(MERGED_PATH, "Dividend Ex Date.csv")
bookvaluepath = path.join(MERGED_PATH, "Book Value per Share.csv")

def get_all_equities():
    new_all, _, _, _ = get_all_equities_from_data(MERGED_PATH, CONVERT_PATH, 'Close')
    return new_all

def get_current_listed():
    new_all, current, newly_listed, delisted = get_all_equities_from_data(MERGED_PATH, CONVERT_PATH, 'Close')
    return current_set

def convert_data(task):
    '''
    '''
    name = task.name.split(':')[1]
    fp = path.join(DL_PATH, name + '.xlsx')
    new_data = load_intebfa_ts_data(fp)

    # drop all data for current month
    dropix = new_data.index[new_data.index > np.datetime64(last_month_end())]
    new_data.drop(dropix).to_csv(task.targets[0])

def merge_data(task):

    name = task.name.split(':')[1]
    new = load_ts(path.join(CONVERT_PATH, name + '.csv'))
    old = load_ts(path.join(MASTER_DATA_PATH, name + '.csv'))

    merged = empty_dataframe(get_all_equities())
    merged.update(old)
    merged.update(new)

    merged.to_csv(task.targets[0])

def calc_adjusted_close(dependencies, targets):
    all_equities = get_all_equities()

    # Import closing price data
    close = load_field_ts(MERGED_PATH, field = "Close")

    # Import dividend ex date data
    divs = load_field_ts(MERGED_PATH, field = "Dividend Ex Date")
    adj_close = calc_adj_close(close, divs, all_equities)
    adj_close.to_csv(targets[0])

def booktomarket(dependencies, targets):
    # Import closing price data
    close = load_field_ts(MERGED_PATH, field = "Close")

    # Import book value per share data
    bookvalue = load_field_ts(MERGED_PATH, field = "Book Value per Share")
    b2m = calc_booktomarket(close, bookvalue)
    b2m.to_csv(targets[0])

def resample_monthly(task):

    name = task.name.split(':')[1]
    data = load_field_ts(MASTER_DATA_PATH, field = name)

    write = True
    out = pd.DataFrame()

    if name == 'Close':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Adjusted Close':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Open':
        out = transf.resample_monthly(data, how = 'first')
    elif name == 'High':
        out = transf.resample_monthly(data, how = 'max')
    elif name == 'Low':
        out = transf.resample_monthly(data, how = 'min')
    elif name == 'DY':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'EY':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'PE':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Book-to-Market':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Volume':
        out = transf.resample_monthly(data, how = 'sum')
    elif name == 'Total Number Of Shares':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Number Of Trades':
        out = transf.resample_monthly(data, how = 'sum')
    elif name == 'Market Cap':
        out = transf.resample_monthly(data, how = 'last')
    else:
        write = False

    if (write):
        out.to_csv(path.join(MASTER_DATA_PATH, name + '-monthly.csv'))

def monthly_avg_momentum(task):
    # load the daily close
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    # resample to monthly average data
    close_m = transf.resample_monthly(close, how = 'mean')

    # calculate the momentum
    mom = transf.momentum_monthly(close_m, 12, 1)
    mom.to_csv(path.join(MASTER_DATA_PATH, "Monthly-Avg-Momentum.csv"))

def monthly_close_momentum(task):
    # load the daily close
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    # resample to monthly close data
    close_m = transf.resample_monthly(close, how = 'last')

    # calculate the momentum
    mom = transf.momentum_monthly(close_m, 12, 1)
    mom.to_csv(path.join(MASTER_DATA_PATH, "Monthly-Close-Momentum.csv"))

def calc_log_returns(task):
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    logret = transf.log_returns(close)
    logret.to_csv(path.join(MASTER_DATA_PATH, "Log-Returns.csv"))

def calc_pead_momentum(task):
    # load close
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    # load dividend decl date
    announcements = load_field_ts(MASTER_DATA_PATH, field = "Dividend Declaration Date")
    pead = transf.pead_momentum(announcements, close)

    pead.to_csv(path.join(MASTER_DATA_PATH, "Normalized-PEAD-Momentum.csv"))

def swapaxes(dependencies, targets):

    temp = {}
    for d in dependencies:
        field = d.split("\\")[-1].split(".")[0]
        temp[field] = pd.read_csv(d, sep = ',', index_col = 0, parse_dates=True)

    # now create panel
    panel = pd.Panel(temp)
    out = panel.swapaxes(0, 2)

    for ticker in out.items:
        out[ticker].dropna(how='all').to_csv(path.join(CONVERT_PATH, "tickers", ticker + '.csv'), index_label = "Date")

##########################################################################################
# DOIT tasks
##########################################################################################

# 1
def task_convert():
    for f in fields:
        yield {
            'name':f,
            'actions':[convert_data],
            'targets':[path.join(CONVERT_PATH, f+ '.csv')],
            'file_dep':[path.join(DL_PATH, f + '.xlsx')],
        }

# 2
def task_merge():
    for f in fields:
        yield {
            'name':f,
            'actions':[merge_data],
            'targets':[path.join(MERGED_PATH, f + '.csv')],
            'file_dep':[path.join(mergein_new, f + '.csv'), path.join(mergein_old, f + '.csv')]
        }
# 3
def task_adjusted_close():
    return {
        'actions':[calc_adjusted_close],
        'file_dep': [closepath,
                     divpath],
        'targets':[path.join(MERGED_PATH, "Adjusted Close.csv")]
    }

# 5
def task_book2market():
    return {
        'actions':[booktomarket],
        'file_dep': [closepath, bookvaluepath],
        'targets':[path.join(MERGED_PATH, "Book-to-Market.csv")]
    }

# 6
def task_data_per_ticker():
    files = [path.join(CONVERT_PATH, f + '.csv') for f in fields]
    return {
        'actions':[swapaxes],
        'file_dep': files,
        'targets':[path.join(CONVERT_PATH, "tickers")]
    }

def task_resample_monthly():
    expanded = fields + ['Book-to-Market', 'Adjusted Close']
    for f in fields:
        yield {
            'name':f,
            'actions':[resample_monthly],
            'targets':[path.join(MASTER_DATA_PATH, f + '-monthly.csv')],
            'file_dep':[path.join(MASTER_DATA_PATH, f + '.csv')],
        }

def task_monthly_close_momentum():
    return {
        'actions':[monthly_close_momentum],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv')],
    }

def task_monthly_avg_momentum():
    return {
        'actions':[monthly_avg_momentum],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv')],
        'targets':[path.join(MASTER_DATA_PATH, "Monthly-Avg-Momentum.csv")]
    }

def task_log_returns():
    return {
        'actions':[calc_log_returns],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv')],
    }

def task_pead_momentum():
    return {
        'actions':[calc_pead_momentum],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv'), path.join(MASTER_DATA_PATH, 'Dividend Declaration Date.csv')],
        'targets':[path.join(MASTER_DATA_PATH, "Normalized-PEAD-Momentum.csv")]
    }<commit_msg>Added JSE index data<commit_after>from os import path, listdir, makedirs
import re, codecs
import datetime as dt
import calendar as cal
import pandas as pd
import string
import numpy as np

from datamanager.envs import *
from datamanager.load import *
from datamanager.adjust import calc_adj_close, calc_booktomarket
from datamanager.utils import last_month_end
import datamanager.transforms as transf
fields = marketdata_fields()

# paths
mergein_old = MASTER_DATA_PATH
mergein_new = CONVERT_PATH

index_src_path = path.join(DL_PATH, 'Indices.xlsx')
closepath = path.join(MERGED_PATH, "Close.csv")
divpath = path.join(MERGED_PATH, "Dividend Ex Date.csv")
bookvaluepath = path.join(MERGED_PATH, "Book Value per Share.csv")

def get_all_equities():
    new_all, _, _, _ = get_all_equities_from_data(MERGED_PATH, CONVERT_PATH, 'Close')
    return new_all

def get_current_listed():
    new_all, current, newly_listed, delisted = get_all_equities_from_data(MERGED_PATH, CONVERT_PATH, 'Close')
    return current_set

def convert_data(task):
    '''
    '''
    name = task.name.split(':')[1]
    fp = path.join(DL_PATH, name + '.xlsx')
    new_data = load_intebfa_ts_data(fp)

    # drop all data for current month

    dropix = new_data.index[new_data.index.values.astype('datetime64[D]') > np.datetime64(last_month_end())]
    new_data.drop(dropix).to_csv(task.targets[0])

def convert_indices(task):
    new_data = load_inetbfa_ts_data(index_src_path)
    dropix = new_data.index[new_data.index.values.astype('datetime64[D]') > np.datetime64(last_month_end())]
    new_data.drop(dropix).to_csv(task.targets[0])

def merge_data(task):

    name = task.name.split(':')[1]
    new = load_ts(path.join(CONVERT_PATH, name + '.csv'))
    old = load_ts(path.join(MASTER_DATA_PATH, name + '.csv'))

    merged = empty_dataframe(get_all_equities())
    merged.update(old)
    merged.update(new)

    merged.to_csv(task.targets[0])

def calc_adjusted_close(dependencies, targets):
    all_equities = get_all_equities()

    # Import closing price data
    close = load_field_ts(MERGED_PATH, field = "Close")

    # Import dividend ex date data
    divs = load_field_ts(MERGED_PATH, field = "Dividend Ex Date")
    adj_close = calc_adj_close(close, divs, all_equities)
    adj_close.to_csv(targets[0])

def booktomarket(dependencies, targets):
    # Import closing price data
    close = load_field_ts(MERGED_PATH, field = "Close")

    # Import book value per share data
    bookvalue = load_field_ts(MERGED_PATH, field = "Book Value per Share")
    b2m = calc_booktomarket(close, bookvalue)
    b2m.to_csv(targets[0])

def resample_monthly(task):

    name = task.name.split(':')[1]
    data = load_field_ts(MASTER_DATA_PATH, field = name)

    write = True
    out = pd.DataFrame()

    if name == 'Close':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Adjusted Close':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Open':
        out = transf.resample_monthly(data, how = 'first')
    elif name == 'High':
        out = transf.resample_monthly(data, how = 'max')
    elif name == 'Low':
        out = transf.resample_monthly(data, how = 'min')
    elif name == 'DY':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'EY':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'PE':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Book-to-Market':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Volume':
        out = transf.resample_monthly(data, how = 'sum')
    elif name == 'Total Number Of Shares':
        out = transf.resample_monthly(data, how = 'last')
    elif name == 'Number Of Trades':
        out = transf.resample_monthly(data, how = 'sum')
    elif name == 'Market Cap':
        out = transf.resample_monthly(data, how = 'last')
    else:
        write = False

    if (write):
        out.to_csv(path.join(MASTER_DATA_PATH, name + '-monthly.csv'))

def monthly_avg_momentum(task):
    # load the daily close
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    # resample to monthly average data
    close_m = transf.resample_monthly(close, how = 'mean')

    # calculate the momentum
    mom = transf.momentum_monthly(close_m, 12, 1)
    mom.to_csv(path.join(MASTER_DATA_PATH, "Monthly-Avg-Momentum.csv"))

def monthly_close_momentum(task):
    # load the daily close
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    # resample to monthly close data
    close_m = transf.resample_monthly(close, how = 'last')

    # calculate the momentum
    mom = transf.momentum_monthly(close_m, 12, 1)
    mom.to_csv(path.join(MASTER_DATA_PATH, "Monthly-Close-Momentum.csv"))

def calc_log_returns(task):
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    logret = transf.log_returns(close)
    logret.to_csv(path.join(MASTER_DATA_PATH, "Log-Returns.csv"))

def calc_pead_momentum(task):
    # load close
    close = load_field_ts(MASTER_DATA_PATH, field = "Close")

    # load dividend decl date
    announcements = load_field_ts(MASTER_DATA_PATH, field = "Dividend Declaration Date")
    pead = transf.pead_momentum(announcements, close)

    pead.to_csv(path.join(MASTER_DATA_PATH, "Normalized-PEAD-Momentum.csv"))

def swapaxes(dependencies, targets):

    temp = {}
    for d in dependencies:
        field = d.split("\\")[-1].split(".")[0]
        temp[field] = pd.read_csv(d, sep = ',', index_col = 0, parse_dates=True)

    # now create panel
    panel = pd.Panel(temp)
    out = panel.swapaxes(0, 2)

    for ticker in out.items:
        out[ticker].dropna(how='all').to_csv(path.join(CONVERT_PATH, "tickers", ticker + '.csv'), index_label = "Date")

##########################################################################################
# DOIT tasks
##########################################################################################

# 1
def task_convert():
    for f in fields:
        yield {
            'name':f,
            'actions':[convert_data],
            'targets':[path.join(CONVERT_PATH, f+ '.csv')],
            'file_dep':[path.join(DL_PATH, f + '.xlsx')],
        }

def task_convert_index():
     return {
        'actions':[convert_indices],
        'file_dep': [path.join(DL_PATH, 'Indices.xlsx')],
        'targets':[path.join(MERGED_PATH, "Indices.csv")]
    }

# 2
def task_merge():
    for f in fields:
        yield {
            'name':f,
            'actions':[merge_data],
            'targets':[path.join(MERGED_PATH, f + '.csv')],
            'file_dep':[path.join(mergein_new, f + '.csv'), path.join(mergein_old, f + '.csv')]
        }
# 3
def task_adjusted_close():
    return {
        'actions':[calc_adjusted_close],
        'file_dep': [closepath,
                     divpath],
        'targets':[path.join(MERGED_PATH, "Adjusted Close.csv")]
    }

# 5
def task_book2market():
    return {
        'actions':[booktomarket],
        'file_dep': [closepath, bookvaluepath],
        'targets':[path.join(MERGED_PATH, "Book-to-Market.csv")]
    }

# 6
def task_data_per_ticker():
    files = [path.join(CONVERT_PATH, f + '.csv') for f in fields]
    return {
        'actions':[swapaxes],
        'file_dep': files,
        'targets':[path.join(CONVERT_PATH, "tickers")]
    }

def task_resample_monthly():
    expanded = fields + ['Book-to-Market', 'Adjusted Close']
    for f in fields:
        yield {
            'name':f,
            'actions':[resample_monthly],
            'targets':[path.join(MASTER_DATA_PATH, f + '-monthly.csv')],
            'file_dep':[path.join(MASTER_DATA_PATH, f + '.csv')],
        }

def task_monthly_close_momentum():
    return {
        'actions':[monthly_close_momentum],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv')],
    }

def task_monthly_avg_momentum():
    return {
        'actions':[monthly_avg_momentum],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv')],
        'targets':[path.join(MASTER_DATA_PATH, "Monthly-Avg-Momentum.csv")]
    }

def task_log_returns():
    return {
        'actions':[calc_log_returns],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv')],
    }

def task_pead_momentum():
    return {
        'actions':[calc_pead_momentum],
        'file_dep':[path.join(MASTER_DATA_PATH, 'Close.csv'), path.join(MASTER_DATA_PATH, 'Dividend Declaration Date.csv')],
        'targets':[path.join(MASTER_DATA_PATH, "Normalized-PEAD-Momentum.csv")]
    }<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import sys
sys.dont_write_bytecode = True

from ruamel import yaml
from doit.task import clean_targets

from sota.utils.shell import call, rglob
from sota.utils.fmt import fmt, pfmt, dbg
from sota.constants import *

DOIT_CONFIG = {
    'verbosity': 2,
    'default_tasks': ['post'],
}

def task_version():
    '''
    write version.py and version.h
    '''
    templates = {}
    for template in rglob('sota/*.template'):
        filename = template[:-len('.template')]
        contents = fmt(open(template).read())
        templates[filename] = contents
    def render():
        for filename, contents in templates.items():
            with open(filename, 'w') as f:
                f.write(contents)
    def uptodate():
        for filename, contents in templates.items():
            if os.path.isfile(filename):
                if contents == open(filename).read():
                    continue
            return False
        return True
    return dict(
        actions=[(render,)],
        uptodate=[(uptodate,)],
        targets=list(templates.keys()),
    )

def task_submod():
    '''
    run ensure git submodules are up to date
    '''
    SYMS = [
        '+',
        '-',
    ]
    for submod, sha1hash in SUBS2SHAS.items():
        yield dict(
            name=submod,
            actions=[
                fmt('git submodule update --init {submod}')
            ],
            uptodate=[all(map(lambda sym: not sha1hash.startswith(sym), SYMS))],
        )

def pre_pylint():
    '''
    run pylint before the build
    '''
    return dict(
        name='pylint',
        task_dep=[
            'submod',
            'version',
        ],
        actions=[
            fmt('pylint -j{JOBS} --rcfile {PREDIR}/pylint.rc {SOTADIR} || true'),
        ]
    )

def pre_pytest():
    '''
    run pytest before the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            fmt('echo "{REPOROOT}"'),
            fmt('{PYTHON} -m pytest -s -vv {PREDIR}'),
        ],
    )

def pre_pycov():
    '''
    run pycov before the build
    '''
    return dict(
        name='pycov',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            fmt('{PYTHON} -m pytest -s -vv --cov={SOTADIR} {PREDIR}'),
        ],
    )

def pre_mypy():
    '''
    run mypy before the build
    '''
    packages = [d for d in os.listdir('sota') if d != 'utils' and os.path.isdir('sota/'+d)]
    return dict(
        name='mypy',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            'mypy' + ''.join([fmt(' --module sota.{package}') for package in packages]),
        ],
    )

def task_pre():
    '''
    run tasks before the build: pylint, pytest, pycov
    '''
    yield pre_pylint()
    yield pre_pytest()
    #yield pre_pycov()
    yield pre_mypy()

def task_colm():
    '''
    build colm binary for use in build
    '''
    return dict(
        task_dep=[
            fmt('submod:repos/colm'),
        ],
        actions=[
            fmt('cd {COLMDIR} && autoreconf -f -i'),
            fmt('cd {COLMDIR} && ./configure --prefix={REPOROOT}'),
            fmt('cd {COLMDIR} && make && make install'),
        ],
        uptodate=[True],
        targets=[COLM],
        clean=[clean_targets],
    )

def task_ragel():
    '''
    build ragel binary for use in build
    '''
    return dict(
        task_dep=[
            fmt('submod:repos/ragel'),
            'colm',
        ],
        actions=[
            fmt('cd {RAGELDIR} && autoreconf -f -i'),
            fmt('cd {RAGELDIR} && ./configure --prefix={REPOROOT} --with-colm={REPOROOT} --disable-manual'),
            fmt('cd {RAGELDIR} && make && make install'),
        ],
        uptodate=[True],
        targets=[RAGEL],
        clean=[clean_targets],
    )

def task_liblexer():
    '''
    build so libary for use as sota's lexer
    '''
    files = [fmt('{LEXERDIR}/lexer.py')] + rglob(LEXERDIR + '/*.{h,rl,c}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            'ragel',
        ],
        actions=[
            fmt('cd {LEXERDIR} && LD_LIBRARY_PATH={LIBDIR} make -j {JOBS} RAGEL={RAGEL}'),
            fmt('install -C -D {LEXERDIR}/liblexer.so {LIBDIR}/liblexer.so'),
        ],
        uptodate=[True],
        targets=[
            fmt('{LEXERDIR}/test'),
            fmt('{LIBDIR}/liblexer.so'),
        ],
        clean=[clean_targets],
    )

def task_libcli():
    '''
    build so libary for use as sota's commandline interface
    '''
    files = [DODO, VERSION_H] + rglob(CLIDIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            fmt('submod:repos/docopt'),
        ],
        actions=[
            fmt('cd {CLIDIR} && make -j {JOBS}'),
            fmt('install -C -D {CLIDIR}/libcli.so {LIBDIR}/libcli.so'),
        ],
        uptodate=[True],
        targets=[
            fmt('{CLIDIR}/test'),
            fmt('{LIBDIR}/libcli.so'),
        ],
        clean=[clean_targets],
    )

def task_libsha256():
    '''
    build so library for use as sota's sha256 hashing functionality
    '''
    files = [DODO] + rglob(SHA256DIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
        ],
        actions=[
            fmt('cd {SHA256DIR} && make -j {JOBS}'),
            fmt('install -C -D {SHA256DIR}/libsha256.so {LIBDIR}/libsha256.so'),
        ],
        uptodate=[True],
        targets=[
            fmt('{SHA256DIR}/test'),
            fmt('{LIBDIR}/libsha256.so'),
        ],
        clean=[clean_targets],
    )

def task_sota():
    '''
    build sota binary using rpython machinery
    '''
    return dict(
        file_dep=[
            DODO,
            TARGET,
            fmt('{LIBDIR}/libcli.so'),
            fmt('{LIBDIR}/liblexer.so'),
            fmt('{LIBDIR}/libsha256.so'),
        ] + rglob(fmt('{SOTADIR}/*.py')),
        task_dep=[
            'version',
            'pre',
            'libcli',
            'liblexer',
            'libsha256',
        ],
        actions=[
            fmt('mkdir -p {BINDIR}'),
            fmt('{PYTHON} -B {RPYTHON} --no-pdb --output {BINDIR}/{BINARY} {TARGET}'),
        ],
        uptodate=[True],
        targets=[fmt('{BINDIR}/{BINARY}')],
        clean=[clean_targets],
    )

def post_pytest():
    '''
    run pytest after the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'sota',
        ],
        actions=[
            fmt('{PYTHON} -m pytest -s -vv {POSTDIR}'),
        ],
    )

def task_post():
    '''
    run tasks after the build: pytest
    '''
    yield post_pytest()

def task_rmcache():
    '''
    recursively delete python cache files
    '''
    pycache = dict(py2='*.pyc', py3='__pycache__')
    exec_rm = '-exec rm -r "{}" \;'
    for py, cache in pycache.items():
        for dir_ in ('sota', 'tests'):
            yield dict(
                name=py + '-' + dir_,
                actions=[
                    fmt('find {dir_} -depth -name {cache} -type d {exec_rm}'),
                ],
            )

def task_tidy():
    '''
    clean submods and sota/lang repo
    '''
    yield dict(
        name='sota/lang',
        actions=['git clean -xfd'],
    )
    for submod in SUBS2SHAS.keys():
        yield dict(
            name=submod,
            actions=[
                fmt('cd {submod} && git reset --hard HEAD && git clean -xfd')
            ],
        )
<commit_msg>added trace to all makes for lib builds<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import re
import sys
sys.dont_write_bytecode = True

from ruamel import yaml
from doit.task import clean_targets

from sota.utils.shell import call, rglob
from sota.utils.fmt import fmt, pfmt, dbg
from sota.constants import *

DOIT_CONFIG = {
    'verbosity': 2,
    'default_tasks': ['post'],
}

def task_version():
    '''
    write version.py and version.h
    '''
    templates = {}
    for template in rglob('sota/*.template'):
        filename = template[:-len('.template')]
        contents = fmt(open(template).read())
        templates[filename] = contents
    def render():
        for filename, contents in templates.items():
            with open(filename, 'w') as f:
                f.write(contents)
    def uptodate():
        for filename, contents in templates.items():
            if os.path.isfile(filename):
                if contents == open(filename).read():
                    continue
            return False
        return True
    return dict(
        actions=[(render,)],
        uptodate=[(uptodate,)],
        targets=list(templates.keys()),
    )

def task_submod():
    '''
    run ensure git submodules are up to date
    '''
    SYMS = [
        '+',
        '-',
    ]
    for submod, sha1hash in SUBS2SHAS.items():
        yield dict(
            name=submod,
            actions=[
                fmt('git submodule update --init {submod}')
            ],
            uptodate=[all(map(lambda sym: not sha1hash.startswith(sym), SYMS))],
        )

def pre_pylint():
    '''
    run pylint before the build
    '''
    return dict(
        name='pylint',
        task_dep=[
            'submod',
            'version',
        ],
        actions=[
            fmt('pylint -j{JOBS} --rcfile {PREDIR}/pylint.rc {SOTADIR} || true'),
        ]
    )

def pre_pytest():
    '''
    run pytest before the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            fmt('echo "{REPOROOT}"'),
            fmt('{PYTHON} -m pytest -s -vv {PREDIR}'),
        ],
    )

def pre_pycov():
    '''
    run pycov before the build
    '''
    return dict(
        name='pycov',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            fmt('{PYTHON} -m pytest -s -vv --cov={SOTADIR} {PREDIR}'),
        ],
    )

def pre_mypy():
    '''
    run mypy before the build
    '''
    packages = [d for d in os.listdir('sota') if d != 'utils' and os.path.isdir('sota/'+d)]
    return dict(
        name='mypy',
        task_dep=[
            'version',
            'submod',
            'liblexer',
            'libsha256',
        ],
        actions=[
            'mypy' + ''.join([fmt(' --module sota.{package}') for package in packages]),
        ],
    )

def task_pre():
    '''
    run tasks before the build: pylint, pytest, pycov
    '''
    yield pre_pylint()
    yield pre_pytest()
    #yield pre_pycov()
    yield pre_mypy()

def task_colm():
    '''
    build colm binary for use in build
    '''
    return dict(
        task_dep=[
            fmt('submod:repos/colm'),
        ],
        actions=[
            fmt('cd {COLMDIR} && autoreconf -f -i'),
            fmt('cd {COLMDIR} && ./configure --prefix={REPOROOT}'),
            fmt('cd {COLMDIR} && make && make install'),
        ],
        uptodate=[True],
        targets=[COLM],
        clean=[clean_targets],
    )

def task_ragel():
    '''
    build ragel binary for use in build
    '''
    return dict(
        task_dep=[
            fmt('submod:repos/ragel'),
            'colm',
        ],
        actions=[
            fmt('cd {RAGELDIR} && autoreconf -f -i'),
            fmt('cd {RAGELDIR} && ./configure --prefix={REPOROOT} --with-colm={REPOROOT} --disable-manual'),
            fmt('cd {RAGELDIR} && make && make install'),
        ],
        uptodate=[True],
        targets=[RAGEL],
        clean=[clean_targets],
    )

def task_liblexer():
    '''
    build so libary for use as sota's lexer
    '''
    files = [fmt('{LEXERDIR}/lexer.py')] + rglob(LEXERDIR + '/*.{h,rl,c}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            'ragel',
        ],
        actions=[
            fmt('cd {LEXERDIR} && LD_LIBRARY_PATH={LIBDIR} make --trace -j {JOBS} RAGEL={RAGEL}'),
            fmt('install -C -D {LEXERDIR}/liblexer.so {LIBDIR}/liblexer.so'),
        ],
        uptodate=[True],
        targets=[
            fmt('{LIBDIR}/liblexer.so'),
            fmt('{LEXERDIR}/lexer.cpp'),
            fmt('{LEXERDIR}/test'),
        ],
        clean=[clean_targets],
    )

def task_libcli():
    '''
    build so libary for use as sota's commandline interface
    '''
    files = [DODO, VERSION_H] + rglob(CLIDIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
            fmt('submod:repos/docopt'),
        ],
        actions=[
            fmt('cd {CLIDIR} && make --trace -j {JOBS}'),
            fmt('install -C -D {CLIDIR}/libcli.so {LIBDIR}/libcli.so'),
        ],
        uptodate=[True],
        targets=[
            fmt('{CLIDIR}/test'),
            fmt('{LIBDIR}/libcli.so'),
        ],
        clean=[clean_targets],
    )

def task_libsha256():
    '''
    build so library for use as sota's sha256 hashing functionality
    '''
    files = [DODO] + rglob(SHA256DIR + '/*.{h,c,cpp}')
    return dict(
        file_dep=files,
        task_dep=[
            'version',
        ],
        actions=[
            fmt('cd {SHA256DIR} && make --trace -j {JOBS}'),
            fmt('install -C -D {SHA256DIR}/libsha256.so {LIBDIR}/libsha256.so'),
        ],
        uptodate=[True],
        targets=[
            fmt('{SHA256DIR}/test'),
            fmt('{LIBDIR}/libsha256.so'),
        ],
        clean=[clean_targets],
    )

def task_sota():
    '''
    build sota binary using rpython machinery
    '''
    return dict(
        file_dep=[
            DODO,
            TARGET,
            fmt('{LIBDIR}/libcli.so'),
            fmt('{LIBDIR}/liblexer.so'),
            fmt('{LIBDIR}/libsha256.so'),
        ] + rglob(fmt('{SOTADIR}/*.py')),
        task_dep=[
            'version',
            'pre',
            'libcli',
            'liblexer',
            'libsha256',
        ],
        actions=[
            fmt('mkdir -p {BINDIR}'),
            fmt('{PYTHON} -B {RPYTHON} --no-pdb --output {BINDIR}/{BINARY} {TARGET}'),
        ],
        uptodate=[True],
        targets=[fmt('{BINDIR}/{BINARY}')],
        clean=[clean_targets],
    )

def post_pytest():
    '''
    run pytest after the build
    '''
    return dict(
        name='pytest',
        task_dep=[
            'version',
            'sota',
        ],
        actions=[
            fmt('{PYTHON} -m pytest -s -vv {POSTDIR}'),
        ],
    )

def task_post():
    '''
    run tasks after the build: pytest
    '''
    yield post_pytest()

def task_rmcache():
    '''
    recursively delete python cache files
    '''
    pycache = dict(py2='*.pyc', py3='__pycache__')
    exec_rm = '-exec rm -r "{}" \;'
    for py, cache in pycache.items():
        for dir_ in ('sota', 'tests'):
            yield dict(
                name=py + '-' + dir_,
                actions=[
                    fmt('find {dir_} -depth -name {cache} -type d {exec_rm}'),
                ],
            )

def task_tidy():
    '''
    clean submods and sota/lang repo
    '''
    yield dict(
        name='sota/lang',
        actions=['git clean -xfd'],
    )
    for submod in SUBS2SHAS.keys():
        yield dict(
            name=submod,
            actions=[
                fmt('cd {submod} && git reset --hard HEAD && git clean -xfd')
            ],
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python3.5

import os
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cm
from glob import glob
import musm

# TODO: suppress plot at --min-regret

plt.style.use('ggplot')

def pad(array, length):
    full = np.zeros((length,) + array.shape[1:], dtype=array.dtype)
    full[:len(array)] = array
    return full

def load(path):
    data = musm.load(path)
    num_users = data['args']['num_users_per_group']
    max_iters = data['args']['max_iters']
    loss1_matrix, time_matrix = [], []
    for trace in data['traces']:
        trace = np.array(trace)
        loss1_matrix.append(pad(trace[:,:num_users], max_iters))
        time_matrix.append(pad(trace[:,-1], max_iters))
    return np.array(loss1_matrix), \
           np.array(time_matrix), \
           data['args']

def prettify(ax, max_iters, title):
    xtick = 1 if max_iters <= 10 else 5 if max_iters <= 50 else 10
    xticks = np.hstack([[1], np.arange(xtick, max_iters + 1, xtick)])
    ax.set_xticks(xticks)

    ax.xaxis.label.set_fontsize(18)
    ax.yaxis.label.set_fontsize(18)
    ax.grid(True)
    for line in ax.get_xgridlines() + ax.get_ygridlines():
        line.set_linestyle('-.')

    #ax.set_title(title, fontsize=18)
    legend = ax.legend(loc='upper right', fancybox=False, shadow=False)
    for label in legend.get_texts():
        label.set_fontsize('x-large')
    for label in legend.get_lines():
        label.set_linewidth(2)

_WINTER = cm.ScalarMappable(cmap=plt.get_cmap('winter'),
                            norm=colors.Normalize(vmin=0, vmax=1))
_SUMMER = cm.ScalarMappable(cmap=plt.get_cmap('summer'),
                            norm=colors.Normalize(vmin=0, vmax=1))

def get_style(args):
    transform, tau, pick = args['transform'], args['tau'], args['pick']
    if transform == 'indep':
        return '#555753', 'indep.'
    elif transform == 'sumcov':
        lmbda = args['lmbda']
        return _WINTER.to_rgba(lmbda), 'k only ({}, λ={})'.format(pick, lmbda)
    elif transform == 'varsumvarcov':
        pick_str = {
                'numqueries': 'min #queries',
                'regret': 'regret',
            }[pick]
        color = {
                'numqueries': '#ef2929',
                'regret': '#729fcf',
            }[pick]
        return color, 'v + k ({})'.format(pick_str)

def draw(args):
    loss1_fig, loss1_ax = plt.subplots(1, 1)
    time_fig, time_ax = plt.subplots(1, 1)

    data = []
    for path in args.pickles:
        data.append(load(path))

    # TODO plot selected users
    # TODO plot per-run debug plots

    max_regret1, max_time = -np.inf, -np.inf
    for loss1_matrix, time_matrix, info in data:
        color, label = get_style(info)

        max_iters = min(args.max_iters or info['max_iters'], info['max_iters'])
        x = np.arange(1, max_iters + 1) / info['num_users_per_group']

        # regret

        loss1_matrix = loss1_matrix.mean(axis=2) # average over all users
        y = {'median': np.median, 'mean': np.mean}[args.aggregate] \
                (loss1_matrix, axis=0)[:max_iters]
        yerr = np.std(loss1_matrix, axis=0)[:max_iters] \
                      / np.sqrt(loss1_matrix.shape[0])
        max_regret1 = max(max_regret1, y.max())

        loss1_ax.plot(x, y, linewidth=2, label=label, color=color,
                      marker='o', markersize=6, markevery=5)
        loss1_ax.fill_between(x, y - yerr, y + yerr, linewidth=0, color=color,
                              alpha=0.35)

        # cumulative time

        cumtime_matrix = time_matrix.cumsum(axis=1)
        y = np.mean(cumtime_matrix, axis=0)[:max_iters]
        yerr = np.std(cumtime_matrix, axis=0)[:max_iters] \
                   / np.sqrt(time_matrix.shape[0])
        max_time = max(max_time, y.max())

        time_ax.plot(x, y, linewidth=2, label=label, color=color,
                     marker='o', markersize=6, markevery=5)
        time_ax.fill_between(x, y - yerr, y + yerr, linewidth=0, color=color,
                             alpha=0.35)

    loss1_ax.set_ylabel('regret')
    loss1_ax.set_xlabel('avg. queries per user')
    loss1_ax.set_xlim([1, max_iters / info['num_users_per_group']])
    loss1_ax.set_ylim([0, 1.05 * max_regret1])
    prettify(loss1_ax, max_iters / info['num_users_per_group'], 'Regret')
    loss1_fig.savefig(args.png_basename + '_loss1.png', bbox_inches='tight',
                      pad_inches=0, dpi=120)

    time_ax.set_ylabel('cumulative time (s)')
    loss1_ax.set_xlabel('avg. queries per user')
    time_ax.set_xlim([1, max_iters / info['num_users_per_group']])
    time_ax.set_ylim([0, 1.05 * max_time])
    prettify(time_ax, max_iters / info['num_users_per_group'], 'Time')
    time_fig.savefig(args.png_basename + '_time.png', bbox_inches='tight',
                     pad_inches=0, dpi=120)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('png_basename', type=str,
                        help='basename of the loss/time PNG plots')
    parser.add_argument('pickles', type=str, nargs='+',
                        help='comma-separated list of result pickles')
    parser.add_argument('--max-iters', type=int, default=None,
                        help='maximum iterations to plot')
    parser.add_argument('--aggregate', type=str, default='median',
                        help='how to aggregate the use regrets')
    args = parser.parse_args()

    draw(args)
<commit_msg>draw: update the style to be more readable.<commit_after>#!/usr/bin/env python3.5

import os
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cm
from glob import glob
import musm

# TODO: suppress plot at --min-regret

plt.style.use('ggplot')

def pad(array, length):
    full = np.zeros((length,) + array.shape[1:], dtype=array.dtype)
    full[:len(array)] = array
    return full

def load(path):
    data = musm.load(path)
    num_users = data['args']['num_users_per_group']
    max_iters = data['args']['max_iters']
    loss1_matrix, time_matrix = [], []
    for trace in data['traces']:
        trace = np.array(trace)
        loss1_matrix.append(pad(trace[:,:num_users], max_iters))
        time_matrix.append(pad(trace[:,-1], max_iters))
    return np.array(loss1_matrix), \
           np.array(time_matrix), \
           data['args']

def prettify(ax, max_iters, title):
    xtick = 1 if max_iters <= 10 else 5 if max_iters <= 50 else 10
    xticks = np.hstack([[1], np.arange(xtick, max_iters + 1, xtick)])
    ax.set_xticks(xticks)

    ax.xaxis.label.set_fontsize(18)
    ax.yaxis.label.set_fontsize(18)
    ax.grid(True)
    for line in ax.get_xgridlines() + ax.get_ygridlines():
        line.set_linestyle('-.')

    #ax.set_title(title, fontsize=18)
    legend = ax.legend(loc='upper right', fancybox=False, shadow=False)
    for label in legend.get_texts():
        label.set_fontsize('x-large')
    for label in legend.get_lines():
        label.set_linewidth(2)

_WINTER = cm.ScalarMappable(cmap=plt.get_cmap('winter'),
                            norm=colors.Normalize(vmin=0, vmax=1))
_SUMMER = cm.ScalarMappable(cmap=plt.get_cmap('summer'),
                            norm=colors.Normalize(vmin=0, vmax=1))

def get_style(args):
    transform, tau, pick = args['transform'], args['tau'], args['pick']
    if transform == 'indep':
        return '#555753', 'indep.', '^', '--'
    elif transform == 'sumcov':
        lmbda = args['lmbda']
        return _WINTER.to_rgba(lmbda), 'k only ({}, λ={})'.format(pick, lmbda)
    elif transform == 'varsumvarcov':
        pick_str = {
                'numqueries': 'min #queries',
                'regret': 'regret',
            }[pick]
        color = {
                'numqueries': '#ef2929',
                'regret': '#729fcf',
            }[pick]
        marker = {
                'numqueries': 'o',
                'regret': 'v',
            }[pick]
        ls = {
                'numqueries': '-',
                'regret': '-.',
            }[pick]
        return color, 'v + k ({})'.format(pick_str), marker, ls

def draw(args):
    loss1_fig, loss1_ax = plt.subplots(1, 1)
    time_fig, time_ax = plt.subplots(1, 1)

    data = []
    for path in args.pickles:
        data.append(load(path))

    # TODO plot selected users
    # TODO plot per-run debug plots

    max_regret1, max_time = -np.inf, -np.inf
    for loss1_matrix, time_matrix, info in data:
        color, label, marker, ls = get_style(info)

        max_iters = min(args.max_iters or info['max_iters'], info['max_iters'])
        x = np.arange(1, max_iters + 1) / info['num_users_per_group']

        # regret

        loss1_matrix = loss1_matrix.mean(axis=2) # average over all users
        y = {'median': np.median, 'mean': np.mean}[args.aggregate] \
                (loss1_matrix, axis=0)[:max_iters]
        yerr = np.std(loss1_matrix, axis=0)[:max_iters] \
                      / np.sqrt(loss1_matrix.shape[0])
        max_regret1 = max(max_regret1, y.max())

        loss1_ax.plot(x, y, linewidth=2, label=label, color=color, linestyle=ls,
                      marker=marker, markersize=6, markevery=5)
        loss1_ax.fill_between(x, y - yerr, y + yerr, linewidth=0, color=color,
                              alpha=0.35)

        # cumulative time

        cumtime_matrix = time_matrix.cumsum(axis=1)
        y = np.mean(cumtime_matrix, axis=0)[:max_iters]
        yerr = np.std(cumtime_matrix, axis=0)[:max_iters] \
                   / np.sqrt(time_matrix.shape[0])
        max_time = max(max_time, y.max())

        time_ax.plot(x, y, linewidth=2, label=label, color=color, linestyle=ls,
                     marker=marker, markersize=6, markevery=5)
        time_ax.fill_between(x, y - yerr, y + yerr, linewidth=0, color=color,
                             alpha=0.35)

    loss1_ax.set_ylabel('regret')
    loss1_ax.set_xlabel('avg. queries per user')
    loss1_ax.set_xlim([1, max_iters / info['num_users_per_group']])
    loss1_ax.set_ylim([0, 1.05 * max_regret1])
    prettify(loss1_ax, max_iters / info['num_users_per_group'], 'Regret')
    loss1_fig.savefig(args.png_basename + '_loss1.png', bbox_inches='tight',
                      pad_inches=0, dpi=120)

    time_ax.set_ylabel('cumulative time (s)')
    loss1_ax.set_xlabel('avg. queries per user')
    time_ax.set_xlim([1, max_iters / info['num_users_per_group']])
    time_ax.set_ylim([0, 1.05 * max_time])
    prettify(time_ax, max_iters / info['num_users_per_group'], 'Time')
    time_fig.savefig(args.png_basename + '_time.png', bbox_inches='tight',
                     pad_inches=0, dpi=120)

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('png_basename', type=str,
                        help='basename of the loss/time PNG plots')
    parser.add_argument('pickles', type=str, nargs='+',
                        help='comma-separated list of result pickles')
    parser.add_argument('--max-iters', type=int, default=None,
                        help='maximum iterations to plot')
    parser.add_argument('--aggregate', type=str, default='median',
                        help='how to aggregate the use regrets')
    args = parser.parse_args()

    draw(args)
<|endoftext|>


<commit_before>import boto
import click
import json
import os
import sys
import zipfile

from boto.dynamodb2.table import Table

__version__ = '0.1.0'

# this is the default calculated endpoint path from boto
DEFAULT_ENDPOINTS_PATH = os.path.join(os.path.dirname(boto.__file__), 'endpoints.json')

def load_endpoint_json(path):
    """
    Loads a given JSON file & returns it. Handle being inside of an egg.

    :param path: The path to the JSON file
    :type path: string

    :returns: The loaded data
    """
    if DEFAULT_ENDPOINTS_PATH == path:
        egg_path = os.path.dirname(__file__)
        with zipfile.ZipFile(egg_path, 'r') as egg_file:
            return json.load(egg_file.open('boto/endpoints.json', 'r'))
    else:
        with open(path, 'r') as endpoints_file:
            return json.load(endpoints_file)

# monkeypatch the original load to account for loading inside of an eggsecutable
boto.regioninfo.load_endpoint_json = load_endpoint_json

@click.command()
@click.option('--aws-access-key-id', metavar='AWS_ACCESS_KEY_ID',
              help="This is the AWS access key id.",
              default=os.getenv('AWS_ACCESS_KEY_ID'))
@click.option('--aws-secret-access-key', metavar='AWS_SECRET_ACCESS_KEY',
              help="This is the AWS secret access key.",
              default=os.getenv('AWS_SECRET_ACCESS_KEY'))
@click.option('--region', metavar='REGION',
              help='The AWS region (e.g. us-east-1).', default='us-east-1')
@click.option('--table-name', metavar='TABLE_NAME',
              help='The target table to query.', required=True)
@click.option('--query', metavar='QUERY',
              help='The query to run on the target table.', required=True)
@click.option('--quiet', metavar='QUIET', is_flag=True,
              help="Don't print the version and header info.", default=False)
@click.version_option(__version__)
def main(aws_access_key_id, aws_secret_access_key, region, table_name, query, quiet):
    if not quiet:
        click.echo('Boto ' + boto.__version__)
        click.echo("{} {} {} {} {}".format(aws_access_key_id, aws_secret_access_key, region, table_name, query))

    conn = boto.dynamodb2.connect_to_region(
        region,
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key
    )
    items = Table(table_name, connection=conn).get_item(**json.loads(query))

    for field, value in items.items():
        click.echo(field + ":" + value)

def table_key():
    """Return a single result for the given table and key"""
    pass

def query():
    pass

if __name__ == "__main__":
    sys.exit(main())

<commit_msg>patch up cacerts.txt handling, add shell env style and json style output<commit_after>import atexit
import json
import os
import sys
import tempfile
import zipfile

import boto
import click
from boto.dynamodb2.table import Table

__version__ = '0.1.0'

DYNQ_DEBUG = os.getenv("DYNQ_DEBUG", False)

# this is the default calculated endpoint path from boto
DEFAULT_ENDPOINTS_PATH = os.path.join(os.path.dirname(boto.__file__), 'endpoints.json')

def load_endpoint_json(path):
    """
    Loads a given JSON file & returns it. Handle being inside of an egg.

    :param path: The path to the JSON file
    :type path: string

    :returns: The loaded data
    """
    if DEFAULT_ENDPOINTS_PATH == path:
        # default is inside the egg now, so load from there
        egg_path = os.path.dirname(__file__)
        with zipfile.ZipFile(egg_path, 'r') as egg_file:
            return json.load(egg_file.open('boto/endpoints.json', 'r'))
    else:
        # it was overridden
        with open(path, 'r') as endpoints_file:
            return json.load(endpoints_file)

# monkeypatch the original load to handle being inside of an eggsecutable
boto.regioninfo.load_endpoint_json = load_endpoint_json

def patch_ca_certs():
    """
    Boto needs an actual file path for the cacerts.txt so we extract it from
    inside the egg. If it's been overridden by Boto's 'ca_certificates_file'
    setting then just skip this patch.
    """

    # skip this if user overrides ca_certificates_file
    detected_ca = boto.config.get('Boto', 'ca_certificates_file')
    if detected_ca is None:
        egg_path = os.path.dirname(__file__)
        with zipfile.ZipFile(egg_path, 'r') as egg_file:
            cert_file = tempfile.NamedTemporaryFile(prefix='cacerts', delete=False)
            cert_file.write(egg_file.open('boto/cacerts/cacerts.txt').read())
            cert_file.close()

            if not boto.config.has_section('Boto'):
                boto.config.add_section('Boto')
            boto.config.set('Boto', 'ca_certificates_file', cert_file.name)
            atexit.register(clean_ca_certs, cert_file.name)

def clean_ca_certs(ca_cert_filename):
    os.remove(ca_cert_filename)

@click.command()
@click.option('--aws-access-key-id', metavar='AWS_ACCESS_KEY_ID',
              help="This is the AWS access key id.",
              default=os.getenv('AWS_ACCESS_KEY_ID'))
@click.option('--aws-secret-access-key', metavar='AWS_SECRET_ACCESS_KEY',
              help="This is the AWS secret access key.",
              default=os.getenv('AWS_SECRET_ACCESS_KEY'))
@click.option('--region', metavar='REGION',
              help='The AWS region (e.g. us-east-1).',
              default='us-east-1')
@click.option('--table-name', metavar='TABLE_NAME',
              help='The target table to query.',
              required=True)
@click.option('--query', metavar='QUERY',
              help='The query to run on the target table.',
              required=False)
@click.option('--key-value', metavar='KEY_VALUE',
              help='The query to run on the target table of the form key=value, (converted to {key:value}).',
              required=False)
@click.option('--output-json', metavar='OUTPUT_JSON', is_flag=True,
              help='Output the returned key/values as JSON.',
              required=False,
              default=False)
@click.option('--verbose', metavar='verbose', is_flag=True,
              help="Print the version and header info.", default=False)
@click.version_option(__version__)
def main(aws_access_key_id, aws_secret_access_key, region, table_name, query, key_value, verbose, output_json):
    if key_value is None and query is None:
        raise click.ClickException("Missing --query or --key-value parameter")

    if verbose:
        click.echo('dynq - ' + __version__ + " - A simple DynamoDB client that just works.")
        click.echo('boto ' + boto.__version__)
        click.echo()

    patch_ca_certs()
    connection = boto.dynamodb2.connect_to_region(
        region,
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key=aws_secret_access_key
    )

    if key_value is not None:
        kv = key_value.split('=')
        query_value = {kv[0]: kv[1]}
    else:
        query_value = query

    try:
        run_query(connection, table_name, query_value, output_json)
        return 0
    except:
        click.echo(sys.exc_info()[1])
        return 1

def run_query(connection, table_name, query_value, output_json):
    if DYNQ_DEBUG:
        boto.set_stream_logger('boto')

    items = Table(table_name, connection=connection).get_item(**query_value)
    if output_json:
        click.echo(json.dumps(items._data))
    else:
        for field, value in items.items():
            click.echo(field + "=" + value)

if __name__ == "__main__":
    # TODO why are these error codes broken?
    sys.exit(main())

<|endoftext|>