<commit_before>#! /usr/bin/env python2.7
import sys
import numpy as np
import shutil
from os.path import join as pjoin
from os import getcwd as cwd
from os.path import expanduser
from os.path import sep
from os import makedirs
from os.path import isfile as isfile
import errno
from glob import glob
import re
import subprocess
import multiprocessing
from argparse import ArgumentParser
np.set_printoptions(precision=15)

keyfile = 'cvodes-analytic-int-log.bin'

def create_dir(path):
    try:
        makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def __check_exit(x):
    if x != 0:
        sys.exit(x)

def __check_error(builder, num_conditions, nvar, t, validator, atol, rtol):
    globtxt = '*-gpu-log.bin' if builder == 'gpu' else '*-int-log.bin'
    match = np.where(np.isclose(validator[:, 0], t, atol=atol))
    print match, validator[match, 0]
    print validator[:, 0]
    key_arr = validator[match, 1:]
    non_zero = np.where(key_arr != 0)
    kview = key_arr[non_zero]
    with open('logfile', 'a') as file:
        file.write('t={}\n'.format(t))
        for f in glob(pjoin('log', globtxt)):
            if keyfile in f:
                continue
            array = np.fromfile(f, dtype='float64')
            array = array.reshape((-1, 1 + num_conditions * nvar))

            file.write(f + '\n')
            data_arr = array[1:, 1:]
            #now compare column by column and get max err

            aview = data_arr[non_zero]

            err = np.abs(aview - kview) / kview
            err = np.sum(np.abs(err)**2)
            err = np.sum(err)
            norm_err = np.sqrt(err) / float(nvar * (num_conditions - 1))

            file.write("{:.16e}\t{:.16e}\t\n".format(max_err, norm_err))

def __execute(builder, num_threads, num_conditions):
    with open('logfile', 'a') as file:
        if builder == 'gpu':
            for exe in glob('*-gpu'):
                file.write('\n' + exe + '\n')
                subprocess.check_call([pjoin(cwd(), exe), str(num_conditions)], stdout=file)
        else:
            for exe in glob('*-int'):
                if exe in keyfile:
                    continue
                file.write('\n' + exe + '\n')
                subprocess.check_call([pjoin(cwd(), exe), str(num_threads), str(num_conditions)], stdout=file)

def __check_valid(nvar, num_conditions, num_steps):
    if not isfile(pjoin('log', 'valid.bin')):
        return None
    validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
    if validator.shape[0] == nvar * (num_conditions + 1) * num_steps:
        validator = validator.reshape((-1, 1 + num_conditions * nvar))
        return validator
    return None

def __run_and_check(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol):
        import pyJac
        #first compile and run cvodes to get the baseline
        __check_exit(pyJac.create_jacobian(lang='c',
            mech_name=mech,
            therm_name=thermo,
            initial_state=initial_conditions,
            optimize_cache=False,
            build_path=build_path))
        small_step = atol
        t_step = np.logspace(-10, -2, num=10)
        t_step = np.array([10.**x for x in t_step])
        nvar = None
        #get num vars
        with open(pjoin(build_path, 'mechanism.h'), 'r') as file:
            for line in file.readlines():
                if 'NN' in line:
                    match = re.search(r'\b(\d+)$', line.strip())
                    if match:
                        nvar = int(match.group(1))
                        break
        assert nvar is not None
        arg_list = ['-j{}'.format(num_threads),
                'DEBUG=FALSE', 'FAST_MATH=FALSE', 'LOG_OUTPUT=TRUE',
                'SHUFFLE=FALSE', 'PRINT=FALSE', 'mechanism_dir={}'.format(build_path),
                'ATOL={}'.format(atol), 'RTOL={}'.format(rtol)]
        if initial_conditions:
            arg_list.append('SAME_IC=TRUE')
            num_conditions = 1 #they're all the same
        else:
            arg_list.append('SAME_IC=FALSE')

        small_step_count = int(t_step[-1]/small_step)
        validator = __check_valid(nvar, num_conditions, small_step_count)
        if validator is None:
            with open('logfile', 'a') as file:
                subprocess.check_call(['scons', 'cpu'] + arg_list +
                            ['t_step={}'.format(small_step),
                             'num_steps={}'.format(small_step_count)], stdout=file)
                #run
                subprocess.check_call([pjoin(cwd(), 'cvodes-analytic-int'),
                    str(num_threads), str(num_conditions)],
                    stdout=file)
                #copy to saved data
                shutil.copy(pjoin(cwd(), 'log', keyfile),
                            pjoin(cwd(), 'log', 'valid.bin'))

            validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
            validator = validator.reshape((-1, 1 + num_conditions * nvar))

        arg_list.append('num_steps=1')
        langs = []
        if not skip_c:
            langs += ['c']
        if not skip_cuda:
            langs += ['cuda']
        if langs == []:
            raise Exception('No languages to test specified')
        builder = {'c':'cpu', 'cuda':'gpu'}
        opt = [False]#[True, False]
        smem = [False, True]
        #now check for various options
        for lang in langs:
            for cache_opt in opt:
                if lang == 'cuda':
                    for shared_mem in smem:
                        for t in t_step:
                            with open('logfile', 'a') as file:
                                #subprocess.check_call(['scons', '-c'])
                                file.write('\ncache_opt: {}\n'
                                       'shared_mem: {}\n'.format(
                                        cache_opt, not shared_mem))
                                __check_exit(pyJac.create_jacobian(lang=lang,
                                mech_name=mech,
                                therm_name=thermo,
                                initial_state=initial_conditions,
                                optimize_cache=cache_opt,
                                multi_thread=num_threads,
                                no_shared=shared_mem,
                                build_path=build_path))

                                subprocess.check_call(['scons', builder[lang]] + arg_list +
                                        ['t_step={}'.format(t)], stdout=file)
                                __execute(builder[lang], num_threads, num_conditions)
                                __check_error(builder[lang], num_conditions, nvar, t,
                                                validator, atol, rtol)

                else:
                    for t in t_step:
                        with open('logfile', 'a') as file:
                            file.write('\ncache_opt: {}\n'.format(
                                        cache_opt))
                            __check_exit(pyJac.create_jacobian(lang=lang,
                            mech_name=mech,
                            therm_name=thermo,
                            initial_state=initial_conditions,
                            optimize_cache=cache_opt,
                            multi_thread=num_threads,
                            build_path=build_path))

                            subprocess.check_call(['scons', builder[lang]] + arg_list +
                                        ['t_step={}'.format(t)], stdout=file)
                            __execute(builder[lang], num_threads, num_conditions)
                            __check_error(builder[lang], num_conditions, nvar, t,
                                            validator, atol, rtol)

def run_log(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol):
    with open('logfile', 'w') as file:
        pass
    if initial_conditions is not None:
        with open('logfile', 'a') as file:
            file.write('Running Same ICs\n')
        __run_and_check(mech, thermo, initial_conditions, build_path,
            num_threads, num_conditions, None, skip_c, skip_cuda,
            atol, rtol)
    if test_data is not None:
        with open('logfile', 'a') as file:
            file.write('PaSR ICs\n')
        try:
            shutil.copyfile(test_data, 'ign_data.bin')
        except shutil.Error:
            pass
        __run_and_check(mech, thermo, '', build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol)

if __name__ == '__main__':
    parser = ArgumentParser(description='logger: Log and compare solver output for the various ODE Solvers')

    parser.add_argument('-i', '--input',
                        type=str,
                        required=True,
                        help='The mechanism to check.')
    parser.add_argument('-t', '--thermo',
                    type=str,
                    default=None,
                    help='Thermodynamic database filename (e.g., '
                         'therm.dat), or nothing if in mechanism.')
    parser.add_argument('-ic', '--initial_conditions',
                        type=str,
                        required=False,
                        default=None,
                        help='The initial conditions for same initial condition testing. '
                             'If not supplied, this is skipped.')
    parser.add_argument('-b', '--build_path',
                        type=str,
                        required=False,
                        default=pjoin(cwd(),'out') + sep,
                        help='The folder to generate the mechanism in')
    parser.add_argument('-nt', '--num_threads',
                        type=int,
                        required=False,
                        default=multiprocessing.cpu_count(),
                        help='The number of threads to run the CPU ode solvers with.')
    parser.add_argument('-nc', '--num_conditions',
                        type=int,
                        required=False,
                        default=1,
                        help='The number of conditions to test')
    parser.add_argument('-td', '--test_data',
                        type=str,
                        required=False,
                        default=None,
                        help='A numpy file output generated from the partially_stirred_reactor component '
                             'of pyJac.  Used for testing if supplied.')
    parser.add_argument('-sc', '--skip_c',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip C testing. Note baseline is still calculated using CVODES')
    parser.add_argument('-scu', '--skip_cuda',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip CUDA testing.')
    parser.add_argument('-atol', '--abs_tolerance',
                        required=False,
                        type=float,
                        default=1e-15,
                        help='The absolute tolerance to use during integration')
    parser.add_argument('-rtol', '--rel_tolerance',
                        required=False,
                        type=float,
                        default=1e-8,
                        help='The relative tolerance to use during integration')
    parser.add_argument('-pjd', '--pyjac_directory',
                        required=False,
                        type=str,
                        default=pjoin(expanduser('~'), 'pyJac'),
                        help='The relative tolerance to use during integration')
    args = parser.parse_args()

    sys.path.append(args.pyjac_directory)
    assert not (args.test_data is None and args.initial_conditions is None), \
    "Either a test data file or initial conditions must be specified"

    create_dir(args.build_path)
    create_dir('./log/')

    run_log(args.input, args.thermo, args.initial_conditions, args.build_path,
        args.num_threads, args.num_conditions, args.test_data,
        args.skip_c, args.skip_cuda, args.abs_tolerance, args.rel_tolerance)<commit_msg>updated for specified output times<commit_after>#! /usr/bin/env python2.7
import sys
import numpy as np
import shutil
from os.path import join as pjoin
from os import getcwd as cwd
from os.path import expanduser
from os.path import sep
from os import makedirs
from os.path import isfile as isfile
import errno
from glob import glob
import re
import subprocess
import multiprocessing
from argparse import ArgumentParser
np.set_printoptions(precision=15)

keyfile = 'cvodes-analytic-int-log.bin'

def create_dir(path):
    try:
        makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def __check_exit(x):
    if x != 0:
        sys.exit(x)

def __check_error(builder, num_conditions, nvar, t, validator, atol, rtol):
    globtxt = '*-gpu-log.bin' if builder == 'gpu' else '*-int-log.bin'
    match = np.where(np.isclose(validator[:, 0], t, atol=atol))
    print match, validator[match, 0]
    print validator[:, 0]
    key_arr = validator[match, 1:]
    non_zero = np.where(key_arr != 0)
    kview = key_arr[non_zero]
    with open('logfile', 'a') as file:
        file.write('t={}\n'.format(t))
        for f in glob(pjoin('log', globtxt)):
            if keyfile in f:
                continue
            array = np.fromfile(f, dtype='float64')
            array = array.reshape((-1, 1 + num_conditions * nvar))

            file.write(f + '\n')
            data_arr = array[1:, 1:]
            #now compare column by column and get max err

            aview = data_arr[non_zero]

            err = np.abs(aview - kview) / kview
            err = np.sum(np.abs(err)**2)
            err = np.sum(err)
            norm_err = np.sqrt(err) / float(nvar * (num_conditions - 1))

            file.write("{:.16e}\t{:.16e}\t\n".format(max_err, norm_err))

def __execute(builder, num_threads, num_conditions):
    with open('logfile', 'a') as file:
        if builder == 'gpu':
            for exe in glob('*-gpu'):
                file.write('\n' + exe + '\n')
                subprocess.check_call([pjoin(cwd(), exe), str(num_conditions)], stdout=file)
        else:
            for exe in glob('*-int'):
                if exe in keyfile:
                    continue
                file.write('\n' + exe + '\n')
                subprocess.check_call([pjoin(cwd(), exe), str(num_threads), str(num_conditions)], stdout=file)

def __check_valid(nvar, num_conditions, step_list):
    if not isfile(pjoin('log', 'valid.bin')):
        return None
    validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
    validator = validator.reshape((-1, 1 + num_conditions * nvar))
    if np.all(x in validator[:, 0] for x in step_list)
        return validator
    return None

def __run_and_check(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol):
        import pyJac
        #first compile and run cvodes to get the baseline
        __check_exit(pyJac.create_jacobian(lang='c',
            mech_name=mech,
            therm_name=thermo,
            initial_state=initial_conditions,
            optimize_cache=False,
            build_path=build_path))
        small_step = atol
        t_step = range(-10, -1)
        t_step = np.array([10.**x for x in t_step])
        nvar = None
        #get num vars
        with open(pjoin(build_path, 'mechanism.h'), 'r') as file:
            for line in file.readlines():
                if 'NN' in line:
                    match = re.search(r'\b(\d+)$', line.strip())
                    if match:
                        nvar = int(match.group(1))
                        break
        assert nvar is not None
        arg_list = ['-j{}'.format(num_threads),
                'DEBUG=FALSE', 'FAST_MATH=FALSE', 'LOG_OUTPUT=TRUE',
                'SHUFFLE=FALSE', 'PRINT=FALSE', 'mechanism_dir={}'.format(build_path),
                'ATOL={}'.format(atol), 'RTOL={}'.format(rtol)]
        if initial_conditions:
            arg_list.append('SAME_IC=TRUE')
            num_conditions = 1 #they're all the same
        else:
            arg_list.append('SAME_IC=FALSE')

        log_step_list = np.array([int(x) for x in t_step / small_step])
        small_step_count = log_step_list[-1]
        validator = __check_valid(nvar, num_conditions, t_step)
        if validator is None:
            with open('logfile', 'a') as file:
                subprocess.check_call(['scons', 'cpu'] + arg_list +
                            ['t_step={}'.format(small_step),
                             'num_steps={}'.format(small_step_count),
                             'log_steps={}'.format(','.join([str(x) for x in log_step_list]))],
                              stdout=file)
                #run
                subprocess.check_call([pjoin(cwd(), 'cvodes-analytic-int'),
                    str(num_threads), str(num_conditions)],
                    stdout=file)
                #copy to saved data
                shutil.copy(pjoin(cwd(), 'log', keyfile),
                            pjoin(cwd(), 'log', 'valid.bin'))

            validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
            validator = validator.reshape((-1, 1 + num_conditions * nvar))

        arg_list.append('num_steps=1')
        langs = []
        if not skip_c:
            langs += ['c']
        if not skip_cuda:
            langs += ['cuda']
        if langs == []:
            raise Exception('No languages to test specified')
        builder = {'c':'cpu', 'cuda':'gpu'}
        opt = [False]#[True, False]
        smem = [False, True]
        #now check for various options
        for lang in langs:
            for cache_opt in opt:
                if lang == 'cuda':
                    for shared_mem in smem:
                        for t in t_step:
                            with open('logfile', 'a') as file:
                                #subprocess.check_call(['scons', '-c'])
                                file.write('\ncache_opt: {}\n'
                                       'shared_mem: {}\n'.format(
                                        cache_opt, not shared_mem))
                                __check_exit(pyJac.create_jacobian(lang=lang,
                                mech_name=mech,
                                therm_name=thermo,
                                initial_state=initial_conditions,
                                optimize_cache=cache_opt,
                                multi_thread=num_threads,
                                no_shared=shared_mem,
                                build_path=build_path))

                                subprocess.check_call(['scons', builder[lang]] + arg_list +
                                        ['t_step={}'.format(t)], stdout=file)
                                __execute(builder[lang], num_threads, num_conditions)
                                __check_error(builder[lang], num_conditions, nvar, t,
                                                validator, atol, rtol)

                else:
                    for t in t_step:
                        with open('logfile', 'a') as file:
                            file.write('\ncache_opt: {}\n'.format(
                                        cache_opt))
                            __check_exit(pyJac.create_jacobian(lang=lang,
                            mech_name=mech,
                            therm_name=thermo,
                            initial_state=initial_conditions,
                            optimize_cache=cache_opt,
                            multi_thread=num_threads,
                            build_path=build_path))

                            subprocess.check_call(['scons', builder[lang]] + arg_list +
                                        ['t_step={}'.format(t)], stdout=file)
                            __execute(builder[lang], num_threads, num_conditions)
                            __check_error(builder[lang], num_conditions, nvar, t,
                                            validator, atol, rtol)

def run_log(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol):
    with open('logfile', 'w') as file:
        pass
    if initial_conditions is not None:
        with open('logfile', 'a') as file:
            file.write('Running Same ICs\n')
        __run_and_check(mech, thermo, initial_conditions, build_path,
            num_threads, num_conditions, None, skip_c, skip_cuda,
            atol, rtol)
    if test_data is not None:
        with open('logfile', 'a') as file:
            file.write('PaSR ICs\n')
        try:
            shutil.copyfile(test_data, 'ign_data.bin')
        except shutil.Error:
            pass
        __run_and_check(mech, thermo, '', build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol)

if __name__ == '__main__':
    parser = ArgumentParser(description='logger: Log and compare solver output for the various ODE Solvers')

    parser.add_argument('-i', '--input',
                        type=str,
                        required=True,
                        help='The mechanism to check.')
    parser.add_argument('-t', '--thermo',
                    type=str,
                    default=None,
                    help='Thermodynamic database filename (e.g., '
                         'therm.dat), or nothing if in mechanism.')
    parser.add_argument('-ic', '--initial_conditions',
                        type=str,
                        required=False,
                        default=None,
                        help='The initial conditions for same initial condition testing. '
                             'If not supplied, this is skipped.')
    parser.add_argument('-b', '--build_path',
                        type=str,
                        required=False,
                        default=pjoin(cwd(),'out') + sep,
                        help='The folder to generate the mechanism in')
    parser.add_argument('-nt', '--num_threads',
                        type=int,
                        required=False,
                        default=multiprocessing.cpu_count(),
                        help='The number of threads to run the CPU ode solvers with.')
    parser.add_argument('-nc', '--num_conditions',
                        type=int,
                        required=False,
                        default=1,
                        help='The number of conditions to test')
    parser.add_argument('-td', '--test_data',
                        type=str,
                        required=False,
                        default=None,
                        help='A numpy file output generated from the partially_stirred_reactor component '
                             'of pyJac.  Used for testing if supplied.')
    parser.add_argument('-sc', '--skip_c',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip C testing. Note baseline is still calculated using CVODES')
    parser.add_argument('-scu', '--skip_cuda',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip CUDA testing.')
    parser.add_argument('-atol', '--abs_tolerance',
                        required=False,
                        type=float,
                        default=1e-15,
                        help='The absolute tolerance to use during integration')
    parser.add_argument('-rtol', '--rel_tolerance',
                        required=False,
                        type=float,
                        default=1e-8,
                        help='The relative tolerance to use during integration')
    parser.add_argument('-pjd', '--pyjac_directory',
                        required=False,
                        type=str,
                        default=pjoin(expanduser('~'), 'pyJac'),
                        help='The relative tolerance to use during integration')
    args = parser.parse_args()

    sys.path.append(args.pyjac_directory)
    assert not (args.test_data is None and args.initial_conditions is None), \
    "Either a test data file or initial conditions must be specified"

    create_dir(args.build_path)
    create_dir('./log/')

    run_log(args.input, args.thermo, args.initial_conditions, args.build_path,
        args.num_threads, args.num_conditions, args.test_data,
        args.skip_c, args.skip_cuda, args.abs_tolerance, args.rel_tolerance)<|endoftext|>


<commit_before>#! /usr/bin/env python2.7
import sys
import numpy as np
import shutil
from os.path import join as pjoin
from os import getcwd as cwd
from os.path import expanduser
from os.path import sep
from os import makedirs
from os.path import isfile as isfile
import errno
from glob import glob
import re
import subprocess
import multiprocessing
from argparse import ArgumentParser
from pyjac import create_jacobian
from optionLoop import optionloop
np.set_printoptions(precision=15)

scons = subprocess.check_output('which scons', shell=True).strip()

valid_int = 'cvodes-analytic-int'
keyfile = valid_int + '-log.bin'

def create_dir(path):
    try:
        makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def __check_exit(x):
    if x != 0:
        sys.exit(x)

def __check_error(builder, num_conditions, nvar, validator, atol, rtol):
    globtxt = '*-gpu-log.bin' if builder == 'gpu' else '*-int-log.bin'
    key_arr = validator[-1, 1:]
    with open('logfile', 'a') as file:
        for f in glob(pjoin('log', globtxt)):
            if keyfile in f:
                continue
            array = np.fromfile(f, dtype='float64')
            array = array.reshape((-1, 1 + num_conditions * nvar))

            file.write(f + '\n')
            data_arr = array[-1, 1:]
            data_arr = data_arr.reshape(-1, nvar)
            key_arr = key_arr.reshape(-1, nvar)
            #now compare column by column and get max err
            err = np.abs(data_arr - key_arr) / (atol + rtol * np.abs(key_arr))
            L2_err = np.linalg.norm(err, axis=1, ord=2)
            Linf_err = np.linalg.norm(err, axis=1, ord=np.inf)

            file.write('L2 (max, mean) = {:.16e}, {:.16e}\n'.format(np.max(L2_err), np.mean(L2_err)))
            file.write('Linf (max, mean) = {:.16e}, {:.16e}\n'.format(np.max(Linf_err), np.mean(Linf_err)))

def __execute(builder, num_threads, num_conditions, t_step=None):
    with open('logerr', 'a') as file:
        if builder == 'gpu':
            for exe in glob('*-gpu'):
                file.write('\n' + exe + '\n')
                try:
                    subprocess.check_call([pjoin(cwd(), exe), str(num_conditions)], stdout=file)
                except CalledProcessError as e:
                    returncode = e.returncode
                    file.write('Error encountered running {}\n'.format(' '.join([exe, str(num_conditions)])))
                    file.write('Error code: {}\n'.format(returncode))
                    sys.exit(-1)
                shutil.copy(pjoin('log', exe + '-log.bin'),
                            pjoin('log', exe + '-log_{:.0e}.bin'.format(t_step)))
        else:
            for exe in glob('*-int'):
                if exe in valid_int:
                    continue
                file.write('\n' + exe + '\n')
                try:
                    subprocess.check_call([pjoin(cwd(), exe), str(num_threads), str(num_conditions)], stdout=file)
                except CalledProcessError, e:
                    returncode = e.returncode
                    file.write('Error encountered running {}\n'.format(' '.join([exe, str(num_threads), str(num_conditions)])))
                    file.write('Error code: {}\n'.format(returncode))
                    sys.exit(-1)
                shutil.copy(pjoin('log', exe + '-log.bin'),
                            pjoin('log', exe + '-log_{:.0e}.bin'.format(t_step)))

def __check_valid(nvar, num_conditions, t_end, t_step):
    if not isfile(pjoin('log', 'valid.bin')):
        return None
    validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
    if validator.shape[0] % (1 + num_conditions * nvar) == 0:
        validator = validator.reshape((-1, 1 + num_conditions * nvar))
        if np.any(np.isclose(validator[:, 0], t_end, atol=t_step/10.)):
            return validator
    return None

def __run_and_check(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol, small_atol, small_rtol):
        #first compile and run the explicit integrator to get the baseline
        __check_exit(create_jacobian(lang='c',
            mech_name=mech,
            therm_name=thermo,
            initial_state=initial_conditions,
            optimize_cache=False,
            build_path=build_path))
        nvar = None
        #get num vars
        with open(pjoin(build_path, 'mechanism.h'), 'r') as file:
            for line in file.readlines():
                if 'NN' in line:
                    match = re.search(r'\b(\d+)$', line.strip())
                    if match:
                        nvar = int(match.group(1))
                        break
        if num_conditions is None and test_data is not None:
            num_conditions = np.fromfile('ign_data.bin').reshape((-1, nvar + 2)).shape[0]
        assert nvar is not None
        arg_list = ['-j{}'.format(num_threads),
                'DEBUG=FALSE', 'FAST_MATH=FALSE', 'LOG_OUTPUT=TRUE', 'LOG_END_ONLY=TRUE',
                'SHUFFLE=FALSE', 'PRINT=FALSE', 'CV_HMAX=0', 'CV_MAX_STEPS=-1',
                'mechanism_dir={}'.format(build_path)]

        if initial_conditions:
            arg_list.append('SAME_IC=TRUE')
            num_conditions = 1 #they're all the same
        else:
            arg_list.append('SAME_IC=FALSE')

        langs = []
        if not skip_c:
            langs += ['c']
        if not skip_cuda:
            langs += ['cuda']
        if langs == []:
            raise Exception('No languages to test specified')
        builder = {'c':'cpu', 'cuda':'gpu'}
        oploop = optionloop({'lang' : ['c'],
                             'cache_opt' : [False, True],
                             'smem' : [False]})
        oploop += optionloop({'lang' : ['cuda'],
                             'cache_opt' : [False, True],
                             'smem' : [False, True]})
        small_tol = ['ATOL={:.0e}'.format(small_atol), 'RTOL={:.0e}'.format(small_rtol)]
        large_tol = ['ATOL={:.0e}'.format(atol), 'RTOL={:.0e}'.format(rtol)]
        with open('logfile', 'a') as file:
            with open('logerr', 'a') as errfile:
                for op in oploop:
                    lang = op['lang']
                    cache_opt = op['cache_opt']
                    shared_mem = op['smem']
                    __check_exit(create_jacobian(lang=lang,
                                    mech_name=mech,
                                    therm_name=thermo,
                                    initial_state=initial_conditions,
                                    optimize_cache=cache_opt,
                                    multi_thread=num_threads,
                                    no_shared=shared_mem,
                                    build_path=build_path))
                    file.write('\ncache_opt: {}\n'
                               'shared_mem: {}\n'.format(
                               cache_opt, not shared_mem))
                    for j in range(-4, -13, -1):
                        t_step = np.power(10.0, j)
                        #build the validation set for this timestep
                        extra_args = ['t_step={:.0e}'.format(t_step), 't_end={:.0e}'.format(t_step)]
                        subprocess.check_call([scons, 'cpu'] + arg_list + extra_args + small_tol, stdout=errfile)
                        #run
                        subprocess.check_call([pjoin(cwd(), valid_int),
                            str(num_threads), str(num_conditions)],
                            stdout=errfile)
                        #copy to saved data
                        shutil.copy(pjoin(cwd(), 'log', keyfile),
                                    pjoin(cwd(), 'log', 'valid.bin'))

                        validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
                        validator = validator.reshape((-1, 1 + num_conditions * nvar))

                        file.write('t_step={:.0e}\n'.format(t_step))
                        file.flush()
                        subprocess.check_call([scons, builder[lang]] + arg_list + extra_args + large_tol,
                            stdout=errfile, stderr=errfile)
                        __execute(builder[lang], num_threads, num_conditions, t_step)
                        __check_error(builder[lang], num_conditions, nvar,
                                        validator, atol, rtol)

def run_log(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol, small_atol, small_rtol):
    with open('logfile', 'w') as file:
        pass
    with open('logerr', 'w') as file:
        pass
    if initial_conditions is not None:
        with open('logfile', 'a') as file:
            file.write('Running Same ICs\n')
        __run_and_check(mech, thermo, initial_conditions, build_path,
            num_threads, 1 if num_conditions is None else num_conditions,
            None, skip_c, skip_cuda, atol, rtol,
            small_atol, small_rtol)
    if test_data is not None:
        with open('logfile', 'a') as file:
            file.write('PaSR ICs\n')
        try:
            shutil.copyfile(test_data, 'ign_data.bin')
        except shutil.Error:
            pass
        __run_and_check(mech, thermo, '', build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol, small_atol, small_rtol)

if __name__ == '__main__':
    parser = ArgumentParser(description='logger: Log and compare solver output for the various ODE Solvers')

    parser.add_argument('-i', '--input',
                        type=str,
                        required=True,
                        help='The mechanism to check.')
    parser.add_argument('-t', '--thermo',
                    type=str,
                    default=None,
                    help='Thermodynamic database filename (e.g., '
                         'therm.dat), or nothing if in mechanism.')
    parser.add_argument('-ic', '--initial_conditions',
                        type=str,
                        required=False,
                        default=None,
                        help='The initial conditions for same initial condition testing. '
                             'If not supplied, this is skipped.')
    parser.add_argument('-b', '--build_path',
                        type=str,
                        required=False,
                        default=pjoin(cwd(),'out') + sep,
                        help='The folder to generate the mechanism in')
    parser.add_argument('-nt', '--num_threads',
                        type=int,
                        required=False,
                        default=multiprocessing.cpu_count(),
                        help='The number of threads to run the CPU ode solvers with.')
    parser.add_argument('-nc', '--num_conditions',
                        type=int,
                        required=False,
                        default=None,
                        help='The number of conditions to test')
    parser.add_argument('-td', '--test_data',
                        type=str,
                        required=False,
                        default=None,
                        help='A numpy file output generated from the partially_stirred_reactor component '
                             'of pyJac.  Used for testing if supplied.')
    parser.add_argument('-sc', '--skip_c',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip C testing. Note baseline is still calculated using an CPU integrator')
    parser.add_argument('-scu', '--skip_cuda',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip CUDA testing.')
    parser.add_argument('-atol', '--abs_tolerance',
                        required=False,
                        type=float,
                        default=1e-13,
                        help='The absolute tolerance to use during integration')
    parser.add_argument('-rtol', '--rel_tolerance',
                        required=False,
                        type=float,
                        default=1e-10,
                        help='The relative tolerance to use during integration')
    parser.add_argument('-satol', '--abs_tolerance_small',
                        required=False,
                        type=float,
                        default=1e-20,
                        help='The absolute tolerance to use during integration')
    parser.add_argument('-srtol', '--rel_tolerance_small',
                        required=False,
                        type=float,
                        default=1e-15,
                        help='The relative tolerance to use during integration')
    args = parser.parse_args()

    assert not (args.test_data is None and args.initial_conditions is None), \
    "Either a test data file or initial conditions must be specified"

    create_dir(args.build_path)
    create_dir('./log/')

    run_log(args.input, args.thermo, args.initial_conditions, args.build_path,
        args.num_threads, args.num_conditions, args.test_data,
        args.skip_c, args.skip_cuda, args.abs_tolerance, args.rel_tolerance,
        args.abs_tolerance_small, args.rel_tolerance_small)<commit_msg>fixed wrong error in try and lowered tol<commit_after>#! /usr/bin/env python2.7
import sys
import numpy as np
import shutil
from os.path import join as pjoin
from os import getcwd as cwd
from os.path import expanduser
from os.path import sep
from os import makedirs
from os.path import isfile as isfile
import errno
from glob import glob
import re
import subprocess
import multiprocessing
from argparse import ArgumentParser
from pyjac import create_jacobian
from optionLoop import optionloop
np.set_printoptions(precision=15)

scons = subprocess.check_output('which scons', shell=True).strip()

valid_int = 'cvodes-analytic-int'
keyfile = valid_int + '-log.bin'

def create_dir(path):
    try:
        makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def __check_exit(x):
    if x != 0:
        sys.exit(x)

def __check_error(builder, num_conditions, nvar, validator, atol, rtol):
    globtxt = '*-gpu-log.bin' if builder == 'gpu' else '*-int-log.bin'
    key_arr = validator[-1, 1:]
    with open('logfile', 'a') as file:
        for f in glob(pjoin('log', globtxt)):
            if keyfile in f:
                continue
            array = np.fromfile(f, dtype='float64')
            array = array.reshape((-1, 1 + num_conditions * nvar))

            file.write(f + '\n')
            data_arr = array[-1, 1:]
            data_arr = data_arr.reshape(-1, nvar)
            key_arr = key_arr.reshape(-1, nvar)
            #now compare column by column and get max err
            err = np.abs(data_arr - key_arr) / (atol + rtol * np.abs(key_arr))
            L2_err = np.linalg.norm(err, axis=1, ord=2)
            Linf_err = np.linalg.norm(err, axis=1, ord=np.inf)

            file.write('L2 (max, mean) = {:.16e}, {:.16e}\n'.format(np.max(L2_err), np.mean(L2_err)))
            file.write('Linf (max, mean) = {:.16e}, {:.16e}\n'.format(np.max(Linf_err), np.mean(Linf_err)))

def __execute(builder, num_threads, num_conditions, t_step=None):
    with open('logerr', 'a') as file:
        if builder == 'gpu':
            for exe in glob('*-gpu'):
                file.write('\n' + exe + '\n')
                try:
                    subprocess.check_call([pjoin(cwd(), exe), str(num_conditions)], stdout=file)
                except subprocess.CalledProcessError as e:
                    returncode = e.returncode
                    file.write('Error encountered running {}\n'.format(' '.join([exe, str(num_conditions)])))
                    file.write('Error code: {}\n'.format(returncode))
                    sys.exit(-1)
                shutil.copy(pjoin('log', exe + '-log.bin'),
                            pjoin('log', exe + '-log_{:.0e}.bin'.format(t_step)))
        else:
            for exe in glob('*-int'):
                if exe in valid_int:
                    continue
                file.write('\n' + exe + '\n')
                try:
                    subprocess.check_call([pjoin(cwd(), exe), str(num_threads), str(num_conditions)], stdout=file)
                except subprocess.CalledProcessError, e:
                    returncode = e.returncode
                    file.write('Error encountered running {}\n'.format(' '.join([exe, str(num_threads), str(num_conditions)])))
                    file.write('Error code: {}\n'.format(returncode))
                    sys.exit(-1)
                shutil.copy(pjoin('log', exe + '-log.bin'),
                            pjoin('log', exe + '-log_{:.0e}.bin'.format(t_step)))

def __check_valid(nvar, num_conditions, t_end, t_step):
    if not isfile(pjoin('log', 'valid.bin')):
        return None
    validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
    if validator.shape[0] % (1 + num_conditions * nvar) == 0:
        validator = validator.reshape((-1, 1 + num_conditions * nvar))
        if np.any(np.isclose(validator[:, 0], t_end, atol=t_step/10.)):
            return validator
    return None

def __run_and_check(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol, small_atol, small_rtol):
        #first compile and run the explicit integrator to get the baseline
        __check_exit(create_jacobian(lang='c',
            mech_name=mech,
            therm_name=thermo,
            initial_state=initial_conditions,
            optimize_cache=False,
            build_path=build_path))
        nvar = None
        #get num vars
        with open(pjoin(build_path, 'mechanism.h'), 'r') as file:
            for line in file.readlines():
                if 'NN' in line:
                    match = re.search(r'\b(\d+)$', line.strip())
                    if match:
                        nvar = int(match.group(1))
                        break
        if num_conditions is None and test_data is not None:
            num_conditions = np.fromfile('ign_data.bin').reshape((-1, nvar + 2)).shape[0]
        assert nvar is not None
        arg_list = ['-j{}'.format(num_threads),
                'DEBUG=FALSE', 'FAST_MATH=FALSE', 'LOG_OUTPUT=TRUE', 'LOG_END_ONLY=TRUE',
                'SHUFFLE=FALSE', 'PRINT=FALSE', 'CV_HMAX=0', 'CV_MAX_STEPS=-1',
                'mechanism_dir={}'.format(build_path)]

        if initial_conditions:
            arg_list.append('SAME_IC=TRUE')
            num_conditions = 1 #they're all the same
        else:
            arg_list.append('SAME_IC=FALSE')

        langs = []
        if not skip_c:
            langs += ['c']
        if not skip_cuda:
            langs += ['cuda']
        if langs == []:
            raise Exception('No languages to test specified')
        builder = {'c':'cpu', 'cuda':'gpu'}
        oploop = optionloop({'lang' : ['c'],
                             'cache_opt' : [False, True],
                             'smem' : [False]})
        oploop += optionloop({'lang' : ['cuda'],
                             'cache_opt' : [False, True],
                             'smem' : [False, True]})
        small_tol = ['ATOL={:.0e}'.format(small_atol), 'RTOL={:.0e}'.format(small_rtol)]
        large_tol = ['ATOL={:.0e}'.format(atol), 'RTOL={:.0e}'.format(rtol)]
        with open('logfile', 'a') as file:
            with open('logerr', 'a') as errfile:
                for op in oploop:
                    lang = op['lang']
                    cache_opt = op['cache_opt']
                    shared_mem = op['smem']
                    __check_exit(create_jacobian(lang=lang,
                                    mech_name=mech,
                                    therm_name=thermo,
                                    initial_state=initial_conditions,
                                    optimize_cache=cache_opt,
                                    multi_thread=num_threads,
                                    no_shared=shared_mem,
                                    build_path=build_path))
                    file.write('\ncache_opt: {}\n'
                               'shared_mem: {}\n'.format(
                               cache_opt, not shared_mem))
                    for j in range(-4, -13, -1):
                        t_step = np.power(10.0, j)
                        #build the validation set for this timestep
                        extra_args = ['t_step={:.0e}'.format(t_step), 't_end={:.0e}'.format(t_step)]
                        subprocess.check_call([scons, 'cpu'] + arg_list + extra_args + small_tol, stdout=errfile)
                        #run
                        subprocess.check_call([pjoin(cwd(), valid_int),
                            str(num_threads), str(num_conditions)],
                            stdout=errfile)
                        #copy to saved data
                        shutil.copy(pjoin(cwd(), 'log', keyfile),
                                    pjoin(cwd(), 'log', 'valid.bin'))

                        validator = np.fromfile(pjoin('log', 'valid.bin'), dtype='float64')
                        validator = validator.reshape((-1, 1 + num_conditions * nvar))

                        file.write('t_step={:.0e}\n'.format(t_step))
                        file.flush()
                        subprocess.check_call([scons, builder[lang]] + arg_list + extra_args + large_tol,
                            stdout=errfile, stderr=errfile)
                        __execute(builder[lang], num_threads, num_conditions, t_step)
                        __check_error(builder[lang], num_conditions, nvar,
                                        validator, atol, rtol)

def run_log(mech, thermo, initial_conditions, build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol, small_atol, small_rtol):
    with open('logfile', 'w') as file:
        pass
    with open('logerr', 'w') as file:
        pass
    if initial_conditions is not None:
        with open('logfile', 'a') as file:
            file.write('Running Same ICs\n')
        __run_and_check(mech, thermo, initial_conditions, build_path,
            num_threads, 1 if num_conditions is None else num_conditions,
            None, skip_c, skip_cuda, atol, rtol,
            small_atol, small_rtol)
    if test_data is not None:
        with open('logfile', 'a') as file:
            file.write('PaSR ICs\n')
        try:
            shutil.copyfile(test_data, 'ign_data.bin')
        except shutil.Error:
            pass
        __run_and_check(mech, thermo, '', build_path,
        num_threads, num_conditions, test_data, skip_c, skip_cuda,
        atol, rtol, small_atol, small_rtol)

if __name__ == '__main__':
    parser = ArgumentParser(description='logger: Log and compare solver output for the various ODE Solvers')

    parser.add_argument('-i', '--input',
                        type=str,
                        required=True,
                        help='The mechanism to check.')
    parser.add_argument('-t', '--thermo',
                    type=str,
                    default=None,
                    help='Thermodynamic database filename (e.g., '
                         'therm.dat), or nothing if in mechanism.')
    parser.add_argument('-ic', '--initial_conditions',
                        type=str,
                        required=False,
                        default=None,
                        help='The initial conditions for same initial condition testing. '
                             'If not supplied, this is skipped.')
    parser.add_argument('-b', '--build_path',
                        type=str,
                        required=False,
                        default=pjoin(cwd(),'out') + sep,
                        help='The folder to generate the mechanism in')
    parser.add_argument('-nt', '--num_threads',
                        type=int,
                        required=False,
                        default=multiprocessing.cpu_count(),
                        help='The number of threads to run the CPU ode solvers with.')
    parser.add_argument('-nc', '--num_conditions',
                        type=int,
                        required=False,
                        default=None,
                        help='The number of conditions to test')
    parser.add_argument('-td', '--test_data',
                        type=str,
                        required=False,
                        default=None,
                        help='A numpy file output generated from the partially_stirred_reactor component '
                             'of pyJac.  Used for testing if supplied.')
    parser.add_argument('-sc', '--skip_c',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip C testing. Note baseline is still calculated using an CPU integrator')
    parser.add_argument('-scu', '--skip_cuda',
                        required=False,
                        default=False,
                        action='store_true',
                        help='Use to skip CUDA testing.')
    parser.add_argument('-atol', '--abs_tolerance',
                        required=False,
                        type=float,
                        default=1e-10,
                        help='The absolute tolerance to use during integration')
    parser.add_argument('-rtol', '--rel_tolerance',
                        required=False,
                        type=float,
                        default=1e-7,
                        help='The relative tolerance to use during integration')
    parser.add_argument('-satol', '--abs_tolerance_small',
                        required=False,
                        type=float,
                        default=1e-20,
                        help='The absolute tolerance to use during integration')
    parser.add_argument('-srtol', '--rel_tolerance_small',
                        required=False,
                        type=float,
                        default=1e-15,
                        help='The relative tolerance to use during integration')
    args = parser.parse_args()

    assert not (args.test_data is None and args.initial_conditions is None), \
    "Either a test data file or initial conditions must be specified"

    create_dir(args.build_path)
    create_dir('./log/')

    run_log(args.input, args.thermo, args.initial_conditions, args.build_path,
        args.num_threads, args.num_conditions, args.test_data,
        args.skip_c, args.skip_cuda, args.abs_tolerance, args.rel_tolerance,
        args.abs_tolerance_small, args.rel_tolerance_small)<|endoftext|>


<commit_before>from handlers.simplehandler import Handler

_BAD_MEMBERS = {
    'title': 'Неправильные члены мультиполигона',
    'help_text': """""",
}

class MultipolygonChecker(Handler):
    def __init__(self):
        self._bad_members = []

    def process_iteration(self, obj, iteration):
        if iteration == 0:
            self.first_iteration(obj)

    def first_iteration(self, obj):
        if obj['@type'] == 'relation' and obj.get('type') == 'multipolygon':
            valid = True
            for m in obj['@members']:
                if not (m['type'] == 'way' and m['role'] in {'inner', 'outer'}):
                    valid = False
                    # Exceptions
                    # FIXME: type=boundary?
                    if m['type'] == 'node' and m['role'] == 'label':
                        valid = True
                    # End Exceptions
                    if not valid:
                        break
            if not valid:
                self._bad_members.append(obj['@id'])

    def is_iteration_required(self, iteration):
        if iteration == 0:
            return True
        return False

    def finish(self, issues):
        issues.add_issue_type('errors/relation/multipolygon/bad_members', _BAD_MEMBERS)
        for relation_id in self._bad_members:
            issues.add_issue_obj('errors/relation/multipolygon/bad_members', 'relation', relation_id)
<commit_msg>add simple geometry check for relation type=multipolygon<commit_after>from handlers.simplehandler import Handler

# To debug possibly bad relation download osm data:
# https://www.openstreetmap.org/api/0.6/relation/{relation_id}/full

_BAD_MEMBERS = {
    'title': 'Неправильные члены отношения',
    'help_text': """""",
}

_BAD_GEOMETRY = {
    'title': 'Неправильная геометрия отношения',
    'help_text': """""",
}

def check_multipolygon(obj, ways):
    outer_pairs = []
    inner_pairs = []
    for m in obj['@members']:
        if m['ref'] not in ways:
            return True
        nodes = ways[m['ref']]
        if m['role'] == 'outer':
            outer_pairs.append((nodes[0], nodes[-1]))
        if m['role'] == 'inner':
            inner_pairs.append((nodes[0], nodes[-1]))
    return _simplify_pairs(inner_pairs) and _simplify_pairs(outer_pairs)

def _simplify_pairs(pairs_list):
    p = None
    changed = True
    while pairs_list and changed:
        changed = False
        if p is None:
            tmp = pairs_list.pop()
            p = [tmp[0], tmp[1]]

        not_used = []
        for p2 in reversed(pairs_list):
            used = False
            if p[0] == p2[0]:
                p[0] = p2[1]
                used = True
            elif p[0] == p2[1]:
                p[0] = p2[0]
                used = True
            elif p[1] == p2[0]:
                p[1] = p2[1]
                used = True
            elif p[1] == p2[1]:
                p[1] = p2[0]
                used = True
            if not used:
                not_used.append(p2)
            else:
                changed = True
        if p[0] == p[1]:
            p = None
            changed = True
        pairs_list = not_used
    return (p is None) and (not bool(pairs_list))

class MultipolygonChecker(Handler):
    def __init__(self):
        self._resolve_ways = set()
        self._ways = dict()
        self._bad_members = set()
        self._bad_geometry = []

    def process_iteration(self, obj, iteration):
        if iteration == 0:
            self.first_iteration(obj)
        elif iteration == 1:
            self.second_iteration(obj)
        elif iteration == 2:
            self.third_iteration(obj)

    def first_iteration(self, obj):
        if obj['@type'] == 'relation' and obj.get('type') == 'multipolygon':
            valid = True
            for m in obj['@members']:
                if m['type'] == 'way' and m['role'] in {'inner', 'outer'}:
                    self._resolve_ways.add(m['ref'])
                else:
                    valid = False
                    # Exceptions
                    if m['type'] == 'node' and m['role'] == 'label': # type=boundary?
                        valid = True
                    if not valid:
                        break
            if not valid:
                self._bad_members.add(obj['@id'])

    def second_iteration(self, obj):
        if obj['@type'] == 'way' and obj['@id'] in self._resolve_ways:
            self._ways[obj['@id']] = obj['@nodes']

    def third_iteration(self, obj):
        if obj['@type'] == 'relation' and obj.get('type') == 'multipolygon' and obj['@id'] not in self._bad_members:
            valid = check_multipolygon(obj, self._ways)
            if not valid:
                self._bad_geometry.append(obj['@id'])

    def is_iteration_required(self, iteration):
        if iteration == 0:
            return True
        elif 1 <= iteration <= 2:
            return bool(self._resolve_ways)
        else:
            return False

    def finish(self, issues):
        issues.add_issue_type('errors/relation/multipolygon/bad_members', _BAD_MEMBERS)
        for relation_id in self._bad_members:
            issues.add_issue_obj('errors/relation/multipolygon/bad_members', 'relation', relation_id)

        issues.add_issue_type('errors/relation/multipolygon/bad_geometry', _BAD_GEOMETRY)
        for relation_id in self._bad_geometry:
            issues.add_issue_obj('errors/relation/multipolygon/bad_geometry', 'relation', relation_id)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Paper oriented operations."""

import glob
import subprocess
import re
import os
import shutil
import platform

import pymmd

import scriptorium

def paper_root(dname):
    """Given a directory, finds the root document for the paper."""
    root_doc = None
    for fname in glob.glob(os.path.join(dname, '*.mmd')):
        #Template metadata only exists in root
        if get_template(fname):
            root_doc = fname
            break

    return os.path.basename(root_doc) if root_doc else None

def _get_template(txt):
    """Extract template name from string containing document text"""
    template = pymmd.value(txt, 'latexfooter', pymmd.COMPLETE)
    template_re = re.compile(r'(?P<template>[a-zA-Z0-9._]*)\/footer.tex')

    match = template_re.search(template)

    return match.group('template') if match else None

def get_template(fname):
    """Attempts to find the template of a paper in a given file."""
    with open(fname, 'r') as mmd_fp:
        return _get_template(mmd_fp.read())

def to_pdf(paper_dir, template_dir=None, use_shell_escape=False):
    """Build paper in the given directory, returning the PDF filename if successful."""
    template_dir = template_dir or scriptorium.TEMPLATE_DIR
    paper_dir = os.path.abspath(paper_dir)
    if not os.path.isdir(paper_dir):
        raise IOError("{0} is not a valid directory".format(paper_dir))
    old_cwd = os.getcwd()
    if old_cwd != paper_dir:
        os.chdir(paper_dir)

    fname = paper_root('.')

    if not fname:
        raise IOError("{0} has no obvious root.".format(paper_dir))

    #Convert all auxillary MMD files to LaTeX
    for mmd in glob.glob('*.mmd'):
        bname = os.path.basename(mmd).split('.')[0]
        tname = '{0}.tex'.format(bname)
        with open(mmd, 'r') as mmd_fp, open(tname, 'w') as tex_fp:
            txt = pymmd.convert(mmd_fp.read(), fmt=pymmd.LATEX, dname=mmd)
            tex_fp.write(txt)

    bname = os.path.basename(fname).split('.')[0]
    tname = '{0}.tex'.format(bname)

    template = get_template(fname)
    if not template:
        raise IOError('{0} does not appear to have lines necessary to load a template.'.format(fname))

    template_loc = scriptorium.find_template(template, template_dir)

    if not template_loc:
        raise IOError('{0} template not installed in {1}'.format(template, template_dir))

    template_loc = os.path.abspath(os.path.join(template_loc, '..'))

    #Need to set up environment here
    new_env = dict(os.environ)
    old_inputs = new_env.get('TEXINPUTS')
    texinputs = './:{0}:{1}'.format(template_loc + '/.//', old_inputs + ':' if old_inputs else '')
    new_env['TEXINPUTS'] = texinputs

    pdf_cmd = ['pdflatex', '-halt-on-error', '-interaction=nonstopmode', tname]

    if platform.system() == 'Windows':
        pdf_cmd.insert(-2, '-include-directory={0}'.format(template_loc))

    if use_shell_escape:
        pdf_cmd.insert(1, '-shell-escape')
    try:
        subprocess.check_output(pdf_cmd, env=new_env, universal_newlines=True).encode('utf-8')
    except subprocess.CalledProcessError as exc:
        raise IOError(exc.output)

    try:
        auxname = '{0}.aux'.format(bname)
        #Check if bibtex is defined in the frontmatter
        bibtex_re = re.compile(r'^bibtex:', re.MULTILINE)
        with open(fname, 'r') as paper_fp:
            if bibtex_re.search(paper_fp.read()):
                biber_re = re.compile(r'\\bibdata', re.MULTILINE)
                full = open('paper.aux').read()
                if biber_re.search(full):
                    subprocess.check_output(['bibtex', auxname], universal_newlines=True)
                else:
                    subprocess.check_output(['biber', bname], universal_newlines=True)

                    subprocess.check_output(pdf_cmd, env=new_env, universal_newlines=True)
                    subprocess.check_output(pdf_cmd, env=new_env, universal_newlines=True)
    except subprocess.CalledProcessError as exc:
        raise IOError(exc.output)

    # Revert working directory
    if os.getcwd() != old_cwd:
        os.chdir(old_cwd)

    return os.path.join(paper_dir, '{0}.pdf'.format(bname))

def create(paper_dir, template, force=False, use_git=True, config=None):
    """Create folder with paper skeleton.
    Returns a list of unpopulated variables if successfully created.
    """
    if os.path.exists(paper_dir) and not force:
        raise IOError('{0} exists'.format(paper_dir))

    template_dir = scriptorium.find_template(template, scriptorium.TEMPLATE_DIR)

    os.makedirs(paper_dir)
    if use_git and not os.path.exists(os.path.join(paper_dir, '.gitignore')):
        shutil.copyfile(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data',
                                     'gitignore'),
                        os.path.join(paper_dir, '.gitignore'))

    files = {'paper.mmd': 'frontmatter.mmd',
             'metadata.tex': 'metadata.tex'}
    texts = {}
    for ofile, ifile in files.items():
        ifile = os.path.join(template_dir, ifile)
        try:
            with open(ifile, 'r') as ifp:
                texts[ofile] = ifp.read()
        except IOError:
            texts[ofile] = ''

    #Inject template as macro argument
    config['TEMPLATE'] = template
    #One line regex thanks to http://stackoverflow.com/a/6117124/59184
    for ofile, text in texts.items():
        texts[ofile] = re.sub("|".join([r'\${0}'.format(ii) for ii in config]),
                              lambda m: config[m.group(0)[1:]], text)

    #Regex to find variable names
    var_re = re.compile(r'\$[A-Z0-9_\.\-]+')
    unset_vars = set()
    for ofile, text in texts.items():
        unset_vars |= set([ii.group(0) for ii in var_re.finditer(text)])
        with open(os.path.join(paper_dir, ofile), 'w') as ofp:
            ofp.write(text)

    return unset_vars
<commit_msg>Rebuild PDF to incorporate bibliography data<commit_after>#!/usr/bin/env python
"""Paper oriented operations."""

import glob
import subprocess
import re
import os
import shutil
import platform

import pymmd

import scriptorium

def paper_root(dname):
    """Given a directory, finds the root document for the paper."""
    root_doc = None
    for fname in glob.glob(os.path.join(dname, '*.mmd')):
        #Template metadata only exists in root
        if get_template(fname):
            root_doc = fname
            break

    return os.path.basename(root_doc) if root_doc else None

def _get_template(txt):
    """Extract template name from string containing document text"""
    template = pymmd.value(txt, 'latexfooter', pymmd.COMPLETE)
    template_re = re.compile(r'(?P<template>[a-zA-Z0-9._]*)\/footer.tex')

    match = template_re.search(template)

    return match.group('template') if match else None

def get_template(fname):
    """Attempts to find the template of a paper in a given file."""
    with open(fname, 'r') as mmd_fp:
        return _get_template(mmd_fp.read())

def to_pdf(paper_dir, template_dir=None, use_shell_escape=False):
    """Build paper in the given directory, returning the PDF filename if successful."""
    template_dir = template_dir or scriptorium.TEMPLATE_DIR
    paper_dir = os.path.abspath(paper_dir)
    if not os.path.isdir(paper_dir):
        raise IOError("{0} is not a valid directory".format(paper_dir))
    old_cwd = os.getcwd()
    if old_cwd != paper_dir:
        os.chdir(paper_dir)

    fname = paper_root('.')

    if not fname:
        raise IOError("{0} has no obvious root.".format(paper_dir))

    #Convert all auxillary MMD files to LaTeX
    for mmd in glob.glob('*.mmd'):
        bname = os.path.basename(mmd).split('.')[0]
        tname = '{0}.tex'.format(bname)
        with open(mmd, 'r') as mmd_fp, open(tname, 'w') as tex_fp:
            txt = pymmd.convert(mmd_fp.read(), fmt=pymmd.LATEX, dname=mmd)
            tex_fp.write(txt)

    bname = os.path.basename(fname).split('.')[0]
    tname = '{0}.tex'.format(bname)

    template = get_template(fname)
    if not template:
        raise IOError('{0} does not appear to have lines necessary to load a template.'.format(fname))

    template_loc = scriptorium.find_template(template, template_dir)

    if not template_loc:
        raise IOError('{0} template not installed in {1}'.format(template, template_dir))

    template_loc = os.path.abspath(os.path.join(template_loc, '..'))

    #Need to set up environment here
    new_env = dict(os.environ)
    old_inputs = new_env.get('TEXINPUTS')
    texinputs = './:{0}:{1}'.format(template_loc + '/.//', old_inputs + ':' if old_inputs else '')
    new_env['TEXINPUTS'] = texinputs

    pdf_cmd = ['pdflatex', '-halt-on-error', '-interaction=nonstopmode', tname]

    if platform.system() == 'Windows':
        pdf_cmd.insert(-2, '-include-directory={0}'.format(template_loc))

    if use_shell_escape:
        pdf_cmd.insert(1, '-shell-escape')
    try:
        subprocess.check_output(pdf_cmd, env=new_env, universal_newlines=True).encode('utf-8')
    except subprocess.CalledProcessError as exc:
        raise IOError(exc.output)

    try:
        auxname = '{0}.aux'.format(bname)
        #Check if bibtex is defined in the frontmatter
        bibtex_re = re.compile(r'^bibtex:', re.MULTILINE)
        with open(fname, 'r') as paper_fp:
            if bibtex_re.search(paper_fp.read()):
                biber_re = re.compile(r'\\bibdata', re.MULTILINE)
                full = open('paper.aux').read()
                if biber_re.search(full):
                    subprocess.check_output(['bibtex', auxname], universal_newlines=True)
                else:
                    subprocess.check_output(['biber', bname], universal_newlines=True)

                subprocess.check_output(pdf_cmd, env=new_env, universal_newlines=True)
                subprocess.check_output(pdf_cmd, env=new_env, universal_newlines=True)
    except subprocess.CalledProcessError as exc:
        raise IOError(exc.output)

    # Revert working directory
    if os.getcwd() != old_cwd:
        os.chdir(old_cwd)

    return os.path.join(paper_dir, '{0}.pdf'.format(bname))

def create(paper_dir, template, force=False, use_git=True, config=None):
    """Create folder with paper skeleton.
    Returns a list of unpopulated variables if successfully created.
    """
    if os.path.exists(paper_dir) and not force:
        raise IOError('{0} exists'.format(paper_dir))

    template_dir = scriptorium.find_template(template, scriptorium.TEMPLATE_DIR)

    os.makedirs(paper_dir)
    if use_git and not os.path.exists(os.path.join(paper_dir, '.gitignore')):
        shutil.copyfile(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data',
                                     'gitignore'),
                        os.path.join(paper_dir, '.gitignore'))

    files = {'paper.mmd': 'frontmatter.mmd',
             'metadata.tex': 'metadata.tex'}
    texts = {}
    for ofile, ifile in files.items():
        ifile = os.path.join(template_dir, ifile)
        try:
            with open(ifile, 'r') as ifp:
                texts[ofile] = ifp.read()
        except IOError:
            texts[ofile] = ''

    #Inject template as macro argument
    config['TEMPLATE'] = template
    #One line regex thanks to http://stackoverflow.com/a/6117124/59184
    for ofile, text in texts.items():
        texts[ofile] = re.sub("|".join([r'\${0}'.format(ii) for ii in config]),
                              lambda m: config[m.group(0)[1:]], text)

    #Regex to find variable names
    var_re = re.compile(r'\$[A-Z0-9_\.\-]+')
    unset_vars = set()
    for ofile, text in texts.items():
        unset_vars |= set([ii.group(0) for ii in var_re.finditer(text)])
        with open(os.path.join(paper_dir, ofile), 'w') as ofp:
            ofp.write(text)

    return unset_vars
<|endoftext|>


<commit_before>import smbus

class TemperatureSensor:
    def __init__(self, address):
        self.bus = smbus.SMBus(1)
        self.address = address

    def get_temp(self):
        MSB = self.bus.read_byte_data(self.address, 0)
        LSB = self.bus.read_byte_data(self.address, 1)

        return ((MSB << 8 | LSB) >> 4) * 0.0625<commit_msg>Smooth Temp Sensor Variability<commit_after>import smbus

class TemperatureSensor:
    temp_history = []

    def __init__(self, address):
        self.bus = smbus.SMBus(1)
        self.address = address

    def get_temp(self):
        MSB = self.bus.read_byte_data(self.address, 0)
        LSB = self.bus.read_byte_data(self.address, 1)

        temp = ((MSB << 8 | LSB) >> 4) * 0.0625
        result = temp

        # smooth the data slightly
        if len(self.temp_history) > 0:
            if self.temp_history[0] - temp < 0.3:
                if len(self.temp_history) > 1:
                    if self.temp_history[1] - temp < 0.3:
                        if len(self.temp_history) > 2:
                            if not self.temp_history[2] - temp < 0.3:
                                result = self.temp_history[2]
                    else:
                        result = self.temp_history[0]
            else:
                result = self.temp_history[0]

        self.temp_history.append(temp)
        self.temp_history = self.temp_history[0:3]

        return result<|endoftext|>


<commit_before>"""
=======================================
Visualizing the stock market structure
=======================================

This example employs several unsupervised learning techniques to extract
the stock market structure from variations in historical quotes.

The quantity that we use is the daily variation in quote price: quotes
that are linked tend to cofluctuate during a day.

.. _stock_market:

Learning a graph structure
--------------------------

We use sparse inverse covariance estimation to find which quotes are
correlated conditionally on the others. Specifically, sparse inverse
covariance gives us a graph, that is a list of connection. For each
symbol, the symbols that it is connected too are those useful to explain
its fluctuations.

Clustering
----------

We use clustering to group together quotes that behave similarly. Here,
amongst the :ref:`various clustering techniques <clustering>` available
in the scikit-learn, we use :ref:`affinity_propagation` as it does
not enforce equal-size clusters, and it can choose automatically the
number of clusters from the data.

Note that this gives us a different indication than the graph, as the
graph reflects conditional relations between variables, while the
clustering reflects marginal properties: variables clustered together can
be considered as having a similar impact at the level of the full stock
market.

Embedding in 2D space
---------------------

For visualization purposes, we need to lay out the different symbols on a
2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D
embedding.

Visualization
-------------

The output of the 3 models are combined in a 2D graph where nodes
represents the stocks and edges the:

- cluster labels are used to define the color of the nodes
- the sparse covariance model is used to display the strength of the edges
- the 2D embedding is used to position the nodes in the plan

This example has a fair amount of visualization-related code, as
visualization is crucial here to display the graph. One of the challenge
is to position the labels minimizing overlap. For this we use an
heuristic based on the direction of the nearest neighbor along each
axis.
"""
from __future__ import print_function

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause

import sys
from datetime import datetime

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection
from six.moves.urllib.request import urlopen
from six.moves.urllib.parse import urlencode
from sklearn import cluster, covariance, manifold

print(__doc__)

def retry(f, n_attempts=3):
    "Wrapper function to retry function calls in case of exceptions"
    def wrapper(*args, **kwargs):
        for i in range(n_attempts):
            try:
                return f(*args, **kwargs)
            except Exception:
                if i == n_attempts - 1:
                    raise
    return wrapper

def quotes_historical_google(symbol, start_date, end_date,
                             expected=None):
    """Get the historical data from Google finance.

    Parameters
    ----------
    symbol : str
        Ticker symbol to query for, for example ``"DELL"``.
    start_date : datetime.datetime
        Start date.
    end_date : datetime.datetime
        End date.

    Returns
    -------
    X : array
        The columns are ``date`` -- date, ``open``, ``high``,
        ``low``, ``close`` and ``volume`` of type float.
    """
    params = urlencode({
        'q': symbol,
        'startdate': start_date.strftime('%b %d, %Y'),
        'enddate': end_date.strftime('%b %d, %Y'),
        'output': 'csv'
    })
    url = 'http://www.google.com/finance/historical?' + params
    response = urlopen(url)
    dtype = {
        'names': ['date', 'open', 'high', 'low', 'close', 'volume'],
        'formats': ['object', 'f4', 'f4', 'f4', 'f4', 'f4']
    }
    converters = {
        0: lambda s: datetime.strptime(s.decode(), '%d-%b-%y').date()}
    data = np.genfromtxt(response, delimiter=',', skip_header=1,
                         dtype=dtype, converters=converters,
                         missing_values='-', filling_values=-1)
    if expected is not None:
        len_data = len(data)
        min_date = min(data['date'], default=None)
        max_date = max(data['date'], default=None)
        if (len_data != expected['len_data'] or
            min_date != expected['min_date'] or
                max_date != expected['max_date']):
            message = (
                'Got wrong data for symbol {}, url {}\n'
                '  - min_date should be {}, got {}\n'
                '  - max_date should be {}, got {}\n'
                '  - len(data) should be {}, got {}'.format(
                    symbol, url,
                    expected['min_date'], min_date,
                    expected['max_date'], max_date,
                    expected['len_data'], len_data))
            raise ValueError(message)
    return data

# #############################################################################
# Retrieve the data from Internet

# Choose a time period reasonably calm (not too long ago so that we get
# high-tech firms, and before the 2008 crash)
start_date = datetime(2003, 1, 2).date()
end_date = datetime(2007, 12, 31).date()

symbol_dict = {
    'NYSE:TOT': 'Total',
    'NYSE:XOM': 'Exxon',
    'NYSE:CVX': 'Chevron',
    'NYSE:COP': 'ConocoPhillips',
    'NYSE:VLO': 'Valero Energy',
    'NASDAQ:MSFT': 'Microsoft',
    'NYSE:IBM': 'IBM',
    'NYSE:TWX': 'Time Warner',
    'NASDAQ:CMCSA': 'Comcast',
    'NYSE:CVC': 'Cablevision',
    'NASDAQ:YHOO': 'Yahoo',
    'NASDAQ:DELL': 'Dell',
    'NYSE:HPQ': 'HP',
    'NASDAQ:AMZN': 'Amazon',
    'NYSE:TM': 'Toyota',
    'NYSE:CAJ': 'Canon',
    'NYSE:SNE': 'Sony',
    'NYSE:F': 'Ford',
    'NYSE:HMC': 'Honda',
    'NYSE:NAV': 'Navistar',
    'NYSE:NOC': 'Northrop Grumman',
    'NYSE:BA': 'Boeing',
    'NYSE:KO': 'Coca Cola',
    'NYSE:MMM': '3M',
    'NYSE:MCD': 'McDonald\'s',
    'NYSE:PEP': 'Pepsi',
    'NYSE:K': 'Kellogg',
    'NYSE:UN': 'Unilever',
    'NASDAQ:MAR': 'Marriott',
    'NYSE:PG': 'Procter Gamble',
    'NYSE:CL': 'Colgate-Palmolive',
    'NYSE:GE': 'General Electrics',
    'NYSE:WFC': 'Wells Fargo',
    'NYSE:JPM': 'JPMorgan Chase',
    'NYSE:AIG': 'AIG',
    'NYSE:AXP': 'American express',
    'NYSE:BAC': 'Bank of America',
    'NYSE:GS': 'Goldman Sachs',
    'NASDAQ:AAPL': 'Apple',
    'NYSE:SAP': 'SAP',
    'NASDAQ:CSCO': 'Cisco',
    'NASDAQ:TXN': 'Texas Instruments',
    'NYSE:XRX': 'Xerox',
    'NYSE:WMT': 'Wal-Mart',
    'NYSE:HD': 'Home Depot',
    'NYSE:GSK': 'GlaxoSmithKline',
    'NYSE:PFE': 'Pfizer',
    'NYSE:SNY': 'Sanofi-Aventis',
    'NYSE:NVS': 'Novartis',
    'NYSE:KMB': 'Kimberly-Clark',
    'NYSE:R': 'Ryder',
    'NYSE:GD': 'General Dynamics',
    'NYSE:RTN': 'Raytheon',
    'NYSE:CVS': 'CVS',
    'NYSE:CAT': 'Caterpillar',
    'NYSE:DD': 'DuPont de Nemours'}

symbols, names = np.array(sorted(symbol_dict.items())).T

# retry is used because quotes_historical_google can temporarily fail
# for various reasons (e.g. empty result from Google API).
quotes = []
# expected min_date, max_date and length for each stock timeseries
expected = {'min_date': start_date, 'max_date': end_date, 'len_data': 1258}

for symbol in symbols:
    print('Fetching quote history for %r' % symbol, file=sys.stderr)
    quotes.append(retry(quotes_historical_google)(symbol, start_date, end_date,
                                                  expected=expected))

close_prices = np.vstack([q['close'] for q in quotes])
open_prices = np.vstack([q['open'] for q in quotes])

# The daily variations of the quotes are what carry most information
variation = close_prices - open_prices

# #############################################################################
# Learn a graphical structure from the correlations
edge_model = covariance.GraphLassoCV()

# standardize the time series: using correlations rather than covariance
# is more efficient for structure recovery
X = variation.copy().T
X /= X.std(axis=0)
edge_model.fit(X)

# #############################################################################
# Cluster using affinity propagation

_, labels = cluster.affinity_propagation(edge_model.covariance_)
n_labels = labels.max()

for i in range(n_labels + 1):
    print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i])))

# #############################################################################
# Find a low-dimension embedding for visualization: find the best position of
# the nodes (the stocks) on a 2D plane

# We use a dense eigen_solver to achieve reproducibility (arpack is
# initiated with random vectors that we don't control). In addition, we
# use a large number of neighbors to capture the large-scale structure.
node_position_model = manifold.LocallyLinearEmbedding(
    n_components=2, eigen_solver='dense', n_neighbors=6)

embedding = node_position_model.fit_transform(X.T).T

# #############################################################################
# Visualization
plt.figure(1, facecolor='w', figsize=(10, 8))
plt.clf()
ax = plt.axes([0., 0., 1., 1.])
plt.axis('off')

# Display a graph of the partial correlations
partial_correlations = edge_model.precision_.copy()
d = 1 / np.sqrt(np.diag(partial_correlations))
partial_correlations *= d
partial_correlations *= d[:, np.newaxis]
non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02)

# Plot the nodes using the coordinates of our embedding
plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels,
            cmap=plt.cm.spectral)

# Plot the edges
start_idx, end_idx = np.where(non_zero)
# a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[embedding[:, start], embedding[:, stop]]
            for start, stop in zip(start_idx, end_idx)]
values = np.abs(partial_correlations[non_zero])
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.hot_r,
                    norm=plt.Normalize(0, .7 * values.max()))
lc.set_array(values)
lc.set_linewidths(15 * values)
ax.add_collection(lc)

# Add a label to each node. The challenge here is that we want to
# position the labels to avoid overlap with other labels
for index, (name, label, (x, y)) in enumerate(
        zip(names, labels, embedding.T)):

    dx = x - embedding[0]
    dx[index] = 1
    dy = y - embedding[1]
    dy[index] = 1
    this_dx = dx[np.argmin(np.abs(dy))]
    this_dy = dy[np.argmin(np.abs(dx))]
    if this_dx > 0:
        horizontalalignment = 'left'
        x = x + .002
    else:
        horizontalalignment = 'right'
        x = x - .002
    if this_dy > 0:
        verticalalignment = 'bottom'
        y = y + .002
    else:
        verticalalignment = 'top'
        y = y - .002
    plt.text(x, y, name, size=10,
             horizontalalignment=horizontalalignment,
             verticalalignment=verticalalignment,
             bbox=dict(facecolor='w',
                       edgecolor=plt.cm.spectral(label / float(n_labels)),
                       alpha=.6))

plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(),
         embedding[0].max() + .10 * embedding[0].ptp(),)
plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(),
         embedding[1].max() + .03 * embedding[1].ptp())

plt.show()
<commit_msg>FIX? Use ISO8601 dates and resolved URL for Google Finance<commit_after>"""
=======================================
Visualizing the stock market structure
=======================================

This example employs several unsupervised learning techniques to extract
the stock market structure from variations in historical quotes.

The quantity that we use is the daily variation in quote price: quotes
that are linked tend to cofluctuate during a day.

.. _stock_market:

Learning a graph structure
--------------------------

We use sparse inverse covariance estimation to find which quotes are
correlated conditionally on the others. Specifically, sparse inverse
covariance gives us a graph, that is a list of connection. For each
symbol, the symbols that it is connected too are those useful to explain
its fluctuations.

Clustering
----------

We use clustering to group together quotes that behave similarly. Here,
amongst the :ref:`various clustering techniques <clustering>` available
in the scikit-learn, we use :ref:`affinity_propagation` as it does
not enforce equal-size clusters, and it can choose automatically the
number of clusters from the data.

Note that this gives us a different indication than the graph, as the
graph reflects conditional relations between variables, while the
clustering reflects marginal properties: variables clustered together can
be considered as having a similar impact at the level of the full stock
market.

Embedding in 2D space
---------------------

For visualization purposes, we need to lay out the different symbols on a
2D canvas. For this we use :ref:`manifold` techniques to retrieve 2D
embedding.

Visualization
-------------

The output of the 3 models are combined in a 2D graph where nodes
represents the stocks and edges the:

- cluster labels are used to define the color of the nodes
- the sparse covariance model is used to display the strength of the edges
- the 2D embedding is used to position the nodes in the plan

This example has a fair amount of visualization-related code, as
visualization is crucial here to display the graph. One of the challenge
is to position the labels minimizing overlap. For this we use an
heuristic based on the direction of the nearest neighbor along each
axis.
"""
from __future__ import print_function

# Author: Gael Varoquaux gael.varoquaux@normalesup.org
# License: BSD 3 clause

import sys
from datetime import datetime

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection
from six.moves.urllib.request import urlopen
from six.moves.urllib.parse import urlencode
from sklearn import cluster, covariance, manifold

print(__doc__)

def retry(f, n_attempts=3):
    "Wrapper function to retry function calls in case of exceptions"
    def wrapper(*args, **kwargs):
        for i in range(n_attempts):
            try:
                return f(*args, **kwargs)
            except Exception:
                if i == n_attempts - 1:
                    raise
    return wrapper

def quotes_historical_google(symbol, start_date, end_date,
                             expected=None):
    """Get the historical data from Google finance.

    Parameters
    ----------
    symbol : str
        Ticker symbol to query for, for example ``"DELL"``.
    start_date : datetime.datetime
        Start date.
    end_date : datetime.datetime
        End date.

    Returns
    -------
    X : array
        The columns are ``date`` -- date, ``open``, ``high``,
        ``low``, ``close`` and ``volume`` of type float.
    """
    params = {
        'q': symbol,
        'startdate': start_date.strftime('%Y-%m-%d'),
        'enddate': end_date.strftime('%Y-%m-%d'),
        'output': 'csv',
    }
    url = 'https://finance.google.com/finance/historical?' + urlencode(params)
    response = urlopen(url)
    dtype = {
        'names': ['date', 'open', 'high', 'low', 'close', 'volume'],
        'formats': ['object', 'f4', 'f4', 'f4', 'f4', 'f4']
    }
    converters = {
        0: lambda s: datetime.strptime(s.decode(), '%d-%b-%y').date()}
    data = np.genfromtxt(response, delimiter=',', skip_header=1,
                         dtype=dtype, converters=converters,
                         missing_values='-', filling_values=-1)
    if expected is not None:
        len_data = len(data)
        min_date = min(data['date'], default=None)
        max_date = max(data['date'], default=None)
        if (len_data != expected['len_data'] or
            min_date != expected['min_date'] or
                max_date != expected['max_date']):
            message = (
                'Got wrong data for symbol {}, url {}\n'
                '  - min_date should be {}, got {}\n'
                '  - max_date should be {}, got {}\n'
                '  - len(data) should be {}, got {}'.format(
                    symbol, url,
                    expected['min_date'], min_date,
                    expected['max_date'], max_date,
                    expected['len_data'], len_data))
            raise ValueError(message)
    return data

# #############################################################################
# Retrieve the data from Internet

# Choose a time period reasonably calm (not too long ago so that we get
# high-tech firms, and before the 2008 crash)
start_date = datetime(2003, 1, 2).date()
end_date = datetime(2007, 12, 31).date()

symbol_dict = {
    'NYSE:TOT': 'Total',
    'NYSE:XOM': 'Exxon',
    'NYSE:CVX': 'Chevron',
    'NYSE:COP': 'ConocoPhillips',
    'NYSE:VLO': 'Valero Energy',
    'NASDAQ:MSFT': 'Microsoft',
    'NYSE:IBM': 'IBM',
    'NYSE:TWX': 'Time Warner',
    'NASDAQ:CMCSA': 'Comcast',
    'NYSE:CVC': 'Cablevision',
    'NASDAQ:YHOO': 'Yahoo',
    'NASDAQ:DELL': 'Dell',
    'NYSE:HPQ': 'HP',
    'NASDAQ:AMZN': 'Amazon',
    'NYSE:TM': 'Toyota',
    'NYSE:CAJ': 'Canon',
    'NYSE:SNE': 'Sony',
    'NYSE:F': 'Ford',
    'NYSE:HMC': 'Honda',
    'NYSE:NAV': 'Navistar',
    'NYSE:NOC': 'Northrop Grumman',
    'NYSE:BA': 'Boeing',
    'NYSE:KO': 'Coca Cola',
    'NYSE:MMM': '3M',
    'NYSE:MCD': 'McDonald\'s',
    'NYSE:PEP': 'Pepsi',
    'NYSE:K': 'Kellogg',
    'NYSE:UN': 'Unilever',
    'NASDAQ:MAR': 'Marriott',
    'NYSE:PG': 'Procter Gamble',
    'NYSE:CL': 'Colgate-Palmolive',
    'NYSE:GE': 'General Electrics',
    'NYSE:WFC': 'Wells Fargo',
    'NYSE:JPM': 'JPMorgan Chase',
    'NYSE:AIG': 'AIG',
    'NYSE:AXP': 'American express',
    'NYSE:BAC': 'Bank of America',
    'NYSE:GS': 'Goldman Sachs',
    'NASDAQ:AAPL': 'Apple',
    'NYSE:SAP': 'SAP',
    'NASDAQ:CSCO': 'Cisco',
    'NASDAQ:TXN': 'Texas Instruments',
    'NYSE:XRX': 'Xerox',
    'NYSE:WMT': 'Wal-Mart',
    'NYSE:HD': 'Home Depot',
    'NYSE:GSK': 'GlaxoSmithKline',
    'NYSE:PFE': 'Pfizer',
    'NYSE:SNY': 'Sanofi-Aventis',
    'NYSE:NVS': 'Novartis',
    'NYSE:KMB': 'Kimberly-Clark',
    'NYSE:R': 'Ryder',
    'NYSE:GD': 'General Dynamics',
    'NYSE:RTN': 'Raytheon',
    'NYSE:CVS': 'CVS',
    'NYSE:CAT': 'Caterpillar',
    'NYSE:DD': 'DuPont de Nemours'}

symbols, names = np.array(sorted(symbol_dict.items())).T

# retry is used because quotes_historical_google can temporarily fail
# for various reasons (e.g. empty result from Google API).
quotes = []
# expected min_date, max_date and length for each stock timeseries
expected = {'min_date': start_date, 'max_date': end_date, 'len_data': 1258}

for symbol in symbols:
    print('Fetching quote history for %r' % symbol, file=sys.stderr)
    quotes.append(retry(quotes_historical_google)(symbol, start_date, end_date,
                                                  expected=expected))

close_prices = np.vstack([q['close'] for q in quotes])
open_prices = np.vstack([q['open'] for q in quotes])

# The daily variations of the quotes are what carry most information
variation = close_prices - open_prices

# #############################################################################
# Learn a graphical structure from the correlations
edge_model = covariance.GraphLassoCV()

# standardize the time series: using correlations rather than covariance
# is more efficient for structure recovery
X = variation.copy().T
X /= X.std(axis=0)
edge_model.fit(X)

# #############################################################################
# Cluster using affinity propagation

_, labels = cluster.affinity_propagation(edge_model.covariance_)
n_labels = labels.max()

for i in range(n_labels + 1):
    print('Cluster %i: %s' % ((i + 1), ', '.join(names[labels == i])))

# #############################################################################
# Find a low-dimension embedding for visualization: find the best position of
# the nodes (the stocks) on a 2D plane

# We use a dense eigen_solver to achieve reproducibility (arpack is
# initiated with random vectors that we don't control). In addition, we
# use a large number of neighbors to capture the large-scale structure.
node_position_model = manifold.LocallyLinearEmbedding(
    n_components=2, eigen_solver='dense', n_neighbors=6)

embedding = node_position_model.fit_transform(X.T).T

# #############################################################################
# Visualization
plt.figure(1, facecolor='w', figsize=(10, 8))
plt.clf()
ax = plt.axes([0., 0., 1., 1.])
plt.axis('off')

# Display a graph of the partial correlations
partial_correlations = edge_model.precision_.copy()
d = 1 / np.sqrt(np.diag(partial_correlations))
partial_correlations *= d
partial_correlations *= d[:, np.newaxis]
non_zero = (np.abs(np.triu(partial_correlations, k=1)) > 0.02)

# Plot the nodes using the coordinates of our embedding
plt.scatter(embedding[0], embedding[1], s=100 * d ** 2, c=labels,
            cmap=plt.cm.spectral)

# Plot the edges
start_idx, end_idx = np.where(non_zero)
# a sequence of (*line0*, *line1*, *line2*), where::
#            linen = (x0, y0), (x1, y1), ... (xm, ym)
segments = [[embedding[:, start], embedding[:, stop]]
            for start, stop in zip(start_idx, end_idx)]
values = np.abs(partial_correlations[non_zero])
lc = LineCollection(segments,
                    zorder=0, cmap=plt.cm.hot_r,
                    norm=plt.Normalize(0, .7 * values.max()))
lc.set_array(values)
lc.set_linewidths(15 * values)
ax.add_collection(lc)

# Add a label to each node. The challenge here is that we want to
# position the labels to avoid overlap with other labels
for index, (name, label, (x, y)) in enumerate(
        zip(names, labels, embedding.T)):

    dx = x - embedding[0]
    dx[index] = 1
    dy = y - embedding[1]
    dy[index] = 1
    this_dx = dx[np.argmin(np.abs(dy))]
    this_dy = dy[np.argmin(np.abs(dx))]
    if this_dx > 0:
        horizontalalignment = 'left'
        x = x + .002
    else:
        horizontalalignment = 'right'
        x = x - .002
    if this_dy > 0:
        verticalalignment = 'bottom'
        y = y + .002
    else:
        verticalalignment = 'top'
        y = y - .002
    plt.text(x, y, name, size=10,
             horizontalalignment=horizontalalignment,
             verticalalignment=verticalalignment,
             bbox=dict(facecolor='w',
                       edgecolor=plt.cm.spectral(label / float(n_labels)),
                       alpha=.6))

plt.xlim(embedding[0].min() - .15 * embedding[0].ptp(),
         embedding[0].max() + .10 * embedding[0].ptp(),)
plt.ylim(embedding[1].min() - .03 * embedding[1].ptp(),
         embedding[1].max() + .03 * embedding[1].ptp())

plt.show()
<|endoftext|>


<commit_before>import csv

file_train = 'train.csv'
file_test = 'test.csv'
file_result = 'result.csv'

## learn
f = open(file_train, 'r')
reader = csv.reader(f)

count = [0] * 10
pixels = {}
for num in range(10):
    pixels[num] = [0] * 784

next(reader) # read header

# calculate average pixel values
for r in reader:
    cols = list(map(int, r[:]))
    number = cols[0]
    count[number] += 1
    for i in range(len(pixels[number])):
        pixels[number][i] += cols[1:][i]

for i in range(10):
    for j in range(784):
        pixels[i][j] = pixels[i][j] / count[i]

## predict
fp = open(file_test, 'r')
fresult = open(file_result, 'w')
readerp = csv.reader(fp)

next(readerp) # read header

fresult.write('ImageId,Label\n')

row=1
for r in readerp:
    test_pics = list(map(int, r))
    diffMin = 255*255*784+1
    diffMinIndex = -1
    for nm in range(len(pixels)): # 10
        diff = 0.0
        for pc in range(784):
            diff += (pixels[nm][pc]-test_pics[pc])*(pixels[nm][pc]-test_pics[pc])

        if diffMin > diff:
            diffMin = diff
            diffMinIndex = nm

    fresult.write(str(row)+","+str(diffMinIndex)+"\n")
    row += 1

fresult.close()
fp.close()
f.close()
<commit_msg>Using numpy and improve score a bit<commit_after>import numpy as np
import csv

file_train = 'train.csv'
file_test = 'test.csv'
file_result = 'result_np.csv'

## learn
f = open(file_train, 'r')
reader = csv.reader(f)

count = np.zeros(10)
pixels = np.zeros([10, 784])

next(reader) # read header

# calculate average pixel values
for r in reader:
    cols = list(map(int, r[:]))
    number = cols[0]
    count[number] += 1
    pixels[number] += cols[1:]

for i in range(10):
        pixels[i] = pixels[i] / count[i]

## predict
fp = open(file_test, 'r')
fresult = open(file_result, 'w')
readerp = csv.reader(fp)

next(readerp) # read header

fresult.write('ImageId,Label\n')

row=1
for r in readerp:
    test_pics = np.array(list(map(int, r)))
    pic = np.zeros([10, 784])
    for i in range(10):
        pic[i] = test_pics

    diffMin = 255*255*784+1
    diffMinIndex = -1
    # to avoid overflow, difference of pixel values are divided by 255
    arDiff = np.sum(((pixels-pic)/255.0)*((pixels-pic)/255.0)*((pixels-pic)/255.0)*((pixels-pic)/255.0), axis=1)
    for nm in range(len(pixels)): # 10
        diff = arDiff[nm]
        if diffMin > diff:
            diffMin = diff
            diffMinIndex = nm

    fresult.write(str(row)+","+str(diffMinIndex)+"\n")
    row += 1

fresult.close()
fp.close()
f.close()
<|endoftext|>