<commit_before>'''create charts showing median and mean prices each month

INVOCATION
  python chart-01.py [--data] [--test]

INPUT FILES
 INPUT/samples-train-validate.csv

OUTPUT FILES
 WORKING/chart-01/data.pickle
 WORKING/chart-01/txt
 WORKING/chart-01/pdf
'''

from __future__ import division

import collections
import cPickle as pickle
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pdb
from pprint import pprint
import random
import sys

from Bunch import Bunch
from columns_contain import columns_contain
from Logger import Logger
from ParseCommandLine import ParseCommandLine
from Path import Path
from Report import Report
import layout_transactions as t
cc = columns_contain

def usage(msg=None):
    print __doc__
    if msg is not None:
        print msg
    sys.exit(1)

def make_control(argv):
    # return a Bunch

    print argv
    if len(argv) not in (1, 2, 3):
        usage('invalid number of arguments')

    pcl = ParseCommandLine(argv)
    arg = Bunch(
        base_name='chart-01',
        data=pcl.has_arg('--data'),
        test=pcl.has_arg('--test'),
    )

    random_seed = 123
    random.seed(random_seed)

    dir_working = Path().dir_working()

    debug = False

    reduced_file_name = ('test-' if arg.test else '') + 'data.pickle'

    # assure output directory exists
    dir_path = dir_working + arg.base_name + '/'
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

    return Bunch(
        arg=arg,
        debug=debug,
        path_in_samples=dir_working + 'samples-train-validate.csv',
        path_out_pdf=dir_path + 'median-price.pdf',
        path_out_txt=dir_path + 'median-price.txt',
        path_out_txt_2006_2007=dir_path + 'median-price_2006_2007.txt',
        path_reduction=dir_path + reduced_file_name,
        random_seed=random_seed,
        test=arg.test,
    )

def key(year, month):
    return year * 100 + month

def make_chart_pdf(data, control):
    counts, means, medians, prices = data
    years = (2003, 2004, 2005, 2006, 2007, 2008, 2009)
    months = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
    months_2009 = (1, 2, 3)
    year_month = ['%s-%02d' % (year, month)
                  for year in years
                  for month in (months_2009 if year == 2009 else months)
                  ]
    x = range(len(year_month))
    y = []
    for year in years:
        for month in (months_2009 if year == 2009 else months):
            y.append(medians[key(year, month)])
    plt.plot(x, y)
    x_ticks = [year_month[i] if i % 12 == 0 else ' '
               for i in xrange(len(year_month))
               ]
    plt.xticks(range(len(year_month)),
               x_ticks,
               # pad=8,
               size='xx-small',
               rotation=-30,
               # rotation='vertical',
               )
    plt.yticks(size='xx-small')
    plt.xlabel('year-month')
    plt.ylabel('median price ($)')
    plt.ylim([0, 700000])
    plt.savefig(control.path_out_pdf)

    plt.close()

def make_chart_txt(data, control):
    counts, means, medians, prices = data
    r = Report()
    format_header = '%7s %7s %7s %7s %7s'
    format_detail = '%7d %7d %7.0f %7.0f %7d'
    r.append('Prices by Month')
    r.append('')
    r.append(format_header % (' ', ' ', 'mean', 'median', 'number'))
    r.append(format_header % ('year', 'month', 'price', 'price', 'trades'))
    for year in xrange(2003, 2010):
        for month in (xrange(1, 4) if year == 2009 else xrange(1, 13)):
            k = key(year, month)
            r.append(format_detail % (
                year,
                month,
                means[k],
                medians[k],
                counts[k],
            ))
    r.write(control.path_out_txt)

def make_chart_txt_2006_2007(data, control):
    counts, means, medians, prices = data
    r = Report()
    format_header = '%7s %7s %7s %7s'
    format_detail = '%7d %7d %7.0f %7d'
    r.append('Median Prices and Transaction Counts by Month')
    r.append('')
    r.append(format_header % (' ', ' ', 'median', 'number'))
    r.append(format_header % ('year', 'month', 'price', 'trades'))
    for year in (2006, 2007):
        for month in xrange(1, 13):
            k = key(year, month)
            r.append(format_detail % (
                year,
                month,
                medians[k],
                counts[k],
            ))
    r.write(control.path_out_txt_2006_2007)

def make_data(control):
    transactions = pd.read_csv(control.path_in_samples,
                               nrows=1000 if control.test else None,
                               )
    prices = collections.defaultdict(list)
    for index, row in transactions.iterrows():
        yyyymm = row[t.yyyymm]
        prices[yyyymm].append(row[t.price])
    counts = {}
    means = {}
    medians = {}
    for k, v in prices.iteritems():
        a = np.array(v)
        counts[k] = len(a)
        means[k] = np.mean(a)
        medians[k] = np.median(a)
    return counts, means, medians, prices

def main(argv):
    control = make_control(argv)
    sys.stdout = Logger(base_name=control.arg.base_name)
    print control

    if control.arg.data:
        data = make_data(control)
        with open(control.path_reduction, 'wb') as f:
            pickle.dump((data, control), f)
    else:
        with open(control.path_reduction, 'rb') as f:
            data, reduction_control = pickle.load(f)
        make_chart_txt_2006_2007(data, control)
        make_chart_txt(data, control)
        make_chart_pdf(data, control)

    print control
    if control.test:
        print 'DISCARD OUTPUT: test'
    print 'done'

    return

if __name__ == '__main__':
    if False:
        # avoid pyflakes warnings
        pdb.set_trace()
        pprint()
        pd.DataFrame()
        np.array()

    main(sys.argv)
<commit_msg>fix documentation<commit_after>'''create charts showing median and mean prices each month

INVOCATION
  python chart-01.py [--data] [--test]

INPUT FILES
 INPUT/samples-train-validate.csv

OUTPUT FILES
 WORKING/chart-01/data.pickle
 WORKING/chart-01/median-price.pdf
 WORKING/chart-01/median-price.txt
 WORKING/chart-01/median-price_2006_2007.txt
'''

from __future__ import division

import collections
import cPickle as pickle
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import pdb
from pprint import pprint
import random
import sys

from Bunch import Bunch
from columns_contain import columns_contain
from Logger import Logger
from ParseCommandLine import ParseCommandLine
from Path import Path
from Report import Report
import layout_transactions as t
cc = columns_contain

def usage(msg=None):
    print __doc__
    if msg is not None:
        print msg
    sys.exit(1)

def make_control(argv):
    # return a Bunch

    print argv
    if len(argv) not in (1, 2, 3):
        usage('invalid number of arguments')

    pcl = ParseCommandLine(argv)
    arg = Bunch(
        base_name='chart-01',
        data=pcl.has_arg('--data'),
        test=pcl.has_arg('--test'),
    )

    random_seed = 123
    random.seed(random_seed)

    dir_working = Path().dir_working()

    debug = False

    reduced_file_name = ('test-' if arg.test else '') + 'data.pickle'

    # assure output directory exists
    dir_path = dir_working + arg.base_name + '/'
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

    return Bunch(
        arg=arg,
        debug=debug,
        path_in_samples=dir_working + 'samples-train-validate.csv',
        path_out_pdf=dir_path + 'median-price.pdf',
        path_out_txt=dir_path + 'median-price.txt',
        path_out_txt_2006_2007=dir_path + 'median-price_2006_2007.txt',
        path_reduction=dir_path + reduced_file_name,
        random_seed=random_seed,
        test=arg.test,
    )

def key(year, month):
    return year * 100 + month

def make_chart_pdf(data, control):
    counts, means, medians, prices = data
    years = (2003, 2004, 2005, 2006, 2007, 2008, 2009)
    months = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)
    months_2009 = (1, 2, 3)
    year_month = ['%s-%02d' % (year, month)
                  for year in years
                  for month in (months_2009 if year == 2009 else months)
                  ]
    x = range(len(year_month))
    y = []
    for year in years:
        for month in (months_2009 if year == 2009 else months):
            y.append(medians[key(year, month)])
    plt.plot(x, y)
    x_ticks = [year_month[i] if i % 12 == 0 else ' '
               for i in xrange(len(year_month))
               ]
    plt.xticks(range(len(year_month)),
               x_ticks,
               # pad=8,
               size='xx-small',
               rotation=-30,
               # rotation='vertical',
               )
    plt.yticks(size='xx-small')
    plt.xlabel('year-month')
    plt.ylabel('median price ($)')
    plt.ylim([0, 700000])
    plt.savefig(control.path_out_pdf)

    plt.close()

def make_chart_txt(data, control):
    counts, means, medians, prices = data
    r = Report()
    format_header = '%7s %7s %7s %7s %7s'
    format_detail = '%7d %7d %7.0f %7.0f %7d'
    r.append('Prices by Month')
    r.append('')
    r.append(format_header % (' ', ' ', 'mean', 'median', 'number'))
    r.append(format_header % ('year', 'month', 'price', 'price', 'trades'))
    for year in xrange(2003, 2010):
        for month in (xrange(1, 4) if year == 2009 else xrange(1, 13)):
            k = key(year, month)
            r.append(format_detail % (
                year,
                month,
                means[k],
                medians[k],
                counts[k],
            ))
    r.write(control.path_out_txt)

def make_chart_txt_2006_2007(data, control):
    counts, means, medians, prices = data
    r = Report()
    format_header = '%7s %7s %7s %7s'
    format_detail = '%7d %7d %7.0f %7d'
    r.append('Median Prices and Transaction Counts by Month')
    r.append('')
    r.append(format_header % (' ', ' ', 'median', 'number'))
    r.append(format_header % ('year', 'month', 'price', 'trades'))
    for year in (2006, 2007):
        for month in xrange(1, 13):
            k = key(year, month)
            r.append(format_detail % (
                year,
                month,
                medians[k],
                counts[k],
            ))
    r.write(control.path_out_txt_2006_2007)

def make_data(control):
    transactions = pd.read_csv(control.path_in_samples,
                               nrows=1000 if control.test else None,
                               )
    prices = collections.defaultdict(list)
    for index, row in transactions.iterrows():
        yyyymm = row[t.yyyymm]
        prices[yyyymm].append(row[t.price])
    counts = {}
    means = {}
    medians = {}
    for k, v in prices.iteritems():
        a = np.array(v)
        counts[k] = len(a)
        means[k] = np.mean(a)
        medians[k] = np.median(a)
    return counts, means, medians, prices

def main(argv):
    control = make_control(argv)
    sys.stdout = Logger(base_name=control.arg.base_name)
    print control

    if control.arg.data:
        data = make_data(control)
        with open(control.path_reduction, 'wb') as f:
            pickle.dump((data, control), f)
    else:
        with open(control.path_reduction, 'rb') as f:
            data, reduction_control = pickle.load(f)
        make_chart_txt_2006_2007(data, control)
        make_chart_txt(data, control)
        make_chart_pdf(data, control)

    print control
    if control.test:
        print 'DISCARD OUTPUT: test'
    print 'done'

    return

if __name__ == '__main__':
    if False:
        # avoid pyflakes warnings
        pdb.set_trace()
        pprint()
        pd.DataFrame()
        np.array()

    main(sys.argv)
<|endoftext|>


<commit_before>
import imp
import os
import sys
import subprocess

from distutils.version import LooseVersion
from SCons.Script import BoolVariable, Dir, Environment, File, PathVariable, SCons, Variables

monoreg = imp.load_source('mono_reg_utils', 'modules/mono/mono_reg_utils.py')

def can_build(env, platform):
    if platform in ['javascript']:
        return False # Not yet supported
    return True

def is_enabled():
    # The module is disabled by default. Use module_mono_enabled=yes to enable it.
    return False

def get_doc_classes():
    return [
        '@C#',
        'CSharpScript',
        'GodotSharp',
    ]

def get_doc_path():
    return 'doc_classes'

def find_file_in_dir(directory, files, prefix='', extension=''):
    if not extension.startswith('.'):
        extension = '.' + extension
    for curfile in files:
        if os.path.isfile(os.path.join(directory, prefix + curfile + extension)):
            return curfile
    return ''

def copy_file(src_dir, dst_dir, name):
    from shutil import copyfile

    src_path = os.path.join(src_dir, name)
    dst_path = os.path.join(dst_dir, name)

    if not os.path.isdir(dst_dir):
        os.mkdir(dst_dir)

    copyfile(src_path, dst_path)

def custom_path_is_dir_create(key, val, env):
    """Validator to check if Path is a directory, creating it if it does not exist.
       Similar to PathIsDirCreate, except it uses SCons.Script.Dir() and
       SCons.Script.File() in order to support the '#' top level directory token.
       """
    # Dir constructor will throw an error if the path points to a file
    fsDir = Dir(val)
    if not fsDir.exists:
        os.makedirs(fsDir.abspath)

def configure(env):
    env.use_ptrcall = True
    env.add_module_version_string('mono')

    envvars = Variables()
    envvars.Add(BoolVariable('mono_static', 'Statically link mono', False))
    envvars.Add(PathVariable('mono_assemblies_output_dir', 'Path to the assemblies output directory', '#bin', custom_path_is_dir_create))
    envvars.Update(env)

    bits = env['bits']

    mono_static = env['mono_static']
    assemblies_output_dir = Dir(env['mono_assemblies_output_dir']).abspath

    mono_lib_names = ['mono-2.0-sgen', 'monosgen-2.0']

    if env['platform'] == 'windows':
        mono_root = None
        if bits == '32':
            if os.getenv('MONO32_PREFIX'):
                mono_root = os.getenv('MONO32_PREFIX')
            elif os.name == 'nt':
                mono_root = monoreg.find_mono_root_dir(bits)
        else:
            if os.getenv('MONO64_PREFIX'):
                mono_root = os.getenv('MONO64_PREFIX')
            elif os.name == 'nt':
                mono_root = monoreg.find_mono_root_dir(bits)

        if not mono_root:
            raise RuntimeError('Mono installation directory not found')

        mono_version = mono_root_try_find_mono_version(mono_root)
        configure_for_mono_version(env, mono_version)

        mono_lib_path = os.path.join(mono_root, 'lib')

        env.Append(LIBPATH=mono_lib_path)
        env.Append(CPPPATH=os.path.join(mono_root, 'include', 'mono-2.0'))

        if mono_static:
            lib_suffix = Environment()['LIBSUFFIX']

            if env.msvc:
                mono_static_lib_name = 'libmono-static-sgen'
            else:
                mono_static_lib_name = 'libmonosgen-2.0'

            if not os.path.isfile(os.path.join(mono_lib_path, mono_static_lib_name + lib_suffix)):
                raise RuntimeError('Could not find static mono library in: ' + mono_lib_path)

            if env.msvc:
                env.Append(LINKFLAGS=mono_static_lib_name + lib_suffix)

                env.Append(LINKFLAGS='Mincore' + lib_suffix)
                env.Append(LINKFLAGS='msvcrt' + lib_suffix)
                env.Append(LINKFLAGS='LIBCMT' + lib_suffix)
                env.Append(LINKFLAGS='Psapi' + lib_suffix)
            else:
                env.Append(LINKFLAGS=os.path.join(mono_lib_path, mono_static_lib_name + lib_suffix))

                env.Append(LIBS='psapi')
                env.Append(LIBS='version')
        else:
            mono_lib_name = find_file_in_dir(mono_lib_path, mono_lib_names, extension='.lib')

            if not mono_lib_name:
                raise RuntimeError('Could not find mono library in: ' + mono_lib_path)

            if env.msvc:
                env.Append(LINKFLAGS=mono_lib_name + Environment()['LIBSUFFIX'])
            else:
                env.Append(LIBS=mono_lib_name)

            mono_bin_path = os.path.join(mono_root, 'bin')

            mono_dll_name = find_file_in_dir(mono_bin_path, mono_lib_names, extension='.dll')

            if not mono_dll_name:
                raise RuntimeError('Could not find mono shared library in: ' + mono_bin_path)

            copy_file(mono_bin_path, 'bin', mono_dll_name + '.dll')

        copy_file(os.path.join(mono_lib_path, 'mono', '4.5'), assemblies_output_dir, 'mscorlib.dll')
    else:
        sharedlib_ext = '.dylib' if sys.platform == 'darwin' else '.so'

        mono_root = ''
        mono_lib_path = ''

        if bits == '32':
            if os.getenv('MONO32_PREFIX'):
                mono_root = os.getenv('MONO32_PREFIX')
        else:
            if os.getenv('MONO64_PREFIX'):
                mono_root = os.getenv('MONO64_PREFIX')

        # We can't use pkg-config to link mono statically,
        # but we can still use it to find the mono root directory
        if not mono_root and mono_static:
            mono_root = pkgconfig_try_find_mono_root(mono_lib_names, sharedlib_ext)
            if not mono_root:
                raise RuntimeError('Building with mono_static=yes, but failed to find the mono prefix with pkg-config. Specify one manually')

        if mono_root:
            mono_version = mono_root_try_find_mono_version(mono_root)
            configure_for_mono_version(env, mono_version)

            mono_lib_path = os.path.join(mono_root, 'lib')

            env.Append(LIBPATH=mono_lib_path)
            env.Append(CPPPATH=os.path.join(mono_root, 'include', 'mono-2.0'))

            mono_lib = find_file_in_dir(mono_lib_path, mono_lib_names, prefix='lib', extension='.a')

            if not mono_lib:
                raise RuntimeError('Could not find mono library in: ' + mono_lib_path)

            env.Append(CPPFLAGS=['-D_REENTRANT'])

            if mono_static:
                mono_lib_file = os.path.join(mono_lib_path, 'lib' + mono_lib + '.a')

                if sys.platform == 'darwin':
                    env.Append(LINKFLAGS=['-Wl,-force_load,' + mono_lib_file])
                elif sys.platform == 'linux' or sys.platform == 'linux2':
                    env.Append(LINKFLAGS=['-Wl,-whole-archive', mono_lib_file, '-Wl,-no-whole-archive'])
                else:
                    raise RuntimeError('mono-static: Not supported on this platform')
            else:
                env.Append(LIBS=[mono_lib])

            if sys.platform == 'darwin':
                env.Append(LIBS=['iconv', 'pthread'])
            elif sys.platform == 'linux' or sys.platform == 'linux2':
                env.Append(LIBS=['m', 'rt', 'dl', 'pthread'])

            if not mono_static:
                mono_so_name = find_file_in_dir(mono_lib_path, mono_lib_names, prefix='lib', extension=sharedlib_ext)

                if not mono_so_name:
                    raise RuntimeError('Could not find mono shared library in: ' + mono_lib_path)

                copy_file(mono_lib_path, 'bin', 'lib' + mono_so_name + sharedlib_ext)

            copy_file(os.path.join(mono_lib_path, 'mono', '4.5'), assemblies_output_dir, 'mscorlib.dll')
        else:
            assert not mono_static

            mono_version = pkgconfig_try_find_mono_version()
            configure_for_mono_version(env, mono_version)

            env.ParseConfig('pkg-config monosgen-2 --cflags --libs')

            mono_lib_path = ''
            mono_so_name = ''
            mono_prefix = subprocess.check_output(['pkg-config', 'mono-2', '--variable=prefix']).decode('utf8').strip()

            tmpenv = Environment()
            tmpenv.AppendENVPath('PKG_CONFIG_PATH', os.getenv('PKG_CONFIG_PATH'))
            tmpenv.ParseConfig('pkg-config monosgen-2 --libs-only-L')

            for hint_dir in tmpenv['LIBPATH']:
                name_found = find_file_in_dir(hint_dir, mono_lib_names, prefix='lib', extension=sharedlib_ext)
                if name_found:
                    mono_lib_path = hint_dir
                    mono_so_name = name_found
                    break

            if not mono_so_name:
                raise RuntimeError('Could not find mono shared library in: ' + str(tmpenv['LIBPATH']))

            copy_file(mono_lib_path, 'bin', 'lib' + mono_so_name + sharedlib_ext)
            copy_file(os.path.join(mono_prefix, 'lib', 'mono', '4.5'), assemblies_output_dir, 'mscorlib.dll')

        env.Append(LINKFLAGS='-rdynamic')

def configure_for_mono_version(env, mono_version):
    if mono_version is None:
        raise RuntimeError('Mono JIT compiler version not found')
    print('Mono JIT compiler version: ' + str(mono_version))
    if mono_version >= LooseVersion("5.12.0"):
        env.Append(CPPFLAGS=['-DHAS_PENDING_EXCEPTIONS'])

def pkgconfig_try_find_mono_root(mono_lib_names, sharedlib_ext):
    tmpenv = Environment()
    tmpenv.AppendENVPath('PKG_CONFIG_PATH', os.getenv('PKG_CONFIG_PATH'))
    tmpenv.ParseConfig('pkg-config monosgen-2 --libs-only-L')
    for hint_dir in tmpenv['LIBPATH']:
        name_found = find_file_in_dir(hint_dir, mono_lib_names, prefix='lib', extension=sharedlib_ext)
        if name_found and os.path.isdir(os.path.join(hint_dir, '..', 'include', 'mono-2.0')):
            return os.path.join(hint_dir, '..')
    return ''

def pkgconfig_try_find_mono_version():
    lines = subprocess.check_output(['pkg-config', 'monosgen-2', '--modversion']).splitlines()
    greater_version = None
    for line in lines:
        try:
            version = LooseVersion(line)
            if greater_version is None or version > greater_version:
                greater_version = version
        except ValueError:
            pass
    return greater_version

def mono_root_try_find_mono_version(mono_root):
    from compat import decode_utf8

    output = subprocess.check_output([os.path.join(mono_root, 'bin', 'mono'), '--version'])
    first_line = decode_utf8(output.splitlines()[0])
    try:
        return LooseVersion(first_line.split()[len('Mono JIT compiler version'.split())])
    except (ValueError, IndexError):
        return None
<commit_msg>SCons: Fix mono_root initialization<commit_after>
import imp
import os
import sys
import subprocess

from distutils.version import LooseVersion
from SCons.Script import BoolVariable, Dir, Environment, File, PathVariable, SCons, Variables

monoreg = imp.load_source('mono_reg_utils', 'modules/mono/mono_reg_utils.py')

def can_build(env, platform):
    if platform in ['javascript']:
        return False # Not yet supported
    return True

def is_enabled():
    # The module is disabled by default. Use module_mono_enabled=yes to enable it.
    return False

def get_doc_classes():
    return [
        '@C#',
        'CSharpScript',
        'GodotSharp',
    ]

def get_doc_path():
    return 'doc_classes'

def find_file_in_dir(directory, files, prefix='', extension=''):
    if not extension.startswith('.'):
        extension = '.' + extension
    for curfile in files:
        if os.path.isfile(os.path.join(directory, prefix + curfile + extension)):
            return curfile
    return ''

def copy_file(src_dir, dst_dir, name):
    from shutil import copyfile

    src_path = os.path.join(src_dir, name)
    dst_path = os.path.join(dst_dir, name)

    if not os.path.isdir(dst_dir):
        os.mkdir(dst_dir)

    copyfile(src_path, dst_path)

def custom_path_is_dir_create(key, val, env):
    """Validator to check if Path is a directory, creating it if it does not exist.
       Similar to PathIsDirCreate, except it uses SCons.Script.Dir() and
       SCons.Script.File() in order to support the '#' top level directory token.
       """
    # Dir constructor will throw an error if the path points to a file
    fsDir = Dir(val)
    if not fsDir.exists:
        os.makedirs(fsDir.abspath)

def configure(env):
    env.use_ptrcall = True
    env.add_module_version_string('mono')

    envvars = Variables()
    envvars.Add(BoolVariable('mono_static', 'Statically link mono', False))
    envvars.Add(PathVariable('mono_assemblies_output_dir', 'Path to the assemblies output directory', '#bin', custom_path_is_dir_create))
    envvars.Update(env)

    bits = env['bits']

    mono_static = env['mono_static']
    assemblies_output_dir = Dir(env['mono_assemblies_output_dir']).abspath

    mono_lib_names = ['mono-2.0-sgen', 'monosgen-2.0']

    if env['platform'] == 'windows':
        mono_root = ''

        if bits == '32':
            if os.getenv('MONO32_PREFIX'):
                mono_root = os.getenv('MONO32_PREFIX')
            elif os.name == 'nt':
                mono_root = monoreg.find_mono_root_dir(bits)
        else:
            if os.getenv('MONO64_PREFIX'):
                mono_root = os.getenv('MONO64_PREFIX')
            elif os.name == 'nt':
                mono_root = monoreg.find_mono_root_dir(bits)

        if not mono_root:
            raise RuntimeError('Mono installation directory not found')

        mono_version = mono_root_try_find_mono_version(mono_root)
        configure_for_mono_version(env, mono_version)

        mono_lib_path = os.path.join(mono_root, 'lib')

        env.Append(LIBPATH=mono_lib_path)
        env.Append(CPPPATH=os.path.join(mono_root, 'include', 'mono-2.0'))

        if mono_static:
            lib_suffix = Environment()['LIBSUFFIX']

            if env.msvc:
                mono_static_lib_name = 'libmono-static-sgen'
            else:
                mono_static_lib_name = 'libmonosgen-2.0'

            if not os.path.isfile(os.path.join(mono_lib_path, mono_static_lib_name + lib_suffix)):
                raise RuntimeError('Could not find static mono library in: ' + mono_lib_path)

            if env.msvc:
                env.Append(LINKFLAGS=mono_static_lib_name + lib_suffix)

                env.Append(LINKFLAGS='Mincore' + lib_suffix)
                env.Append(LINKFLAGS='msvcrt' + lib_suffix)
                env.Append(LINKFLAGS='LIBCMT' + lib_suffix)
                env.Append(LINKFLAGS='Psapi' + lib_suffix)
            else:
                env.Append(LINKFLAGS=os.path.join(mono_lib_path, mono_static_lib_name + lib_suffix))

                env.Append(LIBS='psapi')
                env.Append(LIBS='version')
        else:
            mono_lib_name = find_file_in_dir(mono_lib_path, mono_lib_names, extension='.lib')

            if not mono_lib_name:
                raise RuntimeError('Could not find mono library in: ' + mono_lib_path)

            if env.msvc:
                env.Append(LINKFLAGS=mono_lib_name + Environment()['LIBSUFFIX'])
            else:
                env.Append(LIBS=mono_lib_name)

            mono_bin_path = os.path.join(mono_root, 'bin')

            mono_dll_name = find_file_in_dir(mono_bin_path, mono_lib_names, extension='.dll')

            if not mono_dll_name:
                raise RuntimeError('Could not find mono shared library in: ' + mono_bin_path)

            copy_file(mono_bin_path, 'bin', mono_dll_name + '.dll')

        copy_file(os.path.join(mono_lib_path, 'mono', '4.5'), assemblies_output_dir, 'mscorlib.dll')
    else:
        sharedlib_ext = '.dylib' if sys.platform == 'darwin' else '.so'

        mono_root = ''
        mono_lib_path = ''

        if bits == '32':
            if os.getenv('MONO32_PREFIX'):
                mono_root = os.getenv('MONO32_PREFIX')
        else:
            if os.getenv('MONO64_PREFIX'):
                mono_root = os.getenv('MONO64_PREFIX')

        # We can't use pkg-config to link mono statically,
        # but we can still use it to find the mono root directory
        if not mono_root and mono_static:
            mono_root = pkgconfig_try_find_mono_root(mono_lib_names, sharedlib_ext)
            if not mono_root:
                raise RuntimeError('Building with mono_static=yes, but failed to find the mono prefix with pkg-config. Specify one manually')

        if mono_root:
            mono_version = mono_root_try_find_mono_version(mono_root)
            configure_for_mono_version(env, mono_version)

            mono_lib_path = os.path.join(mono_root, 'lib')

            env.Append(LIBPATH=mono_lib_path)
            env.Append(CPPPATH=os.path.join(mono_root, 'include', 'mono-2.0'))

            mono_lib = find_file_in_dir(mono_lib_path, mono_lib_names, prefix='lib', extension='.a')

            if not mono_lib:
                raise RuntimeError('Could not find mono library in: ' + mono_lib_path)

            env.Append(CPPFLAGS=['-D_REENTRANT'])

            if mono_static:
                mono_lib_file = os.path.join(mono_lib_path, 'lib' + mono_lib + '.a')

                if sys.platform == 'darwin':
                    env.Append(LINKFLAGS=['-Wl,-force_load,' + mono_lib_file])
                elif sys.platform == 'linux' or sys.platform == 'linux2':
                    env.Append(LINKFLAGS=['-Wl,-whole-archive', mono_lib_file, '-Wl,-no-whole-archive'])
                else:
                    raise RuntimeError('mono-static: Not supported on this platform')
            else:
                env.Append(LIBS=[mono_lib])

            if sys.platform == 'darwin':
                env.Append(LIBS=['iconv', 'pthread'])
            elif sys.platform == 'linux' or sys.platform == 'linux2':
                env.Append(LIBS=['m', 'rt', 'dl', 'pthread'])

            if not mono_static:
                mono_so_name = find_file_in_dir(mono_lib_path, mono_lib_names, prefix='lib', extension=sharedlib_ext)

                if not mono_so_name:
                    raise RuntimeError('Could not find mono shared library in: ' + mono_lib_path)

                copy_file(mono_lib_path, 'bin', 'lib' + mono_so_name + sharedlib_ext)

            copy_file(os.path.join(mono_lib_path, 'mono', '4.5'), assemblies_output_dir, 'mscorlib.dll')
        else:
            assert not mono_static

            mono_version = pkgconfig_try_find_mono_version()
            configure_for_mono_version(env, mono_version)

            env.ParseConfig('pkg-config monosgen-2 --cflags --libs')

            mono_lib_path = ''
            mono_so_name = ''
            mono_prefix = subprocess.check_output(['pkg-config', 'mono-2', '--variable=prefix']).decode('utf8').strip()

            tmpenv = Environment()
            tmpenv.AppendENVPath('PKG_CONFIG_PATH', os.getenv('PKG_CONFIG_PATH'))
            tmpenv.ParseConfig('pkg-config monosgen-2 --libs-only-L')

            for hint_dir in tmpenv['LIBPATH']:
                name_found = find_file_in_dir(hint_dir, mono_lib_names, prefix='lib', extension=sharedlib_ext)
                if name_found:
                    mono_lib_path = hint_dir
                    mono_so_name = name_found
                    break

            if not mono_so_name:
                raise RuntimeError('Could not find mono shared library in: ' + str(tmpenv['LIBPATH']))

            copy_file(mono_lib_path, 'bin', 'lib' + mono_so_name + sharedlib_ext)
            copy_file(os.path.join(mono_prefix, 'lib', 'mono', '4.5'), assemblies_output_dir, 'mscorlib.dll')

        env.Append(LINKFLAGS='-rdynamic')

def configure_for_mono_version(env, mono_version):
    if mono_version is None:
        raise RuntimeError('Mono JIT compiler version not found')
    print('Mono JIT compiler version: ' + str(mono_version))
    if mono_version >= LooseVersion("5.12.0"):
        env.Append(CPPFLAGS=['-DHAS_PENDING_EXCEPTIONS'])

def pkgconfig_try_find_mono_root(mono_lib_names, sharedlib_ext):
    tmpenv = Environment()
    tmpenv.AppendENVPath('PKG_CONFIG_PATH', os.getenv('PKG_CONFIG_PATH'))
    tmpenv.ParseConfig('pkg-config monosgen-2 --libs-only-L')
    for hint_dir in tmpenv['LIBPATH']:
        name_found = find_file_in_dir(hint_dir, mono_lib_names, prefix='lib', extension=sharedlib_ext)
        if name_found and os.path.isdir(os.path.join(hint_dir, '..', 'include', 'mono-2.0')):
            return os.path.join(hint_dir, '..')
    return ''

def pkgconfig_try_find_mono_version():
    lines = subprocess.check_output(['pkg-config', 'monosgen-2', '--modversion']).splitlines()
    greater_version = None
    for line in lines:
        try:
            version = LooseVersion(line)
            if greater_version is None or version > greater_version:
                greater_version = version
        except ValueError:
            pass
    return greater_version

def mono_root_try_find_mono_version(mono_root):
    from compat import decode_utf8

    output = subprocess.check_output([os.path.join(mono_root, 'bin', 'mono'), '--version'])
    first_line = decode_utf8(output.splitlines()[0])
    try:
        return LooseVersion(first_line.split()[len('Mono JIT compiler version'.split())])
    except (ValueError, IndexError):
        return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import recalboxFiles
import re
from settings.unixSettings import UnixSettings
import time
import subprocess
import json
import eslog

# Set a specific video mode
def changeResolution(videomode):
    eslog.log("videomode: " + videomode)
    if videomode != 'default':
        cmd = createVideoModeLine(videomode)
        eslog.log("setVideoMode(" + videomode + "): " + cmd)
        os.system(cmd)
        time.sleep(0.5) # let time for the video to change the resolution (the commands returns before it's really done ;-(

def createVideoModeLine(videoMode):
    # pattern (CEA|DMT) [0-9]{1,2} HDMI
    if re.match("^(CEA|DMT) [0-9]{1,2}( HDMI)?$", videoMode):
        return "tvservice -e '{}'".format(videoMode)
    if re.match("^hdmi_cvt [\d\s]{10,20}$", videoMode):
        return "vcgencmd {} && tvservice -e 'DMT 87'".format(videoMode)
    if re.match("^hdmi_timings [\d\s]{48,58}$", videoMode):
        return "vcgencmd {} && tvservice -e 'DMT 87'".format(videoMode)

# Switch to prefered mode
def resetResolution():
    recalSettings = UnixSettings(recalboxFiles.recalboxConf)
    esVideoMode = recalSettings.load('system.es.videomode')
    if esVideoMode is None:
        cmd = "tvservice -p"
        eslog.log("resetting video mode: " + cmd)
        os.system(cmd)
    else:
        setVideoMode(esVideoMode)

def getCurrentResolution():
	proc = subprocess.Popen(["tvservice.current"], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	tvmodes = json.loads(out)

	for tvmode in tvmodes:
	    return { "width": tvmode["width"], "height": tvmode["height"] }

        raise Exception("No current resolution found")
<commit_msg>fix bad videomode crash<commit_after>#!/usr/bin/env python
import os
import sys
import recalboxFiles
import re
from settings.unixSettings import UnixSettings
import time
import subprocess
import json
import eslog

# Set a specific video mode
def changeResolution(videomode):
    eslog.log("videomode: " + videomode)
    if videomode != 'default':
        cmd = createVideoModeLine(videomode)
        if cmd is not None:
            eslog.log("setVideoMode(" + videomode + "): " + cmd)
            os.system(cmd)
            time.sleep(0.5) # let time for the video to change the resolution (the commands returns before it's really done ;-(

def createVideoModeLine(videoMode):
    # pattern (CEA|DMT) [0-9]{1,2} HDMI
    if re.match("^(CEA|DMT) [0-9]{1,2}( HDMI)?$", videoMode):
        return "tvservice -e '{}'".format(videoMode)
    if re.match("^hdmi_cvt [\d\s]{10,20}$", videoMode):
        return "vcgencmd {} && tvservice -e 'DMT 87'".format(videoMode)
    if re.match("^hdmi_timings [\d\s]{48,58}$", videoMode):
        return "vcgencmd {} && tvservice -e 'DMT 87'".format(videoMode)
    return None

# Switch to prefered mode
def resetResolution():
    recalSettings = UnixSettings(recalboxFiles.recalboxConf)
    esVideoMode = recalSettings.load('system.es.videomode')
    if esVideoMode is None:
        cmd = "tvservice -p"
        eslog.log("resetting video mode: " + cmd)
        os.system(cmd)
    else:
        setVideoMode(esVideoMode)

def getCurrentResolution():
	proc = subprocess.Popen(["tvservice.current"], stdout=subprocess.PIPE, shell=True)
	(out, err) = proc.communicate()
	tvmodes = json.loads(out)

	for tvmode in tvmodes:
	    return { "width": tvmode["width"], "height": tvmode["height"] }

        raise Exception("No current resolution found")
<|endoftext|>


<commit_before>from PyDictionary import PyDictionary
from aiohttp import ClientSession
from datetime import datetime
from geopy import geocoders
from bin import data
import random as ran
import discord
import re

# Check if value is an int
def is_int_value(_value):
    try:
        int(_value)
        return True
    except ValueError:
        return False

# Check if value is a float
def is_float_value(_value):
    try:
        float(_value)
        return True
    except ValueError:
        return False

# Check if value is ID # or @user
def is_user_value(_value):
    if re.match('^<@[0-9]{18}>$', _value):  # <@304040086482321408>
        return True
    elif re.match('^[0-9]{18}$', _value):  # 304040086482321408
        return True
    return False

# Get ID from ID # or @user
def get_user_id(_value):
    if re.match('^<@[0-9]{18}>$', _value):  # <@304040086482321408>
        return str(_value[2:-1])
    elif re.match('^[0-9]{18}$', _value):  # 304040086482321408
        return str(_value)
    return -1

def get_name(_dclient, _server, _user_id):
    return discord.utils.get(discord.utils.get(_dclient.servers,
                                               id=_server.id).members,
                             id=_user_id).name

# Construct usage messages
def usage_msg(_cmd, _mention, _cmdchar):
    return ':warning: {} **USAGE:** {}{}'.format(_mention, _cmdchar, _cmd)

# Gif command
async def gif(_dclient, _channel, _mention, _cmdchar, _msg):
    _msg = _msg.split(' ')
    if len(_msg) > 1:
        _msg.pop(0)
        keywords = ''
        for m in _msg:
            keywords += m + '+'
        async with ClientSession() as cs:
            async with cs.get('http://api.giphy.com/v1/gifs/random?api_key=dc6zaTOxFJmzC&tag={}'.format(_msg)) \
                    as response:
                gif_data = await response.json()
                await _dclient.send_message(_channel, '{}'.format(gif_data['data']['fixed_height_downsampled_url']))
    else:
        await _dclient.send_message(_channel, usage_msg('gif <keywords...>', _mention, _cmdchar))

# Cat command
async def cat(_dclient, _channel):
    async with ClientSession() as cs:
        async with cs.get('http://random.cat/meow') as response:
            cat_data = await response.json()
            await _dclient.send_message(_channel, '{}'.format(cat_data['file']))

# Choose command
async def choose(_dclient, _channel, _mention, _cmdchar, _msg):
    options = _msg.split(' ')
    if len(options) >= 3:
        options.pop(0)
        await _dclient.send_message(_channel, '{} I choose `{}`!'.format(_mention, ran.choice(options)))
    else:
        await _dclient.send_message(_channel, usage_msg('choose <options...>', _mention, _cmdchar))

# Dice command
async def dice(_dclient, _channel, _mention):
    await _dclient.send_message(_channel, '{} *rolls dice...* I got a `{}`!'
                                .format(_mention, ran.choice([1, 2, 3, 4, 5, 6])))

# Dict command (Dictionary)
async def dict(_dclient, _channel, _mention, _cmdchar, _msg):
    _msg = _msg.split(' ')
    if len(_msg) == 2:
        word = _msg[1].lower()
        dict = PyDictionary()
        try:
            result = dict.meaning(word)
            nouns = []
            verbs = []
            adjectives = []
            if 'Noun' in result:
                for i in range(0, 2):
                    nouns.append(result['Noun'][i])
            if 'Verb' in result:
                for i in range(0, 2):
                    verbs.append(result['Verb'][i])
            if 'Adjective' in result:
                for i in range(0, 2):
                    adjectives.append(result['Adjective'][i])

            embed = discord.Embed(title='Dictionary Results', color=0xff5700)
            embed.set_author(name='Car')
            embed.set_thumbnail(url='https://i.imgur.com/VCck2RQ.png')
            count = 1
            for noun in nouns:
                embed.add_field(name='{}. {}'.format(count, 'noun'), value=noun, inline=False)
                count += 1
            count = 1
            for verb in verbs:
                embed.add_field(name='{}. {}'.format(count, 'verb'), value=verb, inline=False)
                count += 1
            count = 1
            for adj in adjectives:
                embed.add_field(name='{}. {}'.format(count, 'adjective'), value=adj, inline=False)
                count += 1
            await _dclient.send_message(_channel, embed=embed)
        except Exception:
            await _dclient.send_message(_channel, ':warning: {} There is no meaning for `{}`!'.format(_mention, word))
    else:
        await _dclient.send_message(_channel, usage_msg('dict <word>', _mention, _cmdchar))

# Purge command
async def purge(_dclient, _channel, _mention, _cmdchar, _msg, _author, _server):
    _msg = _msg.split(' ')
    if _channel.permissions_for(_author).administrator:
        if len(_msg) > 1:
            _msg.pop(0)
            if _msg[0].lower() == 'all':
                if len(_msg) == 2:
                    if is_int_value(_msg[1]):
                        try:
                            value = int(_msg[1])
                            msg_list = list()
                            async for msg in _dclient.logs_from(_channel, limit=value):
                                msg_list.append(msg)
                            await _dclient.delete_messages(msg_list)
                            await _dclient.send_message(_channel, '{} `{}` messages purged from this channel!'
                                                        .format(_mention, value))
                        except discord.Forbidden:
                            await _dclient.send_message(_channel, ':warning: {} I don\'t have access to `manage_'
                                                                  'messages`! Please notify an admin!'.format(_mention))
                    else:
                        await _dclient.send_message(_channel, ':warning: {} `{}` must be in numeric value!'
                                                    .format(_mention, _msg[1]))
                else:
                    await _dclient.send_message(_channel, usage_msg('purge all <#>', _mention, _cmdchar))
            elif _msg[0].lower() == 'user':
                if len(_msg) == 3:
                    if is_user_value(_msg[1]):
                        if is_int_value(_msg[2]):
                            try:
                                user_id = get_user_id(_msg[1])
                                value = int(_msg[2])
                                msg_list = list()
                                async for msg in _dclient.logs_from(_channel, limit=value):
                                    if msg.author.id == user_id:
                                        msg_list.append(msg)
                                await _dclient.delete_messages(msg_list)
                                await _dclient.send_message(_channel, '{} `{}` messages from `{}` has been purged from '
                                                                      'this channel!'
                                                            .format(_mention, len(msg_list),
                                                                    get_name(_dclient, _server, user_id)))
                            except discord.Forbidden:
                                await _dclient.send_message(_channel,
                                                            ':warning: {} I don\'t have access to `manage_messages`! '
                                                            'Please notify an admin!'.format(_mention))
                        else:
                            await _dclient.send_message(_channel, ':warning: {} `{}` must be in numeric value!'
                                                        .format(_mention, _msg[1]))
                    else:
                        await _dclient.send_message(_channel, ':warning: {} `{}` must be an ID # or @user value!'
                                                    .format(_mention, _msg[1]))
                else:
                    await _dclient.send_message(_channel, usage_msg('purge user <id> <#>', _mention, _cmdchar))
            elif _msg[0].lower() == 'today':
                try:
                    time = datetime.now().replace(hour=0, minute=0, second=0)
                    msg_list = list()
                    async for msg in _dclient.logs_from(_channel, after=time):
                        msg_list.append(msg)
                    await _dclient.delete_messages(msg_list)
                    await _dclient.send_message(_channel, '{} `{}` messages sent today has been purged from this '
                                                          'channel!'.format(_mention, len(msg_list)))
                except discord.Forbidden:
                    await _dclient.send_message(_channel,
                                                ':warning: {} I don\'t have access to `manage_messages`! Please notify'
                                                ' an admin!'.format(_mention))
            else:
                await _dclient.send_message(_channel, usage_msg('purge <all|user|today>', _mention, _cmdchar))
        else:
            await _dclient.send_message(_channel, usage_msg('purge <all|user|today>', _mention, _cmdchar))
    else:
        await _dclient.send_message(_channel, ':warning: {} You must be an administrator to use this command!'
                                    .format(_mention))

# RPS command
async def rps(_dclient, _channel, _mention, _cmdchar, _msg, _name):
    _msg = _msg.split(' ')
    if len(_msg) > 1:
        _msg.pop(0)
        user_choice = _msg[0].lower()
        if user_choice in ['rock', 'paper', 'scissors']:
            bot_choice = ran.choice(['rock', 'paper', 'scissors'])
            result = data.rps_result['{}_{}'.format(user_choice, bot_choice)]
            if result['type'] == 0:
                await _dclient.send_message(_channel, result['response'].format(ran.choice(data.rps_tie_response)))
            elif result['type'] == 1:
                await _dclient.send_message(_channel, result['response'].format(ran.choice(data.rps_bot_response)))
            else:
                await _dclient.send_message(_channel, result['response'].format(ran.choice(data.rps_player_response)))
        else:
            await _dclient.send_message(_channel, ':warning: {} Please choose an option between `rock`, `paper`, and '
                                                  '`scissors`.'.format(_mention))
    else:
        await _dclient.send_message(_channel, usage_msg('rps <rock|paper|scissors>', _mention, _cmdchar))

# Temp command
async def temp(_dclient, _channel, _mention, _cmdchar, _msg):
    _msg = _msg.split(' ')
    if len(_msg) == 4 and _msg[2].upper() in ['F', 'K', 'C'] and _msg[3].upper() in ['F', 'K', 'C']:
        _msg.pop(0)
        if is_float_value(_msg[0]):
            value = float(_msg[0])
            from_metric = _msg[1].upper()
            to_metric = _msg[2].upper()
            if from_metric != to_metric:
                result = 0
                if from_metric == 'F' and to_metric == 'K':
                    result = (value + 459.67) * (5 / 9)
                elif from_metric == 'F' and to_metric == 'C':
                    result = (value - 32) * .5556
                elif from_metric == 'K' and to_metric == 'F':
                    result = (value * (9 / 5)) - 459.67
                elif from_metric == 'K' and to_metric == 'C':
                    result = (value * (9 / 5)) - 273.15
                elif from_metric == 'C' and to_metric == 'F':
                    result = (value * 1.8) + 32
                elif from_metric == 'C' and to_metric == 'K':
                    result = (value + 273.15) * (5 / 9)
                await _dclient.send_message(_channel, data.temp_response['{}_{}'.format(from_metric, to_metric)]
                                            .format(value, result))
            else:
                await _dclient.send_message(_channel, ':warning: {} You can\'t convert the temperature that way, '
                                                      'silly!'.format(_mention))
        else:
            await _dclient.send_message(_channel, ':warning: {} The temperature value must be in numeric value!'
                                        .format(_mention))
    else:
        await _dclient.send_message(_channel, usage_msg('temp <#> <from F|K|C> <to F|K|C>', _mention, _cmdchar))

# Time command
async def time(_dclient, _channel, _mention, _cmdchar, _msg):
    if len(_msg.split(' ')) > 1:
        _msg = _msg[6:]
        google_api = geocoders.GoogleV3()
        try:
            place, (lat, lng) = google_api.geocode(_msg)
            timezone = google_api.timezone((lat, lng))
            time_now = datetime.now(timezone)
            await _dclient.send_message(_channel, '{} It is `{:%H:%M:%S}` or `{:%I:%M:%S %p}` in `{}` right now.'
                                        .format(_mention, time_now, time_now, _msg))
        except Exception:
            await _dclient.send_message(_channel, '{} Location `{}` unknown! Please try to be more specific. I rely on '
                                                  'Google maps to get coordinates for the timezone!'
                                        .format(_mention, _msg))
    else:
        await _dclient.send_message(_channel, usage_msg('time <location>', _mention, _cmdchar))

# Uptime command
async def uptime(_dclient, _channel, _mention, _uptime_start):
    dtime = datetime.now() - _uptime_start
    days = dtime.days
    if days == 0:
        dtime = datetime.strptime(str(dtime), "%H:%M:%S.%f")
        await _dclient.send_message(_channel, '{} I have been up for `{:d} hours`, `{:d} minutes`, and `{:d} seconds`.'
                                    .format(_mention, dtime.hour, dtime.minute, dtime.second))
    elif days == 1:
        dtime = datetime.strptime(str(dtime), "%d day, %H:%M:%S.%f")
        await _dclient.send_message(_channel, '{} I have been up for `1 day`, `{:d} hours`, `{:d} minutes`, and `{:d} '
                                              'seconds`.'.format(_mention, dtime.hour, dtime.minute, dtime.second))
    else:
        dtime = datetime.strptime(str(dtime), "%d day, %H:%M:%S.%f")
        await _dclient.send_message(_channel, '{} I have been up for `{:d} days`, `{:d} hours`, `{:d} minutes`, and '
                                              '`{:d} seconds`.'
                                    .format(_mention, days, dtime.hour, dtime.minute, dtime.second))
<commit_msg>Wolfbitez kept complaining so I had to merge this<commit_after>from PyDictionary import PyDictionary
from aiohttp import ClientSession
from datetime import datetime
from geopy import geocoders
from bin import data
import random as ran
import discord
import re

# Check if value is an int
def is_int_value(_value):
    try:
        int(_value)
        return True
    except ValueError:
        return False

# Check if value is a float
def is_float_value(_value):
    try:
        float(_value)
        return True
    except ValueError:
        return False

# Check if value is ID # or @user
def is_user_value(_value):
    if re.match('^<@[0-9]{18}>$', _value):  # <@304040086482321408>
        return True
    elif re.match('^[0-9]{18}$', _value):  # 304040086482321408
        return True
    return False

# Get ID from ID # or @user
def get_user_id(_value):
    if re.match('^<@[0-9]{18}>$', _value):  # <@304040086482321408>
        return str(_value[2:-1])
    elif re.match('^[0-9]{18}$', _value):  # 304040086482321408
        return str(_value)
    return -1

def get_name(_dclient, _server, _user_id):
    return discord.utils.get(discord.utils.get(_dclient.servers,
                                               id=_server.id).members,
                             id=_user_id).name

# Construct usage messages
def usage_msg(_cmd, _mention, _cmdchar):
    return ':warning: {} **USAGE:** {}{}'.format(_mention, _cmdchar, _cmd)

# Gif command
async def gif(_dclient, _channel, _mention, _cmdchar, _msg):
    _msg = _msg.split(' ')
    if len(_msg) > 1:
        _msg.pop(0)
        keywords = ''
        for m in _msg:
            keywords += m + '+'
        async with ClientSession() as cs:
            async with cs.get('http://api.giphy.com/v1/gifs/random?api_key=dc6zaTOxFJmzC&tag={}'.format(_msg)) \
                    as response:
                gif_data = await response.json()
                await _dclient.send_message(_channel, '{}'.format(gif_data['data']['fixed_height_downsampled_url']))
    else:
        await _dclient.send_message(_channel, usage_msg('gif <keywords...>', _mention, _cmdchar))

# Cat command
async def cat(_dclient, _channel):
    async with ClientSession() as cs:
        async with cs.get('http://random.cat/meow') as response:
            cat_data = await response.json()
            await _dclient.send_message(_channel, '{}'.format(cat_data['file']))

# Choose command
async def choose(_dclient, _channel, _mention, _cmdchar, _msg):
    options = _msg.split(' ')
    if len(options) >= 3:
        options.pop(0)
        await _dclient.send_message(_channel, '{} I choose `{}`!'.format(_mention, ran.choice(options)))
    else:
        await _dclient.send_message(_channel, usage_msg('choose <options...>', _mention, _cmdchar))

# Dice command
async def dice(_dclient, _channel, _mention):
    await _dclient.send_message(_channel, '{} *rolls dice...* I got a `{}`!'
                                .format(_mention, ran.choice([1, 2, 3, 4, 5, 6])))

# Dict command (Dictionary)
async def dict(_dclient, _channel, _mention, _cmdchar, _msg):
    _msg = _msg.split(' ')
    if len(_msg) == 2:
        word = _msg[1].lower()
        dict = PyDictionary()
        try:
            result = dict.meaning(word)
            nouns = []
            verbs = []
            adjectives = []
            if 'Noun' in result:
                for i in range(0, 2):
                    nouns.append(result['Noun'][i])
            if 'Verb' in result:
                for i in range(0, 2):
                    verbs.append(result['Verb'][i])
            if 'Adjective' in result:
                for i in range(0, 2):
                    adjectives.append(result['Adjective'][i])

            embed = discord.Embed(title='Dictionary Results', color=0xff5700)
            embed.set_author(name=word)
            embed.set_thumbnail(url='https://i.imgur.com/VCck2RQ.png')
            count = 1
            for noun in nouns:
                embed.add_field(name='{}. {}'.format(count, 'noun'), value=noun, inline=False)
                count += 1
            count = 1
            for verb in verbs:
                embed.add_field(name='{}. {}'.format(count, 'verb'), value=verb, inline=False)
                count += 1
            count = 1
            for adj in adjectives:
                embed.add_field(name='{}. {}'.format(count, 'adjective'), value=adj, inline=False)
                count += 1
            await _dclient.send_message(_channel, embed=embed)
        except Exception:
            await _dclient.send_message(_channel, ':warning: {} There is no meaning for `{}`!'.format(_mention, word))
    else:
        await _dclient.send_message(_channel, usage_msg('dict <word>', _mention, _cmdchar))

# Purge command
async def purge(_dclient, _channel, _mention, _cmdchar, _msg, _author, _server):
    _msg = _msg.split(' ')
    if _channel.permissions_for(_author).administrator:
        if len(_msg) > 1:
            _msg.pop(0)
            if _msg[0].lower() == 'all':
                if len(_msg) == 2:
                    if is_int_value(_msg[1]):
                        try:
                            value = int(_msg[1])
                            msg_list = list()
                            async for msg in _dclient.logs_from(_channel, limit=value):
                                msg_list.append(msg)
                            await _dclient.delete_messages(msg_list)
                            await _dclient.send_message(_channel, '{} `{}` messages purged from this channel!'
                                                        .format(_mention, value))
                        except discord.Forbidden:
                            await _dclient.send_message(_channel, ':warning: {} I don\'t have access to `manage_'
                                                                  'messages`! Please notify an admin!'.format(_mention))
                    else:
                        await _dclient.send_message(_channel, ':warning: {} `{}` must be in numeric value!'
                                                    .format(_mention, _msg[1]))
                else:
                    await _dclient.send_message(_channel, usage_msg('purge all <#>', _mention, _cmdchar))
            elif _msg[0].lower() == 'user':
                if len(_msg) == 3:
                    if is_user_value(_msg[1]):
                        if is_int_value(_msg[2]):
                            try:
                                user_id = get_user_id(_msg[1])
                                value = int(_msg[2])
                                msg_list = list()
                                async for msg in _dclient.logs_from(_channel, limit=value):
                                    if msg.author.id == user_id:
                                        msg_list.append(msg)
                                await _dclient.delete_messages(msg_list)
                                await _dclient.send_message(_channel, '{} `{}` messages from `{}` has been purged from '
                                                                      'this channel!'
                                                            .format(_mention, len(msg_list),
                                                                    get_name(_dclient, _server, user_id)))
                            except discord.Forbidden:
                                await _dclient.send_message(_channel,
                                                            ':warning: {} I don\'t have access to `manage_messages`! '
                                                            'Please notify an admin!'.format(_mention))
                        else:
                            await _dclient.send_message(_channel, ':warning: {} `{}` must be in numeric value!'
                                                        .format(_mention, _msg[1]))
                    else:
                        await _dclient.send_message(_channel, ':warning: {} `{}` must be an ID # or @user value!'
                                                    .format(_mention, _msg[1]))
                else:
                    await _dclient.send_message(_channel, usage_msg('purge user <id> <#>', _mention, _cmdchar))
            elif _msg[0].lower() == 'today':
                try:
                    time = datetime.now().replace(hour=0, minute=0, second=0)
                    msg_list = list()
                    async for msg in _dclient.logs_from(_channel, after=time):
                        msg_list.append(msg)
                    await _dclient.delete_messages(msg_list)
                    await _dclient.send_message(_channel, '{} `{}` messages sent today has been purged from this '
                                                          'channel!'.format(_mention, len(msg_list)))
                except discord.Forbidden:
                    await _dclient.send_message(_channel,
                                                ':warning: {} I don\'t have access to `manage_messages`! Please notify'
                                                ' an admin!'.format(_mention))
            else:
                await _dclient.send_message(_channel, usage_msg('purge <all|user|today>', _mention, _cmdchar))
        else:
            await _dclient.send_message(_channel, usage_msg('purge <all|user|today>', _mention, _cmdchar))
    else:
        await _dclient.send_message(_channel, ':warning: {} You must be an administrator to use this command!'
                                    .format(_mention))

# RPS command
async def rps(_dclient, _channel, _mention, _cmdchar, _msg, _name):
    _msg = _msg.split(' ')
    if len(_msg) > 1:
        _msg.pop(0)
        user_choice = _msg[0].lower()
        if user_choice in ['rock', 'paper', 'scissors']:
            bot_choice = ran.choice(['rock', 'paper', 'scissors'])
            result = data.rps_result['{}_{}'.format(user_choice, bot_choice)]
            if result['type'] == 0:
                await _dclient.send_message(_channel, result['response'].format(ran.choice(data.rps_tie_response)))
            elif result['type'] == 1:
                await _dclient.send_message(_channel, result['response'].format(ran.choice(data.rps_bot_response)))
            else:
                await _dclient.send_message(_channel, result['response'].format(ran.choice(data.rps_player_response)))
        else:
            await _dclient.send_message(_channel, ':warning: {} Please choose an option between `rock`, `paper`, and '
                                                  '`scissors`.'.format(_mention))
    else:
        await _dclient.send_message(_channel, usage_msg('rps <rock|paper|scissors>', _mention, _cmdchar))

# Temp command
async def temp(_dclient, _channel, _mention, _cmdchar, _msg):
    _msg = _msg.split(' ')
    if len(_msg) == 4 and _msg[2].upper() in ['F', 'K', 'C'] and _msg[3].upper() in ['F', 'K', 'C']:
        _msg.pop(0)
        if is_float_value(_msg[0]):
            value = float(_msg[0])
            from_metric = _msg[1].upper()
            to_metric = _msg[2].upper()
            if from_metric != to_metric:
                result = 0
                if from_metric == 'F' and to_metric == 'K':
                    result = (value + 459.67) * (5 / 9)
                elif from_metric == 'F' and to_metric == 'C':
                    result = (value - 32) * .5556
                elif from_metric == 'K' and to_metric == 'F':
                    result = (value * (9 / 5)) - 459.67
                elif from_metric == 'K' and to_metric == 'C':
                    result = (value * (9 / 5)) - 273.15
                elif from_metric == 'C' and to_metric == 'F':
                    result = (value * 1.8) + 32
                elif from_metric == 'C' and to_metric == 'K':
                    result = (value + 273.15) * (5 / 9)
                await _dclient.send_message(_channel, data.temp_response['{}_{}'.format(from_metric, to_metric)]
                                            .format(value, result))
            else:
                await _dclient.send_message(_channel, ':warning: {} You can\'t convert the temperature that way, '
                                                      'silly!'.format(_mention))
        else:
            await _dclient.send_message(_channel, ':warning: {} The temperature value must be in numeric value!'
                                        .format(_mention))
    else:
        await _dclient.send_message(_channel, usage_msg('temp <#> <from F|K|C> <to F|K|C>', _mention, _cmdchar))

# Time command
async def time(_dclient, _channel, _mention, _cmdchar, _msg):
    if len(_msg.split(' ')) > 1:
        _msg = _msg[6:]
        google_api = geocoders.GoogleV3()
        try:
            place, (lat, lng) = google_api.geocode(_msg)
            timezone = google_api.timezone((lat, lng))
            time_now = datetime.now(timezone)
            await _dclient.send_message(_channel, '{} It is `{:%H:%M:%S}` or `{:%I:%M:%S %p}` in `{}` right now.'
                                        .format(_mention, time_now, time_now, _msg))
        except Exception:
            await _dclient.send_message(_channel, '{} Location `{}` unknown! Please try to be more specific. I rely on '
                                                  'Google maps to get coordinates for the timezone!'
                                        .format(_mention, _msg))
    else:
        await _dclient.send_message(_channel, usage_msg('time <location>', _mention, _cmdchar))

# Uptime command
async def uptime(_dclient, _channel, _mention, _uptime_start):
    dtime = datetime.now() - _uptime_start
    days = dtime.days
    if days == 0:
        dtime = datetime.strptime(str(dtime), "%H:%M:%S.%f")
        await _dclient.send_message(_channel, '{} I have been up for `{:d} hours`, `{:d} minutes`, and `{:d} seconds`.'
                                    .format(_mention, dtime.hour, dtime.minute, dtime.second))
    elif days == 1:
        dtime = datetime.strptime(str(dtime), "%d day, %H:%M:%S.%f")
        await _dclient.send_message(_channel, '{} I have been up for `1 day`, `{:d} hours`, `{:d} minutes`, and `{:d} '
                                              'seconds`.'.format(_mention, dtime.hour, dtime.minute, dtime.second))
    else:
        dtime = datetime.strptime(str(dtime), "%d day, %H:%M:%S.%f")
        await _dclient.send_message(_channel, '{} I have been up for `{:d} days`, `{:d} hours`, `{:d} minutes`, and '
                                              '`{:d} seconds`.'
                                    .format(_mention, days, dtime.hour, dtime.minute, dtime.second))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
from core import utils, ff_utils, ec2_utils
import boto3
from collections import defaultdict
from core.fastqc_utils import parse_qc_table
import requests
import os
import json

LOG = logging.getLogger(__name__)
s3 = boto3.resource('s3')
HIGLASS_SERVER = os.environ.get("HIGLASS_SERVER", "localhost")
HIGLASS_USER = os.environ.get("HIGLASS_USER")
HIGLASS_PASS = os.environ.get("HIGLASS_PASS")
HIGLASS_BUCKETS = ['elasticbeanstalk-fourfront-webprod-wfoutput',
                   'elasticbeanstalk-fourfront-webprod-wfoutput']

def donothing(status, sbg, ff_meta, ff_key=None):
    return None

def update_processed_file_metadata(status, pf, tibanna, export):

    # register mcool with fourfront-higlass
    if pf.file_format == "mcool" and export.bucket in HIGLASS_BUCKETS:
        payload = {"filepath": export.bucket + "/" + export.key,
                   "filetype": "cooler", "datatype": "matrix"}
        authentication = (HIGLASS_USER, HIGLASS_PASS)
        headers = {'Content-Type': 'application/json',
                   'Accept': 'application/json'}
        res = requests.post(HIGLASS_SERVER + '/api/v1/link_tile/',
                            data=json.dumps(payload), auth=authentication,
                            headers=headers)

        pf.__dict__['higlass_uid'] = res.json()['uuid']
        print(res)

    ff_key = tibanna.ff_keys
    try:
        pf.status = 'uploaded'
        pf.md5sum = export.md5
        pf.file_size = export.filesize
    except Exception as e:
        raise Exception("Unable to update processed file metadata json : %s" % e)
    try:
        pf.post(key=ff_key)
    except Exception as e:
        raise Exception("Unable to post processed file metadata : %s" % e)

    return pf

def qc_updater(status, wf_file, ff_meta, tibanna):
    if ff_meta.awsem_app_name == 'fastqc-0-11-4-1':
        return _qc_updater(status, wf_file, ff_meta, tibanna,
                           quality_metric='quality_metric_fastqc',
                           file_argument='input_fastq',
                           report_html='fastqc_report.html',
                           datafiles=['summary.txt', 'fastqc_data.txt'])
    elif ff_meta.awsem_app_name == 'pairsqc-single':
        file_argument = 'input_pairs'
        input_accession = str(wf_file.runner.inputfile_accessions[file_argument])
        return _qc_updater(status, wf_file, ff_meta, tibanna,
                           quality_metric="quality_metric_pairsqc",
                           file_argument=file_argument, report_html='pairsqc_report.html',
                           datafiles=[input_accession + '.summary.out'])
    elif ff_meta.awsem_app_name == 'repliseq-parta':
        return _qc_updater(status, wf_file, ff_meta, tibanna,
                           quality_metric='quality_metric_dedupqc_repliseq',
                           file_argument='filtered_sorted_deduped_bam',
                           datafiles=['summary.txt'])

def _qc_updater(status, wf_file, ff_meta, tibanna, quality_metric='quality_metric_fastqc',
                file_argument='input_fastq', report_html=None,
                datafiles=None):
    # avoid using [] as default argument
    if datafiles is None:
        datafiles = ['summary.txt', 'fastqc_data.txt']
    if status == 'uploading':
        # wait until this bad boy is finished
        return
    # keys
    ff_key = tibanna.ff_keys
    # move files to proper s3 location
    # need to remove sbg from this line
    accession = wf_file.runner.all_file_accessions[file_argument]
    zipped_report = wf_file.key
    files_to_parse = datafiles
    if report_html:
        files_to_parse.append(report_html)
    LOG.info("accession is %s" % accession)

    try:
        files = wf_file.s3.unzip_s3_to_s3(zipped_report, accession, files_to_parse,
                                          acl='public-read')
    except Exception as e:
        LOG.info(tibanna.s3.__dict__)
        raise Exception("%s (key={})\n".format(zipped_report) % e)

    # schema
    qc_schema = ff_utils.get_metadata("profiles/" + quality_metric + ".json", key=ff_key)

    # parse fastqc metadata
    LOG.info("files : %s" % str(files))
    filedata = [files[_]['data'] for _ in datafiles]
    meta = parse_qc_table(filedata,
                          url=files[report_html]['s3key'],
                          qc_schema=qc_schema.get('properties'))
    LOG.info("fastqc meta is %s" % meta)

    # post fastq metadata
    qc_meta = ff_utils.post_to_metadata(meta, quality_metric, key=ff_key)
    if qc_meta.get('@graph'):
        qc_meta = qc_meta['@graph'][0]

    LOG.info("qc_meta is %s" % qc_meta)
    # update original file as well
    try:
        original_file = ff_utils.get_metadata(accession, key=ff_key)
        LOG.info("original_file is %s" % original_file)
    except Exception as e:
        raise Exception("Couldn't get metadata for accession {} : ".format(accession) + str(e))
    patch_file = {'quality_metric': qc_meta['@id']}
    try:
        ff_utils.patch_metadata(patch_file, original_file['uuid'], key=ff_key)
    except Exception as e:
        raise Exception("patch_metadata failed in fastqc_updater." + str(e) +
                        "original_file ={}\n".format(str(original_file)))

    # patch the workflow run, value_qc is used to make drawing graphs easier.
    output_files = ff_meta.output_files
    output_files[0]['value_qc'] = qc_meta['@id']
    retval = {"output_quality_metrics": [{"name": quality_metric, "value": qc_meta['@id']}],
              'output_files': output_files}

    LOG.info("retval is %s" % retval)
    return retval

def _md5_updater(original_file, md5, content_md5):
    original_md5 = original_file.get('md5sum', False)
    original_content_md5 = original_file.get('content_md5sum', False)
    current_status = original_file.get('status', "uploading")
    new_file = {}

    if md5 and original_md5 and original_md5 != md5:
        # file status to be upload failed / md5 mismatch
        print("no matcho")
        return None
    elif md5 and not original_md5:
        new_file['md5sum'] = md5

    if content_md5 and original_content_md5 and original_content_md5 != content_md5:
        # file status to be upload failed / md5 mismatch
        print("no matcho")
        return None
    elif content_md5 and not original_content_md5:
        new_file['content_md5sum'] = content_md5

    if new_file:
        # change status to uploaded only if it is uploading or upload failed
        if current_status in ["uploading", "upload failed"]:
            new_file['status'] = 'uploaded'
    return new_file

def md5_updater(status, wf_file, ff_meta, tibanna):
    # get key
    ff_key = tibanna.ff_keys
    # get metadata about original input file
    accession = wf_file.runner.inputfile_accessions['input_file']
    original_file = ff_utils.get_metadata(accession, key=ff_key)

    if status.lower() == 'uploaded':
        md5_array = wf_file.read().split('\n')
        if not md5_array:
            print("report has no content")
            return md5_updater("upload failed", wf_file, ff_meta, tibanna)
        if len(md5_array) == 1:
            md5 = None
            content_md5 = md5_array[0]
        elif len(md5_array) > 1:
            md5 = md5_array[0]
            content_md5 = md5_array[1]
        new_file = _md5_updater(original_file, md5, content_md5)
        if new_file:
            try:
                ff_utils.patch_metadata(new_file, accession, key=ff_key)
            except Exception as e:
                # TODO specific excpetion
                # if patch fails try to patch worfklow status as failed
                new_file = {}
                new_file['status'] = 'upload failed'
                new_file['description'] = str(e)
                ff_utils.patch_metadata(new_file, original_file['uuid'], key=ff_key)
        else:
            md5_updater("upload failed", wf_file, ff_meta, tibanna)
    elif status == 'upload failed':
            new_file = {}
            new_file['status'] = 'upload failed'
            ff_utils.patch_metadata(new_file, original_file['uuid'], key=ff_key)

    # nothing to patch to ff_meta
    return None

def metadata_only(event):
    # just create a fake awsem config so the handler function does it's magic
    '''
    if not event.get('args'):
        event['args'] = {'app_name': event['ff_meta'].get('awsem_app_name'),
                         'output_S3_bucket': 'metadata_only',
                         'output_target': {'metadata_only': 'metadata_only'}
                         }

    if not event.get('config'):
        event['config'] = {'runmode': 'metadata_only'}
    '''

    return real_handler(event, None)

@utils.powerup('update_ffmeta_awsem', metadata_only)
def handler(event, context):
    return real_handler(event, context)

def real_handler(event, context):
    # check the status and other details of import
    '''
    this is to check if the task run is done:
    http://docs.sevenbridges.com/reference#get-task-execution-details
    '''
    # get data
    # used to automatically determine the environment
    tibanna_settings = event.get('_tibanna', {})
    tibanna = utils.Tibanna(**tibanna_settings)
    # sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys, **event.get('workflow'))
    ff_meta = ff_utils.create_ffmeta_awsem(app_name=event.get('ff_meta').get('awsem_app_name'), **event.get('ff_meta'))
    pf_meta = [ff_utils.ProcessedFileMetadata(**_) for _ in event.get('pf_meta')]

    # ensure this bad boy is always initialized
    patch_meta = False
    awsem = ec2_utils.Awsem(event)

    # go through this and replace export_report with awsf format
    # actually interface should be look through ff_meta files and call
    # give me the status of this thing from the runner, and runner.output_files.length
    # so we just build a runner with interface to sbg and awsem
    # runner.output_files.length()
    # runner.output_files.file.status
    # runner.output_files.file.loc
    # runner.output_files.file.get

    awsem_output = awsem.output_files()
    ff_output = len(ff_meta.output_files)
    if len(awsem_output) != ff_output:
        ff_meta.run_status = 'error'
        ff_meta.description = "%d files output expected %s" % (ff_output, len(awsem_output))
        ff_meta.post(key=tibanna.ff_keys)
        raise Exception("Failing the workflow because outputed files = %d and ffmeta = %d" %
                        (awsem_output, ff_output))

    for _, export in awsem_output.iteritems():
        upload_key = export.key
        status = export.status
        print("export res is %s", status)
        if status == 'COMPLETED':
            patch_meta = OUTFILE_UPDATERS[export.output_type]('uploaded', export, ff_meta, tibanna)
            if pf_meta:
                for pf in pf_meta:
                    if pf.accession == export.accession:
                        pf = update_processed_file_metadata('uploaded', pf, tibanna, export)
        elif status in ['FAILED']:
            patch_meta = OUTFILE_UPDATERS[export.output_type]('upload failed', export, ff_meta, tibanna)
            ff_meta.run_status = 'error'
            ff_meta.post(key=tibanna.ff_keys)
            raise Exception("Failed to export file %s" % (upload_key))

    # if we got all the exports let's go ahead and update our ff_metadata object
    ff_meta.run_status = "complete"

    # allow for a simple way for updater to add appropriate meta_data
    if patch_meta:
        ff_meta.__dict__.update(patch_meta)

    # add postrunjson log file to ff_meta as a url
    ff_meta.awsem_postrun_json = get_postrunjson_url(event)

    # make all the file export meta-data stuff here
    # TODO: fix bugs with ff_meta mapping for output and input file
    try:
        ff_meta.post(key=tibanna.ff_keys)
    except Exception as e:
        raise Exception("Failed to update run_status %s" % str(e))

    event['ff_meta'] = ff_meta.as_dict()
    event['pf_meta'] = [_.as_dict() for _ in pf_meta]

    return event

def get_postrunjson_url(event):
    try:
        logbucket = event['config']['log_bucket']
        jobid = event['jobid']
        postrunjson_url = 'https://s3.amazonaws.com/' + logbucket + '/' + jobid + '.postrun.json'
        return postrunjson_url
    except Exception as e:
        # we don't need this for pseudo runs so just ignore
        if event.get('metadata_only'):
            return ''
        else:
            raise e

# Cardinal knowledge of all workflow updaters
OUTFILE_UPDATERS = defaultdict(lambda: donothing)
OUTFILE_UPDATERS['Output report file'] = md5_updater
OUTFILE_UPDATERS['Output QC file'] = qc_updater
<commit_msg>update trigger to point to new md5 workflow<commit_after># -*- coding: utf-8 -*-
import logging
from core import utils, ff_utils, ec2_utils
import boto3
from collections import defaultdict
from core.fastqc_utils import parse_qc_table
import requests
import os
import json

LOG = logging.getLogger(__name__)
s3 = boto3.resource('s3')
HIGLASS_SERVER = os.environ.get("HIGLASS_SERVER", "localhost")
HIGLASS_USER = os.environ.get("HIGLASS_USER")
HIGLASS_PASS = os.environ.get("HIGLASS_PASS")
HIGLASS_BUCKETS = ['elasticbeanstalk-fourfront-webprod-wfoutput',
                   'elasticbeanstalk-fourfront-webprod-wfoutput']

def donothing(status, sbg, ff_meta, ff_key=None):
    return None

def update_processed_file_metadata(status, pf, tibanna, export):

    # register mcool with fourfront-higlass
    if pf.file_format == "mcool" and export.bucket in HIGLASS_BUCKETS:
        payload = {"filepath": export.bucket + "/" + export.key,
                   "filetype": "cooler", "datatype": "matrix"}
        authentication = (HIGLASS_USER, HIGLASS_PASS)
        headers = {'Content-Type': 'application/json',
                   'Accept': 'application/json'}
        res = requests.post(HIGLASS_SERVER + '/api/v1/link_tile/',
                            data=json.dumps(payload), auth=authentication,
                            headers=headers)

        pf.__dict__['higlass_uid'] = res.json()['uuid']
        print(res)

    ff_key = tibanna.ff_keys
    try:
        pf.status = 'uploaded'
        pf.md5sum = export.md5
        pf.file_size = export.filesize
    except Exception as e:
        raise Exception("Unable to update processed file metadata json : %s" % e)
    try:
        pf.post(key=ff_key)
    except Exception as e:
        raise Exception("Unable to post processed file metadata : %s" % e)

    return pf

def qc_updater(status, wf_file, ff_meta, tibanna):
    if ff_meta.awsem_app_name == 'fastqc-0-11-4-1':
        return _qc_updater(status, wf_file, ff_meta, tibanna,
                           quality_metric='quality_metric_fastqc',
                           file_argument='input_fastq',
                           report_html='fastqc_report.html',
                           datafiles=['summary.txt', 'fastqc_data.txt'])
    elif ff_meta.awsem_app_name == 'pairsqc-single':
        file_argument = 'input_pairs'
        input_accession = str(wf_file.runner.inputfile_accessions[file_argument])
        return _qc_updater(status, wf_file, ff_meta, tibanna,
                           quality_metric="quality_metric_pairsqc",
                           file_argument=file_argument, report_html='pairsqc_report.html',
                           datafiles=[input_accession + '.summary.out'])
    elif ff_meta.awsem_app_name == 'repliseq-parta':
        return _qc_updater(status, wf_file, ff_meta, tibanna,
                           quality_metric='quality_metric_dedupqc_repliseq',
                           file_argument='filtered_sorted_deduped_bam',
                           datafiles=['summary.txt'])

def _qc_updater(status, wf_file, ff_meta, tibanna, quality_metric='quality_metric_fastqc',
                file_argument='input_fastq', report_html=None,
                datafiles=None):
    # avoid using [] as default argument
    if datafiles is None:
        datafiles = ['summary.txt', 'fastqc_data.txt']
    if status == 'uploading':
        # wait until this bad boy is finished
        return
    # keys
    ff_key = tibanna.ff_keys
    # move files to proper s3 location
    # need to remove sbg from this line
    accession = wf_file.runner.all_file_accessions[file_argument]
    zipped_report = wf_file.key
    files_to_parse = datafiles
    if report_html:
        files_to_parse.append(report_html)
    LOG.info("accession is %s" % accession)

    try:
        files = wf_file.s3.unzip_s3_to_s3(zipped_report, accession, files_to_parse,
                                          acl='public-read')
    except Exception as e:
        LOG.info(tibanna.s3.__dict__)
        raise Exception("%s (key={})\n".format(zipped_report) % e)

    # schema
    qc_schema = ff_utils.get_metadata("profiles/" + quality_metric + ".json", key=ff_key)

    # parse fastqc metadata
    LOG.info("files : %s" % str(files))
    filedata = [files[_]['data'] for _ in datafiles]
    meta = parse_qc_table(filedata,
                          url=files[report_html]['s3key'],
                          qc_schema=qc_schema.get('properties'))
    LOG.info("fastqc meta is %s" % meta)

    # post fastq metadata
    qc_meta = ff_utils.post_to_metadata(meta, quality_metric, key=ff_key)
    if qc_meta.get('@graph'):
        qc_meta = qc_meta['@graph'][0]

    LOG.info("qc_meta is %s" % qc_meta)
    # update original file as well
    try:
        original_file = ff_utils.get_metadata(accession, key=ff_key)
        LOG.info("original_file is %s" % original_file)
    except Exception as e:
        raise Exception("Couldn't get metadata for accession {} : ".format(accession) + str(e))
    patch_file = {'quality_metric': qc_meta['@id']}
    try:
        ff_utils.patch_metadata(patch_file, original_file['uuid'], key=ff_key)
    except Exception as e:
        raise Exception("patch_metadata failed in fastqc_updater." + str(e) +
                        "original_file ={}\n".format(str(original_file)))

    # patch the workflow run, value_qc is used to make drawing graphs easier.
    output_files = ff_meta.output_files
    output_files[0]['value_qc'] = qc_meta['@id']
    retval = {"output_quality_metrics": [{"name": quality_metric, "value": qc_meta['@id']}],
              'output_files': output_files}

    LOG.info("retval is %s" % retval)
    return retval

def _md5_updater(original_file, md5, content_md5):
    original_md5 = original_file.get('md5sum', False)
    original_content_md5 = original_file.get('content_md5sum', False)
    current_status = original_file.get('status', "uploading")
    new_file = {}

    if md5 and original_md5 and original_md5 != md5:
        # file status to be upload failed / md5 mismatch
        print("no matcho")
        return None
    elif md5 and not original_md5:
        new_file['md5sum'] = md5

    if content_md5 and original_content_md5 and original_content_md5 != content_md5:
        # file status to be upload failed / md5 mismatch
        print("no matcho")
        return None
    elif content_md5 and not original_content_md5:
        new_file['content_md5sum'] = content_md5

    if new_file:
        # change status to uploaded only if it is uploading or upload failed
        if current_status in ["uploading", "upload failed"]:
            new_file['status'] = 'uploaded'
    return new_file

def md5_updater(status, wf_file, ff_meta, tibanna):
    # get key
    ff_key = tibanna.ff_keys
    # get metadata about original input file
    accession = wf_file.runner.inputfile_accessions['input_file']
    original_file = ff_utils.get_metadata(accession, key=ff_key)

    if status.lower() == 'uploaded':
        md5_array = wf_file.read().split('\n')
        if not md5_array:
            print("report has no content")
            return md5_updater("upload failed", wf_file, ff_meta, tibanna)
        if len(md5_array) == 1:
            md5 = None
            content_md5 = md5_array[0]
        elif len(md5_array) > 1:
            md5 = md5_array[0]
            content_md5 = md5_array[1]
        new_file = _md5_updater(original_file, md5, content_md5)
        if new_file:
            try:
                ff_utils.patch_metadata(new_file, accession, key=ff_key)
            except Exception as e:
                # TODO specific excpetion
                # if patch fails try to patch worfklow status as failed
                new_file = {}
                new_file['status'] = 'upload failed'
                new_file['description'] = str(e)
                ff_utils.patch_metadata(new_file, original_file['uuid'], key=ff_key)
        else:
            md5_updater("upload failed", wf_file, ff_meta, tibanna)
    elif status == 'upload failed':
            new_file = {}
            new_file['status'] = 'upload failed'
            ff_utils.patch_metadata(new_file, original_file['uuid'], key=ff_key)

    # nothing to patch to ff_meta
    return None

def metadata_only(event):
    # just create a fake awsem config so the handler function does it's magic
    '''
    if not event.get('args'):
        event['args'] = {'app_name': event['ff_meta'].get('awsem_app_name'),
                         'output_S3_bucket': 'metadata_only',
                         'output_target': {'metadata_only': 'metadata_only'}
                         }

    if not event.get('config'):
        event['config'] = {'runmode': 'metadata_only'}
    '''

    return real_handler(event, None)

@utils.powerup('update_ffmeta_awsem', metadata_only)
def handler(event, context):
    return real_handler(event, context)

def real_handler(event, context):
    # check the status and other details of import
    '''
    this is to check if the task run is done:
    http://docs.sevenbridges.com/reference#get-task-execution-details
    '''
    # get data
    # used to automatically determine the environment
    tibanna_settings = event.get('_tibanna', {})
    tibanna = utils.Tibanna(**tibanna_settings)
    # sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys, **event.get('workflow'))
    ff_meta = ff_utils.create_ffmeta_awsem(app_name=event.get('ff_meta').get('awsem_app_name'), **event.get('ff_meta'))
    pf_meta = [ff_utils.ProcessedFileMetadata(**_) for _ in event.get('pf_meta')]

    # ensure this bad boy is always initialized
    patch_meta = False
    awsem = ec2_utils.Awsem(event)

    # go through this and replace export_report with awsf format
    # actually interface should be look through ff_meta files and call
    # give me the status of this thing from the runner, and runner.output_files.length
    # so we just build a runner with interface to sbg and awsem
    # runner.output_files.length()
    # runner.output_files.file.status
    # runner.output_files.file.loc
    # runner.output_files.file.get

    awsem_output = awsem.output_files()
    ff_output = len(ff_meta.output_files)
    if len(awsem_output) != ff_output:
        ff_meta.run_status = 'error'
        ff_meta.description = "%d files output expected %s" % (ff_output, len(awsem_output))
        ff_meta.post(key=tibanna.ff_keys)
        raise Exception("Failing the workflow because outputed files = %d and ffmeta = %d" %
                        (awsem_output, ff_output))

    for _, export in awsem_output.iteritems():
        upload_key = export.key
        status = export.status
        print("export res is %s", status)
        if status == 'COMPLETED':
            patch_meta = OUTFILE_UPDATERS[export.output_type]('uploaded', export, ff_meta, tibanna)
            if pf_meta:
                for pf in pf_meta:
                    if pf.accession == export.accession:
                        pf = update_processed_file_metadata('uploaded', pf, tibanna, export)
        elif status in ['FAILED']:
            patch_meta = OUTFILE_UPDATERS[export.output_type]('upload failed', export, ff_meta, tibanna)
            ff_meta.run_status = 'error'
            ff_meta.post(key=tibanna.ff_keys)
            raise Exception("Failed to export file %s" % (upload_key))

    # if we got all the exports let's go ahead and update our ff_metadata object
    ff_meta.run_status = "complete"

    # allow for a simple way for updater to add appropriate meta_data
    if patch_meta:
        ff_meta.__dict__.update(patch_meta)

    # add postrunjson log file to ff_meta as a url
    ff_meta.awsem_postrun_json = get_postrunjson_url(event)

    # make all the file export meta-data stuff here
    # TODO: fix bugs with ff_meta mapping for output and input file
    try:
        ff_meta.post(key=tibanna.ff_keys)
    except Exception as e:
        raise Exception("Failed to update run_status %s" % str(e))

    event['ff_meta'] = ff_meta.as_dict()
    event['pf_meta'] = [_.as_dict() for _ in pf_meta]

    # call next workflow if asked to do so
    if event['trigger_workflow']:

        pass

    return event

def get_postrunjson_url(event):
    try:
        logbucket = event['config']['log_bucket']
        jobid = event['jobid']
        postrunjson_url = 'https://s3.amazonaws.com/' + logbucket + '/' + jobid + '.postrun.json'
        return postrunjson_url
    except Exception as e:
        # we don't need this for pseudo runs so just ignore
        if event.get('metadata_only'):
            return ''
        else:
            raise e

# Cardinal knowledge of all workflow updaters
OUTFILE_UPDATERS = defaultdict(lambda: donothing)
OUTFILE_UPDATERS['Output report file'] = md5_updater
OUTFILE_UPDATERS['Output QC file'] = qc_updater
<|endoftext|>