<commit_before>import sys
import matplotlib.pyplot as plt

sys.path.append('..')
from inverse_covariance.profiling import (
    MonteCarloProfile,
    support_false_positive_count,
    support_false_negative_count,
    support_difference_count,
    has_exact_support,
    has_approx_support,
    error_fro,
    LatticeGraph,
)

plt.ion()

def r_input(val):
    if sys.version_info[0] >= 3:
        return eval(input(val))

    return raw_input(val)

###############################################################################
# Setup metrics

metrics = {
    'fp_rate': support_false_positive_count,
    'fn_rate': support_false_negative_count,
    'support_error': support_difference_count,
    'prob_exact_support': has_exact_support,
    'prob_approx_support': has_approx_support,
    'frobenius': error_fro,
}

###############################################################################
# Run MC trials

mc = MonteCarloProfile(n_features=50, n_trials=10, graph=LatticeGraph(),
                       n_samples_grid=10, alpha_grid=5, metrics=metrics,
                       verbose=True, n_jobs=4)
mc.fit()

###############################################################################
# Plot results for each metric

for key in metrics:
    plt.figure()
    plt.plot(mc.grid_, mc.results_[key].T, linewidth=2)
    plt.title('metric = {}'.format(key))
    legend_items = ['alpha={}'.format(a) for a in mc.alphas_]
    plt.legend(legend_items)
    plt.show()

r_input('Any key to exit.')
<commit_msg>Added docstring<commit_after>"""
Example of Montecarlo Benchmarking
=============================

Run a MonteCarlo simulation on synthetic network structure and compare
estimation and model selection errors

"""

import sys
import matplotlib.pyplot as plt

sys.path.append('..')
from inverse_covariance.profiling import (
    MonteCarloProfile,
    support_false_positive_count,
    support_false_negative_count,
    support_difference_count,
    has_exact_support,
    has_approx_support,
    error_fro,
    LatticeGraph,
)

plt.ion()

def r_input(val):
    if sys.version_info[0] >= 3:
        return eval(input(val))

    return raw_input(val)

###############################################################################
# Setup metrics

metrics = {
    'fp_rate': support_false_positive_count,
    'fn_rate': support_false_negative_count,
    'support_error': support_difference_count,
    'prob_exact_support': has_exact_support,
    'prob_approx_support': has_approx_support,
    'frobenius': error_fro,
}

###############################################################################
# Run MC trials

mc = MonteCarloProfile(n_features=50, n_trials=10, graph=LatticeGraph(),
                       n_samples_grid=10, alpha_grid=5, metrics=metrics,
                       verbose=True, n_jobs=4)
mc.fit()

###############################################################################
# Plot results for each metric

for key in metrics:
    plt.figure()
    plt.plot(mc.grid_, mc.results_[key].T, linewidth=2)
    plt.title('metric = {}'.format(key))
    legend_items = ['alpha={}'.format(a) for a in mc.alphas_]
    plt.legend(legend_items)
    plt.show()

r_input('Any key to exit.')
<|endoftext|>


<commit_before>import argparse
import matplotlib.pyplot as plot

import os
import chainer

from chainercv.links import FasterRCNNVGG16
from chainercv import utils
from chainercv.visualizations import vis_bbox

import download_model
TRAINED_MODEL_DEFAULT = 'trained_model/snapshot_model.npz'
MODEL_URL = 'http://nixeneko.2-d.jp/hatenablog/20170724_facedetection_model/snapshot_model.npz'

def main():
    chainer.config.train = False

    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--pretrained_model', default=TRAINED_MODEL_DEFAULT)
    parser.add_argument('image')
    args = parser.parse_args()

    if args.pretrained_model == TRAINED_MODEL_DEFAULT and \
       not os.path.exists(TRAINED_MODEL_DEFAULT):
        download_model.download_model(MODEL_URL, TRAINED_MODEL_DEFAULT)

    model = FasterRCNNVGG16(
        n_fg_class=1,
        pretrained_model=args.pretrained_model)

    if args.gpu >= 0:
        model.to_gpu(args.gpu)
        chainer.cuda.get_device(args.gpu).use()

    img = utils.read_image(args.image, color=True)
    bboxes, labels, scores = model.predict([img])
    bbox, label, score = bboxes[0], labels[0], scores[0]

    vis_bbox(
        img, bbox, label, score, label_names=('face',))
    plot.show()

if __name__ == '__main__':
    main()
<commit_msg>add workaround for the newer version (maybe v0.7.0 and higer) of chainercv<commit_after>import argparse
import matplotlib.pyplot as plot

import os
import chainer

from chainercv.links import FasterRCNNVGG16
from chainercv import utils
from chainercv.visualizations import vis_bbox

import download_model
TRAINED_MODEL_DEFAULT = 'trained_model/snapshot_model.npz'
MODEL_URL = 'http://nixeneko.2-d.jp/hatenablog/20170724_facedetection_model/snapshot_model.npz'

# A pretrained model for the higher version of chainercv.
TRAINED_MODEL_NEW = 'trained_model/snapshot_model_20180404.npz'
MODEL_URL_NEW = 'http://nixeneko.2-d.jp/hatenablog/20170724_facedetection_model/snapshot_model_20180404.npz'

def main():
    chainer.config.train = False

    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--pretrained_model', default=TRAINED_MODEL_DEFAULT)
    parser.add_argument('image')
    args = parser.parse_args()

    if args.pretrained_model == TRAINED_MODEL_DEFAULT and \
       not os.path.exists(TRAINED_MODEL_DEFAULT):
        download_model.download_model(MODEL_URL, TRAINED_MODEL_DEFAULT)
    try:
        model = FasterRCNNVGG16(
            n_fg_class=1,
            pretrained_model=args.pretrained_model)

    # Workaround for the newer version of chainercv (maybe v0.7.0 and higher)
    except KeyError:
        if args.pretrained_model == TRAINED_MODEL_DEFAULT and \
           not os.path.exists(TRAINED_MODEL_NEW):
            download_model.download_model(MODEL_URL_NEW, TRAINED_MODEL_NEW)
            model = FasterRCNNVGG16(
                n_fg_class=1,
                pretrained_model=TRAINED_MODEL_NEW)
        else:
            raise

    if args.gpu >= 0:
        model.to_gpu(args.gpu)
        chainer.cuda.get_device(args.gpu).use()

    img = utils.read_image(args.image, color=True)
    bboxes, labels, scores = model.predict([img])
    bbox, label, score = bboxes[0], labels[0], scores[0]

    vis_bbox(
        img, bbox, label, score, label_names=('face',))
    plot.show()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2017 Weitian LI <liweitianux@live.com>
# MIT license
#
# 2017-02-27

"""
Reprocess Chandra ACIS raw "secondary" (level=1) data using the
CIAO contrib tool ``chandra_repro``, and build the ``manifest.yaml``.
"""

import os
import argparse
import subprocess
import logging
from glob import glob

from _context import acispy
from acispy.manifest import get_manifest
from acispy.pfiles import setup_pfiles

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def repro_acis(indir=".", outdir="repro", clobber=False):
    """
    Reprocess Chandra ACIS raw level=1 data using ``chandra_repro``.
    """
    logger.info("Reprocess Chandra ACIS raw level=1 data ...")
    clobber = "yes" if clobber else "no"
    subprocess.check_call(["punlearn", "ardlib"])
    subprocess.check_call(["punlearn", "chandra_repro"])
    subprocess.check_call([
        "chandra_repro", "indir=%s" % indir, "outdir=%s" % outdir,
        "verbose=2", "clobber=%s" % clobber
    ])
    logger.info("Fix asol.lis by striping the absolute directory ...")
    asol_lis = glob("%s/acisf*_asol1.lis" % outdir)[0]
    asol = [os.path.basename(fp) for fp in open(asol_lis).readlines()]
    os.rename(asol_lis, asol_lis+".orig")
    open(asol_lis, "w").writelines(asol)

def build_manifest(reprodir="repro", manifestfile="manifest.yaml"):
    """
    Build the ``manifest.yaml`` with reprocessed products.
    """
    logger.info("Build manifest with reprocessed products ...")
    if os.path.exists(manifestfile):
        logger.info("Use existing manifest file: %s" % manifestfile)
    else:
        logger.info("Create a new manifest file: %s" % manifestfile)
        open(manifestfile, "w").close()

    manifest = get_manifest(manifestfile)
    key = "directory"
    manifest.set(key, os.path.basename(os.getcwd()))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "evt2"
    manifest.setpath(key, glob("%s/acisf*_repro_evt2.fits" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "bpix"
    manifest.setpath(key, glob("%s/acisf*_repro_bpix1.fits" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "asol"
    manifest.setpath(key, glob("%s/pcadf*_asol1.fits" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "asol_lis"
    manifest.setpath(key, glob("%s/acisf*_asol1.lis" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "fov"
    manifest.setpath(key, glob("%s/acisf*_repro_fov1.fits" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "pbk"
    manifest.setpath(key, glob("%s/acisf*_pbk0.fits" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))
    key = "msk"
    manifest.setpath(key, glob("%s/acisf*_msk1.fits" % reprodir))
    logger.info("Added '%s' to manifest: %s" % (key, manifest.get(key)))

def main():
    parser = argparse.ArgumentParser(
            description="Extract surface brightness profile (SBP)")
    parser.add_argument("-C", "--clobber", dest="clobber", action="store_true",
                        help="overwrite existing file")
    args = parser.parse_args()

    setup_pfiles(["chandra_repro", "ardlib"])

    repro_acis(clobber=args.clobber)
    build_manifest()

if __name__ == "__main__":
    main()
<commit_msg>repro_acis.py: some cleanups by using update_manifest<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2017 Weitian LI <weitian@aaronly.me>
# MIT license
#

"""
Reprocess Chandra ACIS raw "secondary" (level=1) data using the
CIAO contrib tool ``chandra_repro``, and build the ``manifest.yaml``.
"""

import os
import argparse
import subprocess
import logging
from glob import glob

from _context import acispy
from acispy.manifest import get_manifest
from acispy.pfiles import setup_pfiles

import update_manifest

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def repro_acis(indir=".", outdir="repro", clobber=False):
    """
    Reprocess Chandra ACIS raw level=1 data using ``chandra_repro``.
    """
    logger.info("Reprocess Chandra ACIS raw level=1 data ...")
    clobber = "yes" if clobber else "no"
    subprocess.check_call(["punlearn", "ardlib"])
    subprocess.check_call(["punlearn", "chandra_repro"])
    subprocess.check_call([
        "chandra_repro", "indir=%s" % indir, "outdir=%s" % outdir,
        "verbose=2", "clobber=%s" % clobber
    ])
    logger.info("Fix asol.lis by striping the absolute directory ...")
    asol_lis = glob("%s/acisf*_asol1.lis" % outdir)[0]
    asol = [os.path.basename(fp) for fp in open(asol_lis).readlines()]
    os.rename(asol_lis, asol_lis+".orig")
    open(asol_lis, "w").writelines(asol)

def build_manifest(reprodir="repro", manifestfile="manifest.yaml"):
    """
    Build the ``manifest.yaml`` with reprocessed products.
    """
    logger.info("Build manifest with reprocessed products ...")
    if os.path.exists(manifestfile):
        logger.info("Use existing manifest file: %s" % manifestfile)
    else:
        logger.info("Create a new manifest file: %s" % manifestfile)
        open(manifestfile, "w").close()

    manifest = get_manifest(manifestfile)
    update_manifest.add_directory(manifest)
    update_manifest.add_repro(reprodir=reprodir, manifest=manifest)

def main():
    parser = argparse.ArgumentParser(
            description="Reprocess ACIS raw data & create manifest.yaml")
    parser.add_argument("-C", "--clobber",
                        dest="clobber", action="store_true",
                        help="overwrite existing file")
    args = parser.parse_args()

    setup_pfiles(["chandra_repro", "ardlib"])

    repro_acis(clobber=args.clobber)
    build_manifest()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8 tabstop=4 expandtab shiftwidth=4 softtabstop=4
#
# Copyright © Mike Dacre <mike.dacre@gmail.com>
#
# Distributed under terms of the MIT license
"""
#=======================================================================================#
#                                                                                #
#          FILE: run_matlab (python 3)                                                  #
#        AUTHOR: Michael D Dacre, mike.dacre@gmail.com                                  #
#  ORGANIZATION: Stanford University                                                    #
#       LICENSE: MIT License, Property of Stanford, Use as you wish                     #
#       VERSION: 0.1                                                                    #
#       CREATED: 2014-08-22 16:26                                                       #
# Last modified: 2014-08-22 16:56
#                                                                                #
#   DESCRIPTION: Create a bunch of temporary matlab scripts to call some other          #
#                matlab script and then submit to the cluster.                          #
#                                                                                #
#                Requires that the matlab function be written to accept imput           #
#                variables.                                                             #
#                                                                                #
#                Right now only works with torque jobs tools and requires               #
#                pbs_torque from                                                        #
#                https://github.com/MikeDacre/torque_queue_manager and logging          #
#                functions from http://j.mp/python_logme                                #
#                                                                                #
#         USAGE: -p or --path allows the addition of multiple matlab paths              #
#                -v or --variables is a list of variables to pass to the function       #
#                    This list must be space separated. Each space separated item will  #
#                    be run as a separate matlab job.                                   #
#                    You may comma separate variables if you wish those variables to    #
#                    passed as multiple arguments to a single function (e.g. not as     #
#                    separate jobs.                                                     #
#                                                                                #
#                <function> is a positional arg and is the name of the function to run  #
#                                                                                #
#                Run as a script or import as a module.  See '-h' or 'help' for usage   #
#                                                                                #
#=======================================================================================#
"""
import logme
from pbs_torque import job

def _get_args():
    """Command Line Argument Parsing"""
    import argparse, sys

    parser = argparse.ArgumentParser(
                 description=__doc__,
                 formatter_class=argparse.RawDescriptionHelpFormatter)

    # Optional Arguments
    parser.add_argument('-p', '--path', nargs='?',
                        help="Comma or space separated list of matlab paths")
    parser.add_argument('-V', '--variables', nargs='?',
                        help="Space separated list variables to pass to function")
    parser.add_argument('-v', dest='verbose', help="Verbose output")

    # Function name
    parser.add_argument('function',
                        help="Name of the matlab function to run")

    # Optional Files
    parser.add_argument('-l', '--logfile', nargs='?',
                        help="Log File, Default STDERR (append mode)")

    return parser

# Main function for direct running
def main():
    """Run directly"""
    # Get commandline arguments
    parser = _get_args()
    args = parser.parse_args()

# The end
if __name__ == '__main__':
    main()
<commit_msg>Small changes<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:fenc=utf-8 tabstop=4 expandtab shiftwidth=4 softtabstop=4
#
# Copyright © Mike Dacre <mike.dacre@gmail.com>
#
# Distributed under terms of the MIT license
"""
#=======================================================================================#
#                                                                                #
#          FILE: run_matlab (python 3)                                                  #
#        AUTHOR: Michael D Dacre, mike.dacre@gmail.com                                  #
#  ORGANIZATION: Stanford University                                                    #
#       LICENSE: MIT License, Property of Stanford, Use as you wish                     #
#       VERSION: 0.1                                                                    #
#       CREATED: 2014-08-22 16:26                                                       #
# Last modified: 2014-08-25 11:48
#                                                                                #
#   DESCRIPTION: Create a bunch of temporary matlab scripts to call some other          #
#                matlab script and then submit to the cluster.                          #
#                                                                                #
#                Requires that the matlab function be written to accept imput           #
#                variables.                                                             #
#                                                                                #
#                Right now only works with torque jobs tools and requires               #
#                pbs_torque from                                                        #
#                https://github.com/MikeDacre/torque_queue_manager and logging          #
#                functions from http://j.mp/python_logme                                #
#                                                                                #
#         USAGE: -p or --path allows the addition of multiple matlab paths              #
#                -v or --variables is a list of variables to pass to the function       #
#                    This list must be space separated. Each space separated item will  #
#                    be run as a separate matlab job.                                   #
#                    You may comma separate variables if you wish those variables to    #
#                    passed as multiple arguments to a single function (e.g. not as     #
#                    separate jobs.                                                     #
#                                                                                #
#                <function> is a positional arg and is the name of the function to run  #
#                                                                                #
#                Run as a script or import as a module.  See '-h' or 'help' for usage   #
#                                                                                #
#=======================================================================================#
"""
import logme
from pbs_torque import job

def _get_args():
    """Command Line Argument Parsing"""
    import argparse, sys

    parser = argparse.ArgumentParser(
                 description=__doc__,
                 formatter_class=argparse.RawDescriptionHelpFormatter)

    # Optional Arguments
    parser.add_argument('-p', '--path', nargs='?',
                        help="Comma or space separated list of matlab paths")
    parser.add_argument('-p', '--variables', nargs='?',
                        help="Space separated list variables to pass to function. Or use STDIN")
    parser.add_argument('-v', dest='verbose', help="Verbose output")

    # Function name
    parser.add_argument('function',
                        help="Name of the matlab function to run")

    # Optional Files
    parser.add_argument('-l', '--logfile', nargs='?',
                        help="Log File, Default STDERR (append mode)")

    return parser

# Main function for direct running
def main():
    """Run directly"""
    # Get commandline arguments
    parser = _get_args()
    args = parser.parse_args()

# The end
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># flake8: noqa

import warnings

__version__ = '0.31.1'

# explainers
from .explainers.kernel import KernelExplainer, kmeans
from .explainers.sampling import SamplingExplainer
from .explainers.tree import TreeExplainer, Tree
from .explainers.deep import DeepExplainer
from .explainers.gradient import GradientExplainer
from .explainers.linear import LinearExplainer
from .explainers.partition import PartitionExplainer
from .explainers.bruteforce import BruteForceExplainer
from .explainers import other

# plotting (only loaded if matplotlib is present)
def unsupported(*args, **kwargs):
    warnings.warn("matplotlib is not installed so plotting is not available! Run `pip install matplotlib` to fix this.")

try:
    import matplotlib
    have_matplotlib = True
except ImportError:
    have_matplotlib = False
if have_matplotlib:
    from .plots.summary import summary_plot
    from .plots.decision import decision_plot, multioutput_decision_plot
    from .plots.dependence import dependence_plot
    from .plots.force import force_plot, initjs, save_html
    from .plots.image import image_plot
    from .plots.monitoring import monitoring_plot
    from .plots.embedding import embedding_plot
    from .plots.partial_dependence import partial_dependence_plot
    from .plots.bar import bar_plot
else:
    summary_plot = unsupported
    decision_plot = unsupported
    multioutput_decision_plot = unsupported
    dependence_plot = unsupported
    force_plot = unsupported
    initjs = unsupported
    save_html = unsupported
    image_plot = unsupported
    monitoring_plot = unsupported
    embedding_plot = unsupported
    partial_dependence_plot = unsupported
    bar_plot = unsupported

# other stuff :)
from . import datasets
#from . import benchmark
from .common import approximate_interactions, hclust_ordering, sample
<commit_msg>Bump version<commit_after># flake8: noqa

import warnings

__version__ = '0.32.0'

# explainers
from .explainers.kernel import KernelExplainer, kmeans
from .explainers.sampling import SamplingExplainer
from .explainers.tree import TreeExplainer, Tree
from .explainers.deep import DeepExplainer
from .explainers.gradient import GradientExplainer
from .explainers.linear import LinearExplainer
from .explainers.partition import PartitionExplainer
from .explainers.bruteforce import BruteForceExplainer
from .explainers import other

# plotting (only loaded if matplotlib is present)
def unsupported(*args, **kwargs):
    warnings.warn("matplotlib is not installed so plotting is not available! Run `pip install matplotlib` to fix this.")

try:
    import matplotlib
    have_matplotlib = True
except ImportError:
    have_matplotlib = False
if have_matplotlib:
    from .plots.summary import summary_plot
    from .plots.decision import decision_plot, multioutput_decision_plot
    from .plots.dependence import dependence_plot
    from .plots.force import force_plot, initjs, save_html
    from .plots.image import image_plot
    from .plots.monitoring import monitoring_plot
    from .plots.embedding import embedding_plot
    from .plots.partial_dependence import partial_dependence_plot
    from .plots.bar import bar_plot
else:
    summary_plot = unsupported
    decision_plot = unsupported
    multioutput_decision_plot = unsupported
    dependence_plot = unsupported
    force_plot = unsupported
    initjs = unsupported
    save_html = unsupported
    image_plot = unsupported
    monitoring_plot = unsupported
    embedding_plot = unsupported
    partial_dependence_plot = unsupported
    bar_plot = unsupported

# other stuff :)
from . import datasets
#from . import benchmark
from .common import approximate_interactions, hclust_ordering, sample
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys

# CAUTION! This line is only used for a development environment, when pywps is not installed
sys.path.append(os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.path.pardir))

from server import Server
from processes.sleep import Sleep
from processes.ultimate_question import UltimateQuestion
from processes.centroids import Centroids
from processes.sayhello import SayHello
from processes.feature_count import FeatureCount
from processes.brauni import Brauni

def main():
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-w', '--waitress', action='store_true')
    args = parser.parse_args()

    config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pywps.cfg")

    processes = [
        FeatureCount(),
        SayHello(),
        Centroids(),
        UltimateQuestion(),
        Sleep(),
        Brauni()
    ]

    s = Server(processes=processes, config_file=config_file)

    # TODO: need to spawn a different process for different server
    if args.waitress:
        import waitress
        from pywps import config

        config.load_configuration(config_file)
        host = config.get_config_value('wps', 'serveraddress').split('://')[1]
        port = int(config.get_config_value('wps', 'serverport'))

        waitress.serve(s.app, host=host, port=port)
    else:
        s.run()

if __name__ == '__main__':
    main()
<commit_msg>Added Area process to demo Fixed get_data_from_kvp while doing a GET request with empty input identifier<commit_after>#!/usr/bin/env python
import os
import sys

# CAUTION! This line is only used for a development environment, when pywps is not installed
sys.path.append(os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.path.pardir))

from server import Server
from processes.sleep import Sleep
from processes.ultimate_question import UltimateQuestion
from processes.centroids import Centroids
from processes.sayhello import SayHello
from processes.feature_count import FeatureCount
from processes.brauni import Brauni
from processes.area import Area

def main():
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('-w', '--waitress', action='store_true')
    args = parser.parse_args()

    config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pywps.cfg")

    processes = [
        FeatureCount(),
        SayHello(),
        Centroids(),
        UltimateQuestion(),
        Sleep(),
        Brauni(),
        Area()
    ]

    s = Server(processes=processes, config_file=config_file)

    # TODO: need to spawn a different process for different server
    if args.waitress:
        import waitress
        from pywps import config

        config.load_configuration(config_file)
        host = config.get_config_value('wps', 'serveraddress').split('://')[1]
        port = int(config.get_config_value('wps', 'serverport'))

        waitress.serve(s.app, host=host, port=port)
    else:
        s.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import sklearn.datasets
import os

try:
    from urllib.request import urlretrieve
except ImportError:
    from urllib import urlretrieve

github_data_url = "https://github.com/slundberg/shap/raw/master/data/"

def imagenet50(display=False, resolution=224):
    """ This is a set of 50 images representative of ImageNet images.

    This dataset was collected by randomly finding a working ImageNet link and then pasting the
    original ImageNet image into Google image search restricted to images licensed for reuse. A
    similar image (now with rights to reuse) was downloaded as a rough replacment for the original
    ImageNet image. The point is to have a random sample of ImageNet for use as a background
    distribution for explaining models trained on ImageNet data.

    Note that because the images are only rough replacements the labels might no longer be correct.
    """

    prefix = github_data_url + "imagenet50_"
    X = np.load(cache(prefix + "%sx%s.npy" % (resolution, resolution))).astype(np.float32)
    y = np.loadtxt(cache(prefix + "labels.csv"))
    return X, y

def boston(display=False):
    """ Return the boston housing data in a nice package. """

    d = sklearn.datasets.load_boston()
    df = pd.DataFrame(data=d.data, columns=d.feature_names)
    return df, d.target

def communitiesandcrime(display=False):
    """ Predict total number of non-violent crimes per 100K popuation.

    This dataset is from the classic UCI Machine Learning repository:
    https://archive.ics.uci.edu/ml/datasets/Communities+and+Crime+Unnormalized
    """

    raw_data = pd.read_csv(
        cache(github_data_url + "CommViolPredUnnormalizedData.txt"),
        na_values="?"
    )

    # find the indices where the total violent crimes are known
    valid_inds = np.where(np.invert(np.isnan(raw_data.iloc[:,-2])))[0]
    y = np.array(raw_data.iloc[valid_inds,-2], dtype=np.float)

    # extract the predictive features and remove columns with missing values
    X = raw_data.iloc[valid_inds,5:-18]
    valid_cols = np.where(np.isnan(X.values).sum(0) == 0)[0]
    X = X.iloc[:,valid_cols]

    return X, y

def diabetes(display=False):
    """ Return the diabetes housing data in a nice package. """

    d = sklearn.datasets.load_diabetes()
    df = pd.DataFrame(data=d.data, columns=d.feature_names)
    return df, d.target

def iris(display=False):
    """ Return the classic iris data in a nice package. """

    d = sklearn.datasets.load_iris()
    df = pd.DataFrame(data=d.data, columns=d.feature_names)
    if display:
        return df, [d.target_names[v] for v in d.target]
    else:
        return df, d.target

def adult(display=False):
    """ Return the Adult census data in a nice package. """
    dtypes = [
        ("Age", "float32"), ("Workclass", "category"), ("fnlwgt", "float32"),
        ("Education", "category"), ("Education-Num", "float32"), ("Marital Status", "category"),
        ("Occupation", "category"), ("Relationship", "category"), ("Race", "category"),
        ("Sex", "category"), ("Capital Gain", "float32"), ("Capital Loss", "float32"),
        ("Hours per week", "float32"), ("Country", "category"), ("Target", "category")
    ]
    raw_data = pd.read_csv(
        cache(github_data_url + "adult.data"),
        names=[d[0] for d in dtypes],
        na_values="?",
        dtype=dict(dtypes)
    )
    data = raw_data.drop(["Education"], axis=1)  # redundant with Education-Num
    filt_dtypes = list(filter(lambda x: not (x[0] in ["Target", "Education"]), dtypes))
    data["Target"] = data["Target"] == " >50K"
    rcode = {
        "Not-in-family": 0,
        "Unmarried": 1,
        "Other-relative": 2,
        "Own-child": 3,
        "Husband": 4,
        "Wife": 5
    }
    for k, dtype in filt_dtypes:
        if dtype == "category":
            if k == "Relationship":
                data[k] = np.array([rcode[v.strip()] for v in data[k]])
            else:
                data[k] = data[k].cat.codes

    if display:
        return raw_data.drop(["Education", "Target", "fnlwgt"], axis=1), data["Target"].values
    else:
        return data.drop(["Target", "fnlwgt"], axis=1), data["Target"].values

def nhanesi(display=False):
    """ A nicely packages version of NHANES I data with surivival times as labels.
    """
    X = pd.read_csv(cache(github_data_url + "NHANESI_subset_X.csv"))
    y = pd.read_csv(cache(github_data_url + "NHANESI_subset_y.csv"))["y"]
    X_display = X.copy()
    X_display["Sex"] = ["Male" if v == 1 else "Female" for v in X["Sex"]]
    if display:
        return X_display, np.array(y)
    else:
        return X, np.array(y)

def corrgroups60(display=False):
    """ A simulated dataset with tight correlations among distinct groups of features.
    """

    # set a constant seed
    old_seed = np.random.seed()
    np.random.seed(0)

    # generate dataset with known correlation
    N = 1000
    M = 60

    # set one coefficent from each group of 3 to 1
    beta = np.zeros(M)
    beta[0:30:3] = 1

    # build a correlation matrix with groups of 3 tightly correlated features
    x = np.ones(M)
    mu = np.zeros(M)
    C = np.eye(M)
    for i in range(0,30,3):
        C[i,i+1] = C[i+1,i] = 0.99
        C[i,i+2] = C[i+2,i] = 0.99
        C[i+1,i+2] = C[i+2,i+1] = 0.99
    f = lambda X: np.matmul(X, beta)

    # Make sure the sample correlation is a perfect match
    X_start = np.random.randn(N, M)
    X_centered = X_start - X_start.mean(0)
    Sigma = np.matmul(X_centered.T, X_centered) / X_centered.shape[0]
    W = np.linalg.cholesky(np.linalg.inv(Sigma)).T
    X_white = np.matmul(X_centered, W.T)
    assert np.linalg.norm(np.corrcoef(np.matmul(X_centered, W.T).T) - np.eye(M)) < 1e-6 # ensure this decorrelates the data

    # create the final data
    X_final = np.matmul(X_white, np.linalg.cholesky(C).T)
    X = X_final
    y = f(X) + np.random.randn(N) * 1e-2

    # restore the previous numpy random seed
    np.random.seed(old_seed)

    return pd.DataFrame(X), y

def a1a():
    """ A sparse dataset in scipy csr matrix format.
    """
    # todo change this back to github data url once checked in
    tmp_data_url = 'https://raw.githubusercontent.com/imatiach-msft/shap/eada98f882dad1553612109771c79a3e0f426195/data/'
    return sklearn.datasets.load_svmlight_file(cache(tmp_data_url + 'a1a.svmlight'))

def cache(url, file_name=None):
    if file_name is None:
        file_name = os.path.basename(url)
    data_dir = os.path.join(os.path.dirname(__file__), "cached_data")
    if not os.path.isdir(data_dir):
        os.mkdir(data_dir)

    file_path = os.path.join(data_dir, file_name)
    if not os.path.isfile(file_path):
        urlretrieve(url, file_path)

    return file_path
<commit_msg>fixed github data url for svmlight sparse dataset<commit_after>import pandas as pd
import numpy as np
import sklearn.datasets
import os

try:
    from urllib.request import urlretrieve
except ImportError:
    from urllib import urlretrieve

github_data_url = "https://github.com/slundberg/shap/raw/master/data/"

def imagenet50(display=False, resolution=224):
    """ This is a set of 50 images representative of ImageNet images.

    This dataset was collected by randomly finding a working ImageNet link and then pasting the
    original ImageNet image into Google image search restricted to images licensed for reuse. A
    similar image (now with rights to reuse) was downloaded as a rough replacment for the original
    ImageNet image. The point is to have a random sample of ImageNet for use as a background
    distribution for explaining models trained on ImageNet data.

    Note that because the images are only rough replacements the labels might no longer be correct.
    """

    prefix = github_data_url + "imagenet50_"
    X = np.load(cache(prefix + "%sx%s.npy" % (resolution, resolution))).astype(np.float32)
    y = np.loadtxt(cache(prefix + "labels.csv"))
    return X, y

def boston(display=False):
    """ Return the boston housing data in a nice package. """

    d = sklearn.datasets.load_boston()
    df = pd.DataFrame(data=d.data, columns=d.feature_names)
    return df, d.target

def communitiesandcrime(display=False):
    """ Predict total number of non-violent crimes per 100K popuation.

    This dataset is from the classic UCI Machine Learning repository:
    https://archive.ics.uci.edu/ml/datasets/Communities+and+Crime+Unnormalized
    """

    raw_data = pd.read_csv(
        cache(github_data_url + "CommViolPredUnnormalizedData.txt"),
        na_values="?"
    )

    # find the indices where the total violent crimes are known
    valid_inds = np.where(np.invert(np.isnan(raw_data.iloc[:,-2])))[0]
    y = np.array(raw_data.iloc[valid_inds,-2], dtype=np.float)

    # extract the predictive features and remove columns with missing values
    X = raw_data.iloc[valid_inds,5:-18]
    valid_cols = np.where(np.isnan(X.values).sum(0) == 0)[0]
    X = X.iloc[:,valid_cols]

    return X, y

def diabetes(display=False):
    """ Return the diabetes housing data in a nice package. """

    d = sklearn.datasets.load_diabetes()
    df = pd.DataFrame(data=d.data, columns=d.feature_names)
    return df, d.target

def iris(display=False):
    """ Return the classic iris data in a nice package. """

    d = sklearn.datasets.load_iris()
    df = pd.DataFrame(data=d.data, columns=d.feature_names)
    if display:
        return df, [d.target_names[v] for v in d.target]
    else:
        return df, d.target

def adult(display=False):
    """ Return the Adult census data in a nice package. """
    dtypes = [
        ("Age", "float32"), ("Workclass", "category"), ("fnlwgt", "float32"),
        ("Education", "category"), ("Education-Num", "float32"), ("Marital Status", "category"),
        ("Occupation", "category"), ("Relationship", "category"), ("Race", "category"),
        ("Sex", "category"), ("Capital Gain", "float32"), ("Capital Loss", "float32"),
        ("Hours per week", "float32"), ("Country", "category"), ("Target", "category")
    ]
    raw_data = pd.read_csv(
        cache(github_data_url + "adult.data"),
        names=[d[0] for d in dtypes],
        na_values="?",
        dtype=dict(dtypes)
    )
    data = raw_data.drop(["Education"], axis=1)  # redundant with Education-Num
    filt_dtypes = list(filter(lambda x: not (x[0] in ["Target", "Education"]), dtypes))
    data["Target"] = data["Target"] == " >50K"
    rcode = {
        "Not-in-family": 0,
        "Unmarried": 1,
        "Other-relative": 2,
        "Own-child": 3,
        "Husband": 4,
        "Wife": 5
    }
    for k, dtype in filt_dtypes:
        if dtype == "category":
            if k == "Relationship":
                data[k] = np.array([rcode[v.strip()] for v in data[k]])
            else:
                data[k] = data[k].cat.codes

    if display:
        return raw_data.drop(["Education", "Target", "fnlwgt"], axis=1), data["Target"].values
    else:
        return data.drop(["Target", "fnlwgt"], axis=1), data["Target"].values

def nhanesi(display=False):
    """ A nicely packages version of NHANES I data with surivival times as labels.
    """
    X = pd.read_csv(cache(github_data_url + "NHANESI_subset_X.csv"))
    y = pd.read_csv(cache(github_data_url + "NHANESI_subset_y.csv"))["y"]
    X_display = X.copy()
    X_display["Sex"] = ["Male" if v == 1 else "Female" for v in X["Sex"]]
    if display:
        return X_display, np.array(y)
    else:
        return X, np.array(y)

def corrgroups60(display=False):
    """ A simulated dataset with tight correlations among distinct groups of features.
    """

    # set a constant seed
    old_seed = np.random.seed()
    np.random.seed(0)

    # generate dataset with known correlation
    N = 1000
    M = 60

    # set one coefficent from each group of 3 to 1
    beta = np.zeros(M)
    beta[0:30:3] = 1

    # build a correlation matrix with groups of 3 tightly correlated features
    x = np.ones(M)
    mu = np.zeros(M)
    C = np.eye(M)
    for i in range(0,30,3):
        C[i,i+1] = C[i+1,i] = 0.99
        C[i,i+2] = C[i+2,i] = 0.99
        C[i+1,i+2] = C[i+2,i+1] = 0.99
    f = lambda X: np.matmul(X, beta)

    # Make sure the sample correlation is a perfect match
    X_start = np.random.randn(N, M)
    X_centered = X_start - X_start.mean(0)
    Sigma = np.matmul(X_centered.T, X_centered) / X_centered.shape[0]
    W = np.linalg.cholesky(np.linalg.inv(Sigma)).T
    X_white = np.matmul(X_centered, W.T)
    assert np.linalg.norm(np.corrcoef(np.matmul(X_centered, W.T).T) - np.eye(M)) < 1e-6 # ensure this decorrelates the data

    # create the final data
    X_final = np.matmul(X_white, np.linalg.cholesky(C).T)
    X = X_final
    y = f(X) + np.random.randn(N) * 1e-2

    # restore the previous numpy random seed
    np.random.seed(old_seed)

    return pd.DataFrame(X), y

def a1a():
    """ A sparse dataset in scipy csr matrix format.
    """
    return sklearn.datasets.load_svmlight_file(cache(github_data_url + 'a1a.svmlight'))

def cache(url, file_name=None):
    if file_name is None:
        file_name = os.path.basename(url)
    data_dir = os.path.join(os.path.dirname(__file__), "cached_data")
    if not os.path.isdir(data_dir):
        os.mkdir(data_dir)

    file_path = os.path.join(data_dir, file_name)
    if not os.path.isfile(file_path):
        urlretrieve(url, file_path)

    return file_path
<|endoftext|>


<commit_before># vim:set et sts=4 sw=4:
#
# ibus - The Input Bus
#
# Copyright (c) 2007-2008 Huang Peng <shawn.p.huang@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA  02111-1307  USA

import weakref
import gobject
import ibus

class Engine(ibus.Object):
    __gsignals__ = {
        "commit-string" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, )),
        "forward-key-event" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_UINT, gobject.TYPE_BOOLEAN, gobject.TYPE_UINT )),
        "update-preedit" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, gobject.TYPE_PYOBJECT, gobject.TYPE_INT, gobject.TYPE_BOOLEAN)),
        "update-aux-string" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, gobject.TYPE_PYOBJECT, gobject.TYPE_BOOLEAN)),
        "update-lookup-table" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, gobject.TYPE_BOOLEAN)),
        "register-properties" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, )),
        "update-property" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, )),
    }

    def __init__(self, factory, ibusconn, object_path):
        super(Engine, self).__init__()
        self._factory = factory
        self._ibusconn = ibusconn
        self._object_path = object_path
        self._engine = ibusconn.get_object(self._object_path)
        self._lookup_table = ibus.LookupTable()
        self._ibusconn.connect("destroy", self._ibusconn_destroy_cb)

    def get_factory(self):
        return self._factory

    def get_object_path(self):
        return self._object_path

    def handle_dbus_signal(self, message):
        if message.is_signal(ibus.IBUS_ENGINE_IFACE, "CommitString"):
            args = message.get_args_list()
            self.emit("commit-string", args[0])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "ForwardKeyEvent"):
            args = message.get_args_list()
            self.emit("forward-key-event", args[0], bool(arg[1]), arg[2])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdatePreedit"):
            args = message.get_args_list()
            self.emit("update-preedit", args[0], args[1], args[2], args[3])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdateAuxString"):
            args = message.get_args_list()
            self.emit("update-aux-string", args[0], args[1], args[2])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdateLookupTable"):
            args = message.get_args_list()
            self.emit("update-lookup-table", args[0], args[1])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "RegisterProperties"):
            args = message.get_args_list()
            self.emit("register-properties", args[0])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdateProperty"):
            args = message.get_args_list()
            self.emit("update-property", args[0])
        else:
            return False

        return True

    def focus_in(self):
        self._engine.FocusIn(**ibus.DEFAULT_ASYNC_HANDLERS)

    def focus_out(self):
        self._engine.FocusOut(**ibus.DEFAULT_ASYNC_HANDLERS)

    def reset(self):
        self._engine.Reset(**ibus.DEFAULT_ASYNC_HANDLERS)

    def process_key_event(self, keyval, is_press, state, reply_cb, error_cb):
        self._engine.ProcessKeyEvent(keyval, is_press, state,
                                    reply_handler = reply_cb,
                                    error_handler = error_cb)

    def set_cursor_location(self, x, y, w, h):
        self._engine.SetCursorLocation(x, y, w, h,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def set_enable(self, enable):
        self._engine.SetEnable(enable,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    # cursor for lookup table

    def page_up(self):
        self._engine.PageUp(**ibus.DEFAULT_ASYNC_HANDLERS)

    def page_down(self):
        self._engine.PageDown(**ibus.DEFAULT_ASYNC_HANDLERS)

    def cursor_up(self):
        self._engine.CursorUp(**ibus.DEFAULT_ASYNC_HANDLERS)

    def cursor_down(self):
        self._engine.CursorDown(**ibus.DEFAULT_ASYNC_HANDLERS)

    def property_activate(self, prop_name, prop_state):
        self._engine.PropertyActivate(prop_name, prop_state,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def property_show(self, prop_name):
        self._engine.PropertyShow(prop_name,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def property_hide(self, prop_name):
        self._engine.PropertyHide(prop_name,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def destroy(self):
        ibus.Object.destroy(self)
        if self._engine:
            self._engine.Destroy(**ibus.DEFAULT_ASYNC_HANDLERS)
            self._engine = None
        self._ibusconn = None

    def _ibusconn_destroy_cb(self, ibusconn):
        self._engine = None
        self.destroy()

gobject.type_register(Engine)

<commit_msg>Replase SetEnable with Enable & Disable.<commit_after># vim:set et sts=4 sw=4:
#
# ibus - The Input Bus
#
# Copyright (c) 2007-2008 Huang Peng <shawn.p.huang@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA  02111-1307  USA

import weakref
import gobject
import ibus

class Engine(ibus.Object):
    __gsignals__ = {
        "commit-string" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, )),
        "forward-key-event" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_UINT, gobject.TYPE_BOOLEAN, gobject.TYPE_UINT )),
        "update-preedit" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, gobject.TYPE_PYOBJECT, gobject.TYPE_INT, gobject.TYPE_BOOLEAN)),
        "update-aux-string" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, gobject.TYPE_PYOBJECT, gobject.TYPE_BOOLEAN)),
        "update-lookup-table" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, gobject.TYPE_BOOLEAN)),
        "register-properties" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, )),
        "update-property" : (
            gobject.SIGNAL_RUN_FIRST,
            gobject.TYPE_NONE,
            (gobject.TYPE_PYOBJECT, )),
    }

    def __init__(self, factory, ibusconn, object_path):
        super(Engine, self).__init__()
        self._factory = factory
        self._ibusconn = ibusconn
        self._object_path = object_path
        self._engine = ibusconn.get_object(self._object_path)
        self._lookup_table = ibus.LookupTable()
        self._ibusconn.connect("destroy", self._ibusconn_destroy_cb)

    def get_factory(self):
        return self._factory

    def get_object_path(self):
        return self._object_path

    def handle_dbus_signal(self, message):
        if message.is_signal(ibus.IBUS_ENGINE_IFACE, "CommitString"):
            args = message.get_args_list()
            self.emit("commit-string", args[0])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "ForwardKeyEvent"):
            args = message.get_args_list()
            self.emit("forward-key-event", args[0], bool(arg[1]), arg[2])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdatePreedit"):
            args = message.get_args_list()
            self.emit("update-preedit", args[0], args[1], args[2], args[3])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdateAuxString"):
            args = message.get_args_list()
            self.emit("update-aux-string", args[0], args[1], args[2])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdateLookupTable"):
            args = message.get_args_list()
            self.emit("update-lookup-table", args[0], args[1])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "RegisterProperties"):
            args = message.get_args_list()
            self.emit("register-properties", args[0])
        elif message.is_signal(ibus.IBUS_ENGINE_IFACE, "UpdateProperty"):
            args = message.get_args_list()
            self.emit("update-property", args[0])
        else:
            return False

        return True

    def focus_in(self):
        self._engine.FocusIn(**ibus.DEFAULT_ASYNC_HANDLERS)

    def focus_out(self):
        self._engine.FocusOut(**ibus.DEFAULT_ASYNC_HANDLERS)

    def reset(self):
        self._engine.Reset(**ibus.DEFAULT_ASYNC_HANDLERS)

    def process_key_event(self, keyval, is_press, state, reply_cb, error_cb):
        self._engine.ProcessKeyEvent(keyval, is_press, state,
                                    reply_handler = reply_cb,
                                    error_handler = error_cb)

    def set_cursor_location(self, x, y, w, h):
        self._engine.SetCursorLocation(x, y, w, h,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def enable(self):
        self._engine.Enable(**ibus.DEFAULT_ASYNC_HANDLERS)

    def disable(self):
        self._engine.Disable(**ibus.DEFAULT_ASYNC_HANDLERS)

    # cursor for lookup table

    def page_up(self):
        self._engine.PageUp(**ibus.DEFAULT_ASYNC_HANDLERS)

    def page_down(self):
        self._engine.PageDown(**ibus.DEFAULT_ASYNC_HANDLERS)

    def cursor_up(self):
        self._engine.CursorUp(**ibus.DEFAULT_ASYNC_HANDLERS)

    def cursor_down(self):
        self._engine.CursorDown(**ibus.DEFAULT_ASYNC_HANDLERS)

    def property_activate(self, prop_name, prop_state):
        self._engine.PropertyActivate(prop_name, prop_state,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def property_show(self, prop_name):
        self._engine.PropertyShow(prop_name,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def property_hide(self, prop_name):
        self._engine.PropertyHide(prop_name,
                **ibus.DEFAULT_ASYNC_HANDLERS)

    def destroy(self):
        ibus.Object.destroy(self)
        if self._engine:
            self._engine.Destroy(**ibus.DEFAULT_ASYNC_HANDLERS)
            self._engine = None
        self._ibusconn = None

    def _ibusconn_destroy_cb(self, ibusconn):
        self._engine = None
        self.destroy()

gobject.type_register(Engine)

<|endoftext|>


<commit_before>#!/usr/bin/env python
from shi7 import read_fastq
import os

def subsample_fastqs(path_fastqs, num_files=10, num_sequences=100):
    for i, path_fastq in enumerate(path_fastqs):
        with open(path_fastq) as fastq_inf:
            if i >= num_files:
                break
            fastq_gen = read_fastq(fastq_inf)
            yield limit_fastq(fastq_gen, num_sequences=num_sequences)

def limit_fastq(fastq_gen, num_sequences=100):
    for i in range(num_sequences):
        yield next(fastq_gen)

def get_seq_length_qual_scores(path_fastqs, num_files=10, num_sequences=1000):
    subsampled_fastqs = subsample_fastqs(path_fastqs, num_files=num_files, num_sequences=num_sequences)
    sequence_len_sum = 0.
    quality_sum = 0
    num_sequences = 0.
    # TODO: Write the subsampled fastqs
    # sequences = []
    # qualities = []
    for fastq_gen in subsampled_fastqs:
        for header, sequence, quality in fastq_gen:
            # TODO: Write the subsampled fastqs
            # sequences.append(sequences)
            # qualities.append(qualities)
            sequence_len_sum += len(sequence)
            quality_sum += sum([ord(i) for i in quality])
            num_sequences += 1.
    # Return (average length of sequences, average quality score)
    return sequence_len_sum/num_sequences, quality_sum/sequence_len_sum

def count_num_lines(path):
    with open(path) as path_inf:
        return sum(1 for line in path_inf)

def get_file_size(path):
    return os.path.getsize(path)

'''
def check_sequence_name(path):
    ids = []
    with open(path) as path_inf:
        fastq_gen = read_fastq(path_inf)
        for gen in fastq_gen:
            for header in gen: #why not header, sequence, quality?
                ids.append(header[-9])

    return ids #why the output has size of 1497
'''

def detect_paired_end(path_fastqs):
    if len(path_fastqs) % 2 == 1:
        return False
    path_fastqs = sorted(path_fastqs)
    path_R1_fastqs, path_R2_fastqs = path_fastqs[::2], path_fastqs[1::2]

    R1_lines_num = []
    R2_lines_num = []
    R1_files_size = []
    R2_files_size = []
    R1_seqs_name = []
    R2_seqs_name = []
    for path_R1_fastq in path_R1_fastqs:
        R1_lines_num.append(count_num_lines(path_R1_fastq))
        R1_files_size.append(get_file_size(path_R1_fastq))
        #R1_seqs_name.append(check_sequence_name(path_R1_fastq,'1'))
    for path_R2_fastq in path_R2_fastqs:
        R2_lines_num.append(count_num_lines(path_R2_fastq))
        R2_files_size.append(get_file_size(path_R2_fastq))
        #R2_seqs_name.append(check_sequence_name(path_R2_fastq,'2'))

    print(R1_seqs_name)
    print(R2_seqs_name)
    if not R1_lines_num == R2_lines_num or not R1_files_size == R2_files_size: #or not R1_seqs_name == R2_seqs_name:
        return False

    return True

<commit_msg>updated detect_paired_end #37<commit_after>#!/usr/bin/env python
from shi7 import read_fastq
import os

def subsample_fastqs(path_fastqs, num_files=10, num_sequences=100):
    for i, path_fastq in enumerate(path_fastqs):
        with open(path_fastq) as fastq_inf:
            if i >= num_files:
                break
            fastq_gen = read_fastq(fastq_inf)
            yield limit_fastq(fastq_gen, num_sequences=num_sequences)

def limit_fastq(fastq_gen, num_sequences=100):
    for i in range(num_sequences):
        yield next(fastq_gen)

def get_seq_length_qual_scores(path_fastqs, num_files=10, num_sequences=1000):
    subsampled_fastqs = subsample_fastqs(path_fastqs, num_files=num_files, num_sequences=num_sequences)
    sequence_len_sum = 0.
    quality_sum = 0
    num_sequences = 0.
    # TODO: Write the subsampled fastqs
    # sequences = []
    # qualities = []
    for fastq_gen in subsampled_fastqs:
        for header, sequence, quality in fastq_gen:
            # TODO: Write the subsampled fastqs
            # sequences.append(sequences)
            # qualities.append(qualities)
            sequence_len_sum += len(sequence)
            quality_sum += sum([ord(i) for i in quality])
            num_sequences += 1.
    # Return (average length of sequences, average quality score)
    return sequence_len_sum/num_sequences, quality_sum/sequence_len_sum

def count_num_lines(path):
    with open(path) as path_inf:
        return sum(1 for line in path_inf)

def get_file_size(path):
    return os.path.getsize(path)

'''
def check_sequence_name(path):
    ids = []
    with open(path) as path_inf:
        fastq_gen = read_fastq(path_inf)
        for gen in fastq_gen:
            for header in gen: #why not header, sequence, quality?
                ids.append(header[-9])

    return ids #why the output has size of 1497
'''

def detect_paired_end(path_fastqs):
    if len(path_fastqs) % 2 == 1:
        return False
    path_fastqs = sorted(path_fastqs)
    path_R1_fastqs, path_R2_fastqs = path_fastqs[::2], path_fastqs[1::2]
    if len(path_R1_fastqs) != len(path_R2_fastqs) or len(path_R1_fastqs) < 1:
        return False
    R1_lines_num = []
    R2_lines_num = []
    R1_files_size = []
    R2_files_size = []
    R1_seqs_name = []
    R2_seqs_name = []
    for path_R1_fastq in path_R1_fastqs:
        R1_lines_num.append(count_num_lines(path_R1_fastq))
        R1_files_size.append(get_file_size(path_R1_fastq))
        #R1_seqs_name.append(check_sequence_name(path_R1_fastq,'1'))
    for path_R2_fastq in path_R2_fastqs:
        R2_lines_num.append(count_num_lines(path_R2_fastq))
        R2_files_size.append(get_file_size(path_R2_fastq))
        #R2_seqs_name.append(check_sequence_name(path_R2_fastq,'2'))

    if not R1_lines_num == R2_lines_num or not R1_files_size == R2_files_size: #or not R1_seqs_name == R2_seqs_name:
        return False

    return True

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2009-2014:
#     Gabes Jean, naparuba@gmail.com
#     Gerhard Lausser, Gerhard.Lausser@consol.de
#     Gregory Starck, g.starck@gmail.com
#     Hartmut Goebel, h.goebel@goebel-consult.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

import threading
import time
import json
import hashlib
import base64
import socket

from shinken.log import logger

# For old users python-crypto was not mandatory, don't break their setup
try:
    from Crypto.Cipher import AES
except ImportError:
    logger.error('Cannot find python lib crypto: export to kernel.shinken.io isnot available')
    AES = None

from shinken.http_client import HTTPClient, HTTPException

BLOCK_SIZE = 16

def pad(data):
    pad = BLOCK_SIZE - len(data) % BLOCK_SIZE
    return data + pad * chr(pad)

def unpad(padded):
    pad = ord(padded[-1])
    return padded[:-pad]

class Stats(object):
    def __init__(self):
        self.name = ''
        self.type = ''
        self.app = None
        self.stats = {}
        # There are two modes that are not exclusive
        # first the kernel mode
        self.api_key = ''
        self.secret = ''
        self.http_proxy = ''
        self.con = HTTPClient(uri='http://kernel.shinken.io')
        # then the statsd one
        self.statsd_sock = None
        self.statsd_addr = None

    def launch_reaper_thread(self):
        self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
        self.reaper_thread.daemon = True
        self.reaper_thread.start()

    def register(self, app, name, _type, api_key='', secret='', http_proxy='',
                 statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken',
                 statsd_enabled=False):
        self.app = app
        self.name = name
        self.type = _type
        # kernel.io part
        self.api_key = api_key
        self.secret = secret
        self.http_proxy = http_proxy
        # local statsd part
        self.statsd_host = statsd_host
        self.statsd_port = statsd_port
        self.statsd_prefix = statsd_prefix
        self.statsd_enabled = statsd_enabled

        if self.statsd_enabled:
            logger.debug('Loading statsd communication with %s:%s.%s',
                         self.statsd_host, self.statsd_port, self.statsd_prefix)
            self.load_statsd()

        # Also load the proxy if need
        self.con.set_proxy(self.http_proxy)

    # Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
    # how guys did you fuck this up to this point? django by default for the conf?? really?...
    # So raw socket are far better here
    def load_statsd(self):
        try:
            self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
            self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        except (socket.error, socket.gaierror), exp:
            logger.error('Cannot create statsd socket: %s' % exp)
            return

    # Will increment a stat key, if None, start at 0
    def incr(self, k, v):
        _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
        nb += 1
        _sum += v
        if _min is None or v < _min:
            _min = v
        if _max is None or v > _max:
            _max = v
        self.stats[k] = (_min, _max, nb, _sum)

        # Manage local statd part
        if self.statsd_sock and self.name:
            # beware, we are sending ms here, v is in s
            packet = '%s.%s.%s:%d|ms' % (self.statsd_prefix, self.name, k, v * 1000)
            try:
                self.statsd_sock.sendto(packet, self.statsd_addr)
            except (socket.error, socket.gaierror), exp:
                pass  # cannot send? ok not a huge problem here and cannot
                # log because it will be far too verbose :p

    def _encrypt(self, data):
        m = hashlib.md5()
        m.update(self.secret)
        key = m.hexdigest()

        m = hashlib.md5()
        m.update(self.secret + key)
        iv = m.hexdigest()

        data = pad(data)

        aes = AES.new(key, AES.MODE_CBC, iv[:16])

        encrypted = aes.encrypt(data)
        return base64.urlsafe_b64encode(encrypted)

    def reaper(self):
        while True:
            now = int(time.time())
            stats = self.stats
            self.stats = {}

            if len(stats) != 0:
                s = ', '.join(['%s:%s' % (k, v) for (k, v) in stats.iteritems()])
            # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
            # to find the data after this
            if not self.name or not self.api_key or not self.secret:
                time.sleep(60)
                continue

            metrics = []
            for (k, e) in stats.iteritems():
                nk = '%s.%s.%s' % (self.type, self.name, k)
                _min, _max, nb, _sum = e
                _avg = float(_sum) / nb
                # nb can't be 0 here and _min_max can't be None too
                s = '%s.avg %f %d' % (nk, _avg, now)
                metrics.append(s)
                s = '%s.min %f %d' % (nk, _min, now)
                metrics.append(s)
                s = '%s.max %f %d' % (nk, _max, now)
                metrics.append(s)
                s = '%s.count %f %d' % (nk, nb, now)
                metrics.append(s)

            # logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) )
            # get the inner data for the daemon
            struct = self.app.get_stats_struct()
            struct['metrics'].extend(metrics)
            # logger.debug('REAPER whole struct %s' % struct)
            j = json.dumps(struct)
            if AES is not None and self.secret != '':
                logger.debug('Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' % (
                    self.api_key, self.secret))

                # assume a %16 length messagexs
                encrypted_text = self._encrypt(j)
                try:
                    r = self.con.put('/api/v1/put/?api_key=%s' % (self.api_key), encrypted_text)
                except HTTPException, exp:
                    logger.error('Stats REAPER cannot put to the metric server %s' % exp)
            time.sleep(60)

statsmgr = Stats()
<commit_msg>debug instead of error to avoid error log with shinken cli<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright (C) 2009-2014:
#     Gabes Jean, naparuba@gmail.com
#     Gerhard Lausser, Gerhard.Lausser@consol.de
#     Gregory Starck, g.starck@gmail.com
#     Hartmut Goebel, h.goebel@goebel-consult.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

import threading
import time
import json
import hashlib
import base64
import socket

from shinken.log import logger

# For old users python-crypto was not mandatory, don't break their setup
try:
    from Crypto.Cipher import AES
except ImportError:
    logger.debug('Cannot find python lib crypto: export to kernel.shinken.io isnot available')
    AES = None

from shinken.http_client import HTTPClient, HTTPException

BLOCK_SIZE = 16

def pad(data):
    pad = BLOCK_SIZE - len(data) % BLOCK_SIZE
    return data + pad * chr(pad)

def unpad(padded):
    pad = ord(padded[-1])
    return padded[:-pad]

class Stats(object):
    def __init__(self):
        self.name = ''
        self.type = ''
        self.app = None
        self.stats = {}
        # There are two modes that are not exclusive
        # first the kernel mode
        self.api_key = ''
        self.secret = ''
        self.http_proxy = ''
        self.con = HTTPClient(uri='http://kernel.shinken.io')
        # then the statsd one
        self.statsd_sock = None
        self.statsd_addr = None

    def launch_reaper_thread(self):
        self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
        self.reaper_thread.daemon = True
        self.reaper_thread.start()

    def register(self, app, name, _type, api_key='', secret='', http_proxy='',
                 statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken',
                 statsd_enabled=False):
        self.app = app
        self.name = name
        self.type = _type
        # kernel.io part
        self.api_key = api_key
        self.secret = secret
        self.http_proxy = http_proxy
        # local statsd part
        self.statsd_host = statsd_host
        self.statsd_port = statsd_port
        self.statsd_prefix = statsd_prefix
        self.statsd_enabled = statsd_enabled

        if self.statsd_enabled:
            logger.debug('Loading statsd communication with %s:%s.%s',
                         self.statsd_host, self.statsd_port, self.statsd_prefix)
            self.load_statsd()

        # Also load the proxy if need
        self.con.set_proxy(self.http_proxy)

    # Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
    # how guys did you fuck this up to this point? django by default for the conf?? really?...
    # So raw socket are far better here
    def load_statsd(self):
        try:
            self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
            self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        except (socket.error, socket.gaierror), exp:
            logger.error('Cannot create statsd socket: %s' % exp)
            return

    # Will increment a stat key, if None, start at 0
    def incr(self, k, v):
        _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
        nb += 1
        _sum += v
        if _min is None or v < _min:
            _min = v
        if _max is None or v > _max:
            _max = v
        self.stats[k] = (_min, _max, nb, _sum)

        # Manage local statd part
        if self.statsd_sock and self.name:
            # beware, we are sending ms here, v is in s
            packet = '%s.%s.%s:%d|ms' % (self.statsd_prefix, self.name, k, v * 1000)
            try:
                self.statsd_sock.sendto(packet, self.statsd_addr)
            except (socket.error, socket.gaierror), exp:
                pass  # cannot send? ok not a huge problem here and cannot
                # log because it will be far too verbose :p

    def _encrypt(self, data):
        m = hashlib.md5()
        m.update(self.secret)
        key = m.hexdigest()

        m = hashlib.md5()
        m.update(self.secret + key)
        iv = m.hexdigest()

        data = pad(data)

        aes = AES.new(key, AES.MODE_CBC, iv[:16])

        encrypted = aes.encrypt(data)
        return base64.urlsafe_b64encode(encrypted)

    def reaper(self):
        while True:
            now = int(time.time())
            stats = self.stats
            self.stats = {}

            if len(stats) != 0:
                s = ', '.join(['%s:%s' % (k, v) for (k, v) in stats.iteritems()])
            # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
            # to find the data after this
            if not self.name or not self.api_key or not self.secret:
                time.sleep(60)
                continue

            metrics = []
            for (k, e) in stats.iteritems():
                nk = '%s.%s.%s' % (self.type, self.name, k)
                _min, _max, nb, _sum = e
                _avg = float(_sum) / nb
                # nb can't be 0 here and _min_max can't be None too
                s = '%s.avg %f %d' % (nk, _avg, now)
                metrics.append(s)
                s = '%s.min %f %d' % (nk, _min, now)
                metrics.append(s)
                s = '%s.max %f %d' % (nk, _max, now)
                metrics.append(s)
                s = '%s.count %f %d' % (nk, nb, now)
                metrics.append(s)

            # logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) )
            # get the inner data for the daemon
            struct = self.app.get_stats_struct()
            struct['metrics'].extend(metrics)
            # logger.debug('REAPER whole struct %s' % struct)
            j = json.dumps(struct)
            if AES is not None and self.secret != '':
                logger.debug('Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' % (
                    self.api_key, self.secret))

                # assume a %16 length messagexs
                encrypted_text = self._encrypt(j)
                try:
                    r = self.con.put('/api/v1/put/?api_key=%s' % (self.api_key), encrypted_text)
                except HTTPException, exp:
                    logger.error('Stats REAPER cannot put to the metric server %s' % exp)
            time.sleep(60)

statsmgr = Stats()
<|endoftext|>


<commit_before>from flask import Flask, render_template, jsonify, session, request, redirect, send_from_directory
from flask_assets import Environment, Bundle
from flask_redis import FlaskRedis
from flask_secure_headers.core import Secure_Headers
from os.path import join, dirname
from werkzeug.exceptions import BadRequest
import os
import hashlib
import datetime

import config
import auth
from user import User
from s3 import AppFetcher
from op.yaml_loader import Application

from op import authzero

from flask_sse import sse

app = Flask(__name__)

if os.environ.get('ENVIRONMENT') == 'Production':
    app.config.from_object(config.ProductionConfig())
elif os.environ.get('ENVIRONMENT') == 'Development':
    print("Getting development config")
    app.config.from_object(config.DevelopmentConfig())

redis_store = FlaskRedis(app)
assets = Environment(app)

js = Bundle('js/base.js',
            filters='jsmin', output='js/gen/packed.js')
assets.register('js_all', js)

sass = Bundle('*.sass', filters='sass', output='css/gen/sass.css')
css = Bundle(
            'css/base.css', sass,
            filters='cssmin', output="css/gen/all.css"
        )
assets.register('css_all', css)

oidc_config = config.OIDCConfig()

authentication = auth.OpenIDConnect(
    oidc_config
)

oidc = authentication.auth(app)

# Add secure Headers to satify observatory checks

sh = Secure_Headers()
sh.update(
    {
        'CSP': {
            'script-src':
                [
                    'self',
                    'ajax.googleapis.com',
                    'cdn.muicss.com',
                    'netdna.bootstrapcdn.com',
                    's.gravatar.com',
                    'fonts.googleapis.com'
                ],
            'style-src':
                [
                    'self',
                    'ajax.googleapis.com',
                    'cdn.muicss.com',
                    'netdna.bootstrapcdn.com',
                    's.gravatar.com',
                    'fonts.googleapis.com'
                ],
            'img-src':
                [
                    'self',
                    's.gravatar.com',
                ],
            'font-src':
                [
                    'self',
                    'netdna.bootstrapcdn.com',
                    'fonts.googleapis.com',
                    'cdn.muicss.com'
                ]
        }
    }
)

sh.update(
    {
        'HSTS':
            {
                'max-age': 1,
                'includeSubDomains': True,
                'preload': False
            }
    }
)

# Register the flask blueprint for SSE.
app.register_blueprint(sse, url_prefix='/stream')

@sse.before_request
def check_access():
    """Users can only view their own security alerts."""
    user = User(session)
    if request.args.get("channel") == Userhash():
        pass
    else:
        abort(403)

@app.route('/favicon.ico')
@sh.wrapper()
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static/img'),
                               'favicon.ico')

@app.route('/')
@sh.wrapper()
def home():
    return redirect('/dashboard', code=302)

@app.route('/dashboard')
@sh.wrapper()
@oidc.oidc_auth
def dashboard():
    """Primary dashboard the users will interact with."""
    AppFetcher().sync_config_and_images()
    user = User(session)
    alerts = redis_store.lrange(user.userhash(), 0, -1)
    all_apps = Application().apps
    apps = user.apps(all_apps)['apps']

    return render_template(
        'dashboard.html',
        user=user,
        apps=apps,
        alerts=alerts
    )

@app.route('/info')
@sh.wrapper()
@oidc.oidc_auth
def info():
    """Return the JSONified user session for debugging."""
    return jsonify(
            id_token=session['id_token'],
            access_token=session['access_token'],
            userinfo=session['userinfo']
        )

@app.route('/alert', methods = ['POST'])
@sh.wrapper()
def publish_alert():
    """
    Takes JSON post with user e-mail to alert.
    Minimum fields are email and message in the form
    of a dict.

    Example:
        {
          "user": {"email": "andrewkrug@gmail.com"},
            "message": "this is a security alert"
        }
    """
    #try:
    content = request.json
    #Send a real time event to the user
    m = hashlib.md5()
    m.update(content['user']['email'])
    channel = m.hexdigest()

    sse.publish(
        {"message": content['message']},
        type="alert",
        channel=channel
    )

    permanent_message = "Security Alert Logged at {date} : {message}".format(
        date=datetime.datetime.now(),
        message=content['message']
    )
    #Store the event in redis keyed to the users hashed e-mail
    redis_store.lpush(channel, permanent_message)

    return jsonify({'status': 'success'})
    #except:
    #    raise BadRequest('POST does not contain e-mail and message')
    #    return jsonify({'status': 'fail'})

if __name__ == '__main__':
    app.run()
<commit_msg>update to csp<commit_after>from flask import Flask, render_template, jsonify, session, request, redirect, send_from_directory
from flask_assets import Environment, Bundle
from flask_redis import FlaskRedis
from flask_secure_headers.core import Secure_Headers
from os.path import join, dirname
from werkzeug.exceptions import BadRequest
import os
import hashlib
import datetime

import config
import auth
from user import User
from s3 import AppFetcher
from op.yaml_loader import Application

from op import authzero

from flask_sse import sse

app = Flask(__name__)

if os.environ.get('ENVIRONMENT') == 'Production':
    app.config.from_object(config.ProductionConfig())
elif os.environ.get('ENVIRONMENT') == 'Development':
    print("Getting development config")
    app.config.from_object(config.DevelopmentConfig())

redis_store = FlaskRedis(app)
assets = Environment(app)

js = Bundle('js/base.js',
            filters='jsmin', output='js/gen/packed.js')
assets.register('js_all', js)

sass = Bundle('*.sass', filters='sass', output='css/gen/sass.css')
css = Bundle(
            'css/base.css', sass,
            filters='cssmin', output="css/gen/all.css"
        )
assets.register('css_all', css)

oidc_config = config.OIDCConfig()

authentication = auth.OpenIDConnect(
    oidc_config
)

oidc = authentication.auth(app)

# Add secure Headers to satify observatory checks

sh = Secure_Headers()
sh.update(
    {
        'CSP': {
            'script-src':
                [
                    'self',
                    'ajax.googleapis.com',
                    'cdn.muicss.com',
                    'netdna.bootstrapcdn.com',
                    's.gravatar.com',
                    'fonts.googleapis.com'
                ],
            'style-src':
                [
                    'self',
                    'ajax.googleapis.com',
                    'cdn.muicss.com',
                    'netdna.bootstrapcdn.com',
                    's.gravatar.com',
                    'fonts.googleapis.com'
                ],
            'img-src':
                [
                    'self',
                    's.gravatar.com',
                    'i0.wp.com'
                ],
            'font-src':
                [
                    'self',
                    'netdna.bootstrapcdn.com',
                    'fonts.googleapis.com',
                    'cdn.muicss.com'
                ]
        }
    }
)

sh.update(
    {
        'HSTS':
            {
                'max-age': 1,
                'includeSubDomains': True,
                'preload': False
            }
    }
)

# Register the flask blueprint for SSE.
app.register_blueprint(sse, url_prefix='/stream')

@sse.before_request
def check_access():
    """Users can only view their own security alerts."""
    user = User(session)
    if request.args.get("channel") == Userhash():
        pass
    else:
        abort(403)

@app.route('/favicon.ico')
@sh.wrapper()
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static/img'),
                               'favicon.ico')

@app.route('/')
@sh.wrapper()
def home():
    return redirect('/dashboard', code=302)

@app.route('/dashboard')
@sh.wrapper()
@oidc.oidc_auth
def dashboard():
    """Primary dashboard the users will interact with."""
    AppFetcher().sync_config_and_images()
    user = User(session)
    alerts = redis_store.lrange(user.userhash(), 0, -1)
    all_apps = Application().apps
    apps = user.apps(all_apps)['apps']

    return render_template(
        'dashboard.html',
        user=user,
        apps=apps,
        alerts=alerts
    )

@app.route('/info')
@sh.wrapper()
@oidc.oidc_auth
def info():
    """Return the JSONified user session for debugging."""
    return jsonify(
            id_token=session['id_token'],
            access_token=session['access_token'],
            userinfo=session['userinfo']
        )

@app.route('/alert', methods = ['POST'])
@sh.wrapper()
def publish_alert():
    """
    Takes JSON post with user e-mail to alert.
    Minimum fields are email and message in the form
    of a dict.

    Example:
        {
          "user": {"email": "andrewkrug@gmail.com"},
            "message": "this is a security alert"
        }
    """
    #try:
    content = request.json
    #Send a real time event to the user
    m = hashlib.md5()
    m.update(content['user']['email'])
    channel = m.hexdigest()

    sse.publish(
        {"message": content['message']},
        type="alert",
        channel=channel
    )

    permanent_message = "Security Alert Logged at {date} : {message}".format(
        date=datetime.datetime.now(),
        message=content['message']
    )
    #Store the event in redis keyed to the users hashed e-mail
    redis_store.lpush(channel, permanent_message)

    return jsonify({'status': 'success'})
    #except:
    #    raise BadRequest('POST does not contain e-mail and message')
    #    return jsonify({'status': 'fail'})

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Read in pext files from directories with restart runs and output a recarray.
Requires a .lsp file to be in the path, or to be passed.

Usage:
   simple-pext.py [options] <dirs>...

Options:
    --help -h                 This help.
    --lsp=L -L l              Read this .lsp file specifically.
    --late-time=TIME -l TIME  Cut out after this time.
    --reverse -r              Reverse Y and Z.
    --massE=ME                Rest energy of the particle. [default: 0.511e6]
    --verbose -v              Print verbose.
    --range=R -R R            Only restrict to this number range of pexts.
    --output=O -o O           Output to this file. Otherwise, reckon based on the
                              .lsp file name.
'''
from docopt import docopt;
from lspreader.lspreader import read_pext, get_header;
from lspreader.pext import add_quantities;
import numpy as np;
from pys import parse_ituple, mkvprint
import re;
from lspreader.dotlsp import getdim,getpexts
import os
import numpy.lib.recfunctions as rfn;
import gzip;
from itertools import cycle;
opts = docopt(__doc__,help=True);
vprint = mkvprint(opts);

if opts['--lsp']:
    lspf=opts['--lsp'];
else:
    files = os.listdir('.');
    lspf=[f for f in files if re.search(".*\.lsp$",f)][0];
with open(lspf,"r") as f:
    lsp=f.read();
if not opts['--output']:
    outname = re.search("(.*)\.lsp$",lspf).group(1)+"-pext";
else:
    outname = opts['--output'];
dim=getdim(lsp);
pext_info = getpexts(lsp);

if opts['--range']:
    a=parse_ituple(opts['--range'],length=2);
    mnpext,mxpext = min(*a),max(*a);
else:
    mnpext,mxpext = float('-inf'),float('inf');
outkeys = set();
allkeys = [];
def getpextfnames(path):
    files = os.listdir(path);
    pext = [f for f in files if re.search("pext[0-9]+.p4",f)];
    key = [ float(re.search("pext([0-9]+).p4",f).group(1))
            for f in pext ];
    return [ ('{}/{}'.format(path,i),k) for i,k in zip(pext,key)
             if mnpext <= k <= mxpext];
pextfnames = [ i for path in opts['<dirs>'] for i in getpextfnames(path)];
#get headers from the first directory
headers = [ get_header(fname,gzip='guess')
            for fname in pextfnames[:len(opts['<dirs>'])]];
keys = np.unique([ i[1] for i in pextfnames ]);
pextplanes = {k:[] for k in keys};
for d in dirs:
    pextplanes[d[1]].append(d[0]);

def process_plane(paths, header, k):
    #invariant: the files should be in order they are passed.
    vprint('reading in files for {}'.format(k));
    d = [];
    for path in paths:
        openf = gzip.open if re.search(r'\.gz$', path) else open;
        with openf(path) as f:
            d.append(read_pext(f,header));
    d[:] = [
        rfn.rec_append_fields(
            di, 'species',
            np.ones(len(id)).astype(int)*pext_info[k]['species'])
        for di in d ];
    d = [ i for i in d if i['t'].shape[0] > 0];
    #make a mask of times less than the minimum of the next pexts
    #only take those in the previous run
    #only assign up to the last element of d.
    if len(d) > 1:
        d[:-1] = [ i[ i['t'] < j['t'].min() ] for i,j in zip(d[:-1],d[1:]) ];
    return d
vprint('');
d = [ di
      for paths,header, k in zip(pextplanes,cycle(headers),keys)
      for di in process_plane(paths,header,k)   ];
if len(d) == 0:
    print("no pext data");
    quit();
d = np.concatenate(d);
latetime = float(opts['--late-time']) if opts['--late-time'] else None;
vprint('sorting by times')
d.sort(order='t');
if latetime:
    print('cutting out times greater than {}'.format(latetime));
    d = d[ d['t'] <= latetime ];
#calculating quantities
if opts['--reverse']:
    dim = dim[:-2] + list(reversed(dim[-2:]))
massE = float(opts['--massE']) if opts['--massE'] else None;
d = add_quantities(d, dim, massE=massE);
np.save(outname, d);
<commit_msg>neext to add testing locally omg<commit_after>#!/usr/bin/env python
'''
Read in pext files from directories with restart runs and output a recarray.
Requires a .lsp file to be in the path, or to be passed.

Usage:
   simple-pext.py [options] <dirs>...

Options:
    --help -h                 This help.
    --lsp=L -L l              Read this .lsp file specifically.
    --late-time=TIME -l TIME  Cut out after this time.
    --reverse -r              Reverse Y and Z.
    --massE=ME                Rest energy of the particle. [default: 0.511e6]
    --verbose -v              Print verbose.
    --range=R -R R            Only restrict to this number range of pexts.
    --output=O -o O           Output to this file. Otherwise, reckon based on the
                              .lsp file name.
'''
from docopt import docopt;
from lspreader.lspreader import read_pext, get_header;
from lspreader.pext import add_quantities;
import numpy as np;
from pys import parse_ituple, mkvprint
import re;
from lspreader.dotlsp import getdim,getpexts
import os
import numpy.lib.recfunctions as rfn;
import gzip;
from itertools import cycle;
opts = docopt(__doc__,help=True);
vprint = mkvprint(opts);

if opts['--lsp']:
    lspf=opts['--lsp'];
else:
    files = os.listdir('.');
    lspf=[f for f in files if re.search(".*\.lsp$",f)][0];
with open(lspf,"r") as f:
    lsp=f.read();
if not opts['--output']:
    outname = re.search("(.*)\.lsp$",lspf).group(1)+"-pext";
else:
    outname = opts['--output'];
dim=getdim(lsp);
pext_info = getpexts(lsp);

if opts['--range']:
    a=parse_ituple(opts['--range'],length=2);
    mnpext,mxpext = min(*a),max(*a);
else:
    mnpext,mxpext = float('-inf'),float('inf');
outkeys = set();
allkeys = [];
def getpextfnames(path):
    files = os.listdir(path);
    pext = [f for f in files if re.search("pext[0-9]+.p4",f)];
    key = [ float(re.search("pext([0-9]+).p4",f).group(1))
            for f in pext ];
    return [ ('{}/{}'.format(path,i),k) for i,k in zip(pext,key)
             if mnpext <= k <= mxpext];
pextfnames = [ i for path in opts['<dirs>'] for i in getpextfnames(path)];
#get headers from the first directory
headers = [ get_header(fname[0],gzip='guess')
            for fname in pextfnames[:len(opts['<dirs>'])]];
keys = np.unique([ i[1] for i in pextfnames ]);
pextplanes = {k:[] for k in keys};
for d in dirs:
    pextplanes[d[1]].append(d[0]);

def process_plane(paths, header, k):
    #invariant: the files should be in order they are passed.
    vprint('reading in files for {}'.format(k));
    d = [];
    for path in paths:
        openf = gzip.open if re.search(r'\.gz$', path) else open;
        with openf(path) as f:
            d.append(read_pext(f,header));
    d[:] = [
        rfn.rec_append_fields(
            di, 'species',
            np.ones(len(id)).astype(int)*pext_info[k]['species'])
        for di in d ];
    d = [ i for i in d if i['t'].shape[0] > 0];
    #make a mask of times less than the minimum of the next pexts
    #only take those in the previous run
    #only assign up to the last element of d.
    if len(d) > 1:
        d[:-1] = [ i[ i['t'] < j['t'].min() ] for i,j in zip(d[:-1],d[1:]) ];
    return d
vprint('reading planes');
d = [ di
      for paths,header, k in zip(pextplanes,cycle(headers),keys)
      for di in process_plane(paths,header,k)   ];
if len(d) == 0:
    print("no pext data");
    quit();
d = np.concatenate(d);
latetime = float(opts['--late-time']) if opts['--late-time'] else None;
vprint('sorting by times')
d.sort(order='t');
if latetime:
    print('cutting out times greater than {}'.format(latetime));
    d = d[ d['t'] <= latetime ];
#calculating quantities
if opts['--reverse']:
    dim = dim[:-2] + list(reversed(dim[-2:]))
massE = float(opts['--massE']) if opts['--massE'] else None;
d = add_quantities(d, dim, massE=massE);
np.save(outname, d);
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
conference.py -- Udacity conference server-side Python App Engine API;
    uses Google Cloud Endpoints

$Id: conference.py,v 1.25 2014/05/24 23:42:19 wesc Exp wesc $

created by wesc on 2014 apr 21

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

from datetime import datetime
import json
import os
import time

import endpoints
from protorpc import messages
from protorpc import message_types
from protorpc import remote

from google.appengine.api import urlfetch
from google.appengine.ext import ndb

from models import Profile
from models import ProfileMiniForm
from models import ProfileForm
from models import TeeShirtSize

from utils import getUserId

from settings import WEB_CLIENT_ID

EMAIL_SCOPE = endpoints.EMAIL_SCOPE
API_EXPLORER_CLIENT_ID = endpoints.API_EXPLORER_CLIENT_ID

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

@endpoints.api( name='conference',
                version='v1',
                allowed_client_ids=[WEB_CLIENT_ID, API_EXPLORER_CLIENT_ID],
                scopes=[EMAIL_SCOPE])
class ConferenceApi(remote.Service):
    """Conference API v0.1"""

# - - - Profile objects - - - - - - - - - - - - - - - - - - -

    def _copyProfileToForm(self, prof):
        """Copy relevant fields from Profile to ProfileForm."""
        # copy relevant fields from Profile to ProfileForm
        pf = ProfileForm()
        for field in pf.all_fields():
            if hasattr(prof, field.name):
                # convert t-shirt string to Enum; just copy others
                if field.name == 'teeShirtSize':
                    setattr(pf, field.name, getattr(TeeShirtSize, getattr(prof, field.name)))
                else:
                    setattr(pf, field.name, getattr(prof, field.name))
        pf.check_initialized()
        return pf

    def _getProfileFromUser(self):
        """Return user Profile from datastore, creating new one if non-existent."""
        ## TODO 2
        ## step 1: make sure user is authed
        ## uncomment the following lines:
        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        user_id = getUserId(user)
        p_key = ndb.Key(Profile, user_id)
        profile = p_key.get()
        profile = None
        ## step 2: create a new Profile from logged in user data
        ## you can use user.nickname() to get displayName
        ## and user.email() to get mainEmail
        if not profile:
            profile = Profile(
                userId = None,
                key = None,
                displayName = user.nickname(),
                mainEmail= user.email(),
                teeShirtSize = str(TeeShirtSize.NOT_SPECIFIED),
            )
            profile.put()

        return profile      # return Profile

    def _doProfile(self, save_request=None):
        """Get user Profile and return to user, possibly updating it first."""
        # get user Profile
        prof = self._getProfileFromUser()

        # if saveProfile(), process user-modifyable fields
        if save_request:
            for field in ('displayName', 'teeShirtSize'):
                if hasattr(save_request, field):
                    val = getattr(save_request, field)
                    if val:
                        setattr(prof, field, str(val))

        # return ProfileForm
        return self._copyProfileToForm(prof)

    @endpoints.method(message_types.VoidMessage, ProfileForm,
            path='profile', http_method='GET', name='getProfile')
    def getProfile(self, request):
        """Return user profile."""
        return self._doProfile()

    # TODO 1
    # 1. change request class
    # 2. pass request to _doProfile function
    @endpoints.method(ProfileMiniForm, ProfileForm,
            path='profile', http_method='POST', name='saveProfile')
    def saveProfile(self, request):
        """Update & return user profile."""
        return self._doProfile(request)

# registers API
api = endpoints.api_server([ConferenceApi])
<commit_msg> clean out TODO comments<commit_after>#!/usr/bin/env python

"""
conference.py -- Udacity conference server-side Python App Engine API;
    uses Google Cloud Endpoints

$Id: conference.py,v 1.25 2014/05/24 23:42:19 wesc Exp wesc $

created by wesc on 2014 apr 21

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

from datetime import datetime
import json
import os
import time

import endpoints
from protorpc import messages
from protorpc import message_types
from protorpc import remote

from google.appengine.api import urlfetch
from google.appengine.ext import ndb

from models import Profile
from models import ProfileMiniForm
from models import ProfileForm
from models import TeeShirtSize

from utils import getUserId

from settings import WEB_CLIENT_ID

EMAIL_SCOPE = endpoints.EMAIL_SCOPE
API_EXPLORER_CLIENT_ID = endpoints.API_EXPLORER_CLIENT_ID

# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

@endpoints.api( name='conference',
                version='v1',
                allowed_client_ids=[WEB_CLIENT_ID, API_EXPLORER_CLIENT_ID],
                scopes=[EMAIL_SCOPE])
class ConferenceApi(remote.Service):
    """Conference API v0.1"""

# - - - Profile objects - - - - - - - - - - - - - - - - - - -

    def _copyProfileToForm(self, prof):
        """Copy relevant fields from Profile to ProfileForm."""
        # copy relevant fields from Profile to ProfileForm
        pf = ProfileForm()
        for field in pf.all_fields():
            if hasattr(prof, field.name):
                # convert t-shirt string to Enum; just copy others
                if field.name == 'teeShirtSize':
                    setattr(pf, field.name, getattr(TeeShirtSize, getattr(prof, field.name)))
                else:
                    setattr(pf, field.name, getattr(prof, field.name))
        pf.check_initialized()
        return pf

    def _getProfileFromUser(self):
        """Return user Profile from datastore, creating new one if non-existent."""

        user = endpoints.get_current_user()
        if not user:
            raise endpoints.UnauthorizedException('Authorization required')

        user_id = getUserId(user)
        p_key = ndb.Key(Profile, user_id)
        profile = p_key.get()
        profile = None

        if not profile:
            profile = Profile(
                userId = None,
                key = None,
                displayName = user.nickname(),
                mainEmail= user.email(),
                teeShirtSize = str(TeeShirtSize.NOT_SPECIFIED),
            )
            profile.put()

        return profile      # return Profile

    def _doProfile(self, save_request=None):
        """Get user Profile and return to user, possibly updating it first."""
        # get user Profile
        prof = self._getProfileFromUser()

        # if saveProfile(), process user-modifyable fields
        if save_request:
            for field in ('displayName', 'teeShirtSize'):
                if hasattr(save_request, field):
                    val = getattr(save_request, field)
                    if val:
                        setattr(prof, field, str(val))

        # return ProfileForm
        return self._copyProfileToForm(prof)

    @endpoints.method(message_types.VoidMessage, ProfileForm,
            path='profile', http_method='GET', name='getProfile')
    def getProfile(self, request):
        """Return user profile."""
        return self._doProfile()

    @endpoints.method(ProfileMiniForm, ProfileForm,
            path='profile', http_method='POST', name='saveProfile')
    def saveProfile(self, request):
        """Update & return user profile."""
        return self._doProfile(request)

# registers API
api = endpoints.api_server([ConferenceApi])
<|endoftext|>