<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import cgi
import time
from datetime import date, datetime, timedelta
import itertools
from docutils.examples import html_parts

from openquake.commonlib.datastore import read
from openquake.calculators.views import view_fullreport
from openquake.engine.logs import dbcmd

tablecounter = itertools.count(0)

def html(header_rows):
    """
    Convert a list of tuples describing a table into a HTML string
    """
    name = 'table%d' % next(tablecounter)
    return HtmlTable([map(str, row) for row in header_rows], name).render()

class HtmlTable(object):
    """
    Convert a sequence header+body into a HTML table.
    """
    css = """\
    tr.evenRow { background-color: lightgreen }
    tr.oddRow { }
    th { background-color: lightblue }
    """
    maxrows = 5000
    border = "1"
    summary = ""

    def __init__(self, header_plus_body, name='noname',
                 empty_table='Empty table'):
        header, body = header_plus_body[0], header_plus_body[1:]
        self.name = name
        self.empty_table = empty_table
        rows = []  # rows is a finite sequence of tuples
        for i, row in enumerate(body):
            if i == self.maxrows:
                rows.append(
                    ["Table truncated because too big: more than %s rows" % i])
                break
            rows.append(row)
        self.rows = rows
        self.header = tuple(header)  # horizontal header

    def render(self, dummy_ctxt=None):
        out = "\n%s\n" % "".join(list(self._gen_table()))
        if not self.rows:
            out += '%s' % cgi.escape(self.empty_table, quote=True)
        return out

    def _gen_table(self):
        yield '
\n'\
              % (self.name, self.border, self.summary)
        yield '
\n'
        yield '
%s
\n' % ''.join(
            '%s	\n' % h for h in self.header)
        yield '
\n'
        yield '
'
        for r, row in enumerate(self.rows):
            yield '
\n' % ["even", "odd"][r % 2]
            for col in row:
                yield '%s	\n' % col
            yield '
\n'
        yield '
\n'
        yield '
\n'

JOB_STATS = '''
SELECT id, user_name, start_time, stop_time, status FROM job WHERE id=?;
'''

ALL_JOBS = '''
SELECT id, user_name, status, ds_calc_dir FROM job
WHERE start_time >= ? AND start_time < ? ORDER BY stop_time
'''

PAGE_TEMPLATE = '''\
<!DOCTYPE html>

<meta charset="UTF-8">

<link rel="stylesheet" href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/themes/smoothness/jquery-ui.css" />

%s

'''

def make_tabs(tag_ids, tag_status, tag_contents):
    """
    Return a HTML string containing all the tabs we want to display
    """
    templ = '''

%s

%s

'''
    lis = []
    contents = []
    for i, (tag_id, status, tag_content) in enumerate(
            zip(tag_ids, tag_status, tag_contents), 1):
        mark = '.' if status == 'complete' else '!'
        lis.append('
%s%s
' % (i, tag_id, mark))
        contents.append('
%s
' % (
            i, tag_content))
    return templ % ('\n'.join(lis), '\n'.join(contents))

def add_duration(stats):
    START_TIME = 2
    STOP_TIME = 3
    stats[0] = stats[0] + ['duration']
    for i, rec in enumerate(stats[1:], 1):
        duration = (rec[STOP_TIME] - rec[START_TIME]).total_seconds()
        stats[i] = rec + (duration,)

def make_report(isodate='today'):
    """
    Build a HTML report with the computations performed at the given isodate.
    Return the name of the report, which is saved in the current directory.
    """
    if isodate == 'today':
        isodate = date.today()
    else:
        isodate = date(*time.strptime(isodate, '%Y-%m-%d')[:3])
    isodate1 = isodate + timedelta(1)  # +1 day

    tag_ids = []
    tag_status = []
    tag_contents = []

    # the fetcher returns an header which is stripped with [1:]
    jobs = dbcmd(
        'fetch', ALL_JOBS, isodate.isoformat(), isodate1.isoformat())[1:]
    page = '
%d job(s) finished before midnight of %s
' % (
        len(jobs), isodate)
    for job_id, user, status, ds_calc in jobs:
        tag_ids.append(job_id)
        tag_status.append(status)
        stats = dbcmd('fetch', JOB_STATS, job_id)
        if not stats[1:]:
            continue
        (job_id, user, start_time, stop_time, status) = stats[1]
        try:
            ds = read(job_id, datadir=os.path.dirname(ds_calc))
            txt = view_fullreport('fullreport', ds).decode('utf-8')
            report = html_parts(txt)
        except Exception as exc:
            report = dict(
                html_title='Could not generate report: %s' % cgi.escape(
                    unicode(exc), quote=True),
                fragment='')

        page = report['html_title']

        add_duration(stats)
        page += html(stats)

        page += report['fragment']

        tag_contents.append(page)

    page = make_tabs(tag_ids, tag_status, tag_contents) + (
        'Report last updated: %s' % datetime.now())
    fname = 'jobs-%s.html' % isodate
    with open(fname, 'w') as f:
        f.write(PAGE_TEMPLATE % page.encode('utf-8'))
    return fname
<commit_msg>Small fix in make_html_report.add_duration<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2016 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import cgi
import time
from datetime import date, datetime, timedelta
import itertools
from docutils.examples import html_parts

from openquake.commonlib.datastore import read
from openquake.calculators.views import view_fullreport
from openquake.engine.logs import dbcmd

tablecounter = itertools.count(0)

def html(header_rows):
    """
    Convert a list of tuples describing a table into a HTML string
    """
    name = 'table%d' % next(tablecounter)
    return HtmlTable([map(str, row) for row in header_rows], name).render()

class HtmlTable(object):
    """
    Convert a sequence header+body into a HTML table.
    """
    css = """\
    tr.evenRow { background-color: lightgreen }
    tr.oddRow { }
    th { background-color: lightblue }
    """
    maxrows = 5000
    border = "1"
    summary = ""

    def __init__(self, header_plus_body, name='noname',
                 empty_table='Empty table'):
        header, body = header_plus_body[0], header_plus_body[1:]
        self.name = name
        self.empty_table = empty_table
        rows = []  # rows is a finite sequence of tuples
        for i, row in enumerate(body):
            if i == self.maxrows:
                rows.append(
                    ["Table truncated because too big: more than %s rows" % i])
                break
            rows.append(row)
        self.rows = rows
        self.header = tuple(header)  # horizontal header

    def render(self, dummy_ctxt=None):
        out = "\n%s\n" % "".join(list(self._gen_table()))
        if not self.rows:
            out += '%s' % cgi.escape(self.empty_table, quote=True)
        return out

    def _gen_table(self):
        yield '
\n'\
              % (self.name, self.border, self.summary)
        yield '
\n'
        yield '
%s
\n' % ''.join(
            '%s	\n' % h for h in self.header)
        yield '
\n'
        yield '
'
        for r, row in enumerate(self.rows):
            yield '
\n' % ["even", "odd"][r % 2]
            for col in row:
                yield '%s	\n' % col
            yield '
\n'
        yield '
\n'
        yield '
\n'

JOB_STATS = '''
SELECT id, user_name, start_time, stop_time, status FROM job WHERE id=?;
'''

ALL_JOBS = '''
SELECT id, user_name, status, ds_calc_dir FROM job
WHERE start_time >= ? AND start_time < ? ORDER BY stop_time
'''

PAGE_TEMPLATE = '''\
<!DOCTYPE html>

<meta charset="UTF-8">

<link rel="stylesheet" href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.11.2/themes/smoothness/jquery-ui.css" />

%s

'''

def make_tabs(tag_ids, tag_status, tag_contents):
    """
    Return a HTML string containing all the tabs we want to display
    """
    templ = '''

%s

%s

'''
    lis = []
    contents = []
    for i, (tag_id, status, tag_content) in enumerate(
            zip(tag_ids, tag_status, tag_contents), 1):
        mark = '.' if status == 'complete' else '!'
        lis.append('
%s%s
' % (i, tag_id, mark))
        contents.append('
%s
' % (
            i, tag_content))
    return templ % ('\n'.join(lis), '\n'.join(contents))

def add_duration(stats):
    START_TIME = 2
    STOP_TIME = 3
    stats[0] = stats[0] + ['duration']
    for i, rec in enumerate(stats[1:], 1):
        if rec[STOP_TIME] is None:
            duration = '?'
        else:
            duration = (rec[STOP_TIME] - rec[START_TIME]).total_seconds()
        stats[i] = rec + (duration,)

def make_report(isodate='today'):
    """
    Build a HTML report with the computations performed at the given isodate.
    Return the name of the report, which is saved in the current directory.
    """
    if isodate == 'today':
        isodate = date.today()
    else:
        isodate = date(*time.strptime(isodate, '%Y-%m-%d')[:3])
    isodate1 = isodate + timedelta(1)  # +1 day

    tag_ids = []
    tag_status = []
    tag_contents = []

    # the fetcher returns an header which is stripped with [1:]
    jobs = dbcmd(
        'fetch', ALL_JOBS, isodate.isoformat(), isodate1.isoformat())[1:]
    page = '
%d job(s) finished before midnight of %s
' % (
        len(jobs), isodate)
    for job_id, user, status, ds_calc in jobs:
        tag_ids.append(job_id)
        tag_status.append(status)
        stats = dbcmd('fetch', JOB_STATS, job_id)
        if not stats[1:]:
            continue
        (job_id, user, start_time, stop_time, status) = stats[1]
        try:
            ds = read(job_id, datadir=os.path.dirname(ds_calc))
            txt = view_fullreport('fullreport', ds).decode('utf-8')
            report = html_parts(txt)
        except Exception as exc:
            report = dict(
                html_title='Could not generate report: %s' % cgi.escape(
                    unicode(exc), quote=True),
                fragment='')

        page = report['html_title']

        add_duration(stats)
        page += html(stats)

        page += report['fragment']

        tag_contents.append(page)

    page = make_tabs(tag_ids, tag_status, tag_contents) + (
        'Report last updated: %s' % datetime.now())
    fname = 'jobs-%s.html' % isodate
    with open(fname, 'w') as f:
        f.write(PAGE_TEMPLATE % page.encode('utf-8'))
    return fname
<|endoftext|>


<commit_before># custom exceptions
class StillRunningException(Exception):
    """EC2 AWSEM instance is still running (job not complete)"""
    pass

class EC2StartingException(Exception):
    """EC2 AWSEM instance is still starting (job not complete)"""
    pass

class AWSEMJobErrorException(Exception):
    """There is an error from a worklow run on the EC2 AWSEM instance"""
    pass

class DependencyStillRunningException(Exception):
    pass

class DependencyFailedException(Exception):
    pass

class EC2LaunchException(Exception):
    pass

class EC2UnintendedTerminationException(Exception):
    pass

class EC2IdleException(Exception):
    pass

class EC2InstanceLimitException(Exception):
    pass

class EC2InstanceLimitWaitException(Exception):
    pass

class MissingFieldInInputJsonException(Exception):
    pass

class MalFormattedInputJsonException(Exception):
    pass

class MalFormattedPostrunJsonException(Exception):
    pass

class MetricRetrievalException(Exception):
    pass
<commit_msg>added awsem error handler<commit_after>import re

# custom exceptions
class AWSEMJobErrorException(Exception):
    """There is an error from a worklow run on the EC2 AWSEM instance."""
    pass

class AWSEMErrorHandler(object):

    class AWSEMError(object):
        def __init__(self, error_type, pattern_in_log):
            self.error_type = error_type
            self.pattern_in_log = pattern_in_log

    ErrorList = [
        AWSEMError('No peak called', 'Exception: File is empty (.+.regionPeak.gz)')
    ]

    #AWSEMErrorExceptionList = [NoPeakException]

    def parse_log(self, log):
        #for ex in self.AWSEMErrorExceptionList:
        for ex in self.ErrorList:
            res = re.search(ex.pattern_in_log, log)
            if res:
                match = res.string[res.regs[0][0]:res.regs[0][1] + 1]
                msg = "%s: %s" % (ex.error_type, match)
                return AWSEMJobErrorException(msg)
        return

class StillRunningException(Exception):
    """EC2 AWSEM instance is still running (job not complete)"""
    pass

class EC2StartingException(Exception):
    """EC2 AWSEM instance is still starting (job not complete)"""
    pass
class DependencyStillRunningException(Exception):
    pass

class DependencyFailedException(Exception):
    pass

class EC2LaunchException(Exception):
    pass

class EC2UnintendedTerminationException(Exception):
    pass

class EC2IdleException(Exception):
    pass

class EC2InstanceLimitException(Exception):
    pass

class EC2InstanceLimitWaitException(Exception):
    pass

class MissingFieldInInputJsonException(Exception):
    pass

class MalFormattedInputJsonException(Exception):
    pass

class MalFormattedPostrunJsonException(Exception):
    pass

class MetricRetrievalException(Exception):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Module to compute and plot an aggregate loss curve.
"""

import os

from openquake.logs import LOG
from openquake.output import curve
from openquake.risk import job as risk_job
from openquake.risk import probabilistic_event_based as prob

def _filename(job_id):
    """Return the name of the generated file."""
    return "%s-aggregate-loss-curve.svg" % job_id

def _for_plotting(loss_curve, time_span):
    """Translate a loss curve into a dictionary compatible to
    the interface defined in CurvePlot.write."""
    data = {}

    data["AggregateLossCurve"] = {}
    data["AggregateLossCurve"]["abscissa"] = tuple(loss_curve.abscissae)
    data["AggregateLossCurve"]["ordinate"] = tuple(loss_curve.ordinates)
    data["AggregateLossCurve"]["abscissa_property"] = "Economic Losses"
    data["AggregateLossCurve"]["ordinate_property"] = \
            "PoE in %s years" % (str(time_span))

    data["AggregateLossCurve"]["curve_title"] = "Aggregate Loss Curve"

    return data

def compute_aggregate_curve(job):
    """Compute and plot an aggreate loss curve.

    This function expects to find in kvs a set of pre computed
    GMFs and assets.

    This function is triggered only if the AGGREGATE_LOSS_CURVE
    parameter is specified in the configuration file.

    :param job: the job the engine is currently processing.
    :type job: openquake.risk.job.probabilistic.ProbabilisticEventMixin
    """
    print "compute_aggregate_curve"
    if not job.has("AGGREGATE_LOSS_CURVE"):
        LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
                "skipping aggregate loss curve computation...")

        return

    epsilon_provider = risk_job.EpsilonProvider(job.params)
    print "type(epsilon_provider)", type(epsilon_provider)
    aggregate_loss_curve = prob.AggregateLossCurve.from_kvs(job.id, epsilon_provider)

    path = os.path.join(job.params["BASE_PATH"],
            job.params["OUTPUT_DIR"], _filename(job.id))

    plotter = curve.CurvePlot(path)
    plotter.write(_for_plotting(
            aggregate_loss_curve.compute(),
            job.params["INVESTIGATION_TIME"]), autoscale_y=False)

    plotter.close()
    LOG.debug("Aggregate loss curve stored at %s" % path)
<commit_msg>removed debug statement<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Module to compute and plot an aggregate loss curve.
"""

import os

from openquake.logs import LOG
from openquake.output import curve
from openquake.risk import job as risk_job
from openquake.risk import probabilistic_event_based as prob

def _filename(job_id):
    """Return the name of the generated file."""
    return "%s-aggregate-loss-curve.svg" % job_id

def _for_plotting(loss_curve, time_span):
    """Translate a loss curve into a dictionary compatible to
    the interface defined in CurvePlot.write."""
    data = {}

    data["AggregateLossCurve"] = {}
    data["AggregateLossCurve"]["abscissa"] = tuple(loss_curve.abscissae)
    data["AggregateLossCurve"]["ordinate"] = tuple(loss_curve.ordinates)
    data["AggregateLossCurve"]["abscissa_property"] = "Economic Losses"
    data["AggregateLossCurve"]["ordinate_property"] = \
            "PoE in %s years" % (str(time_span))

    data["AggregateLossCurve"]["curve_title"] = "Aggregate Loss Curve"

    return data

def compute_aggregate_curve(job):
    """Compute and plot an aggreate loss curve.

    This function expects to find in kvs a set of pre computed
    GMFs and assets.

    This function is triggered only if the AGGREGATE_LOSS_CURVE
    parameter is specified in the configuration file.

    :param job: the job the engine is currently processing.
    :type job: openquake.risk.job.probabilistic.ProbabilisticEventMixin
    """
    print "compute_aggregate_curve"
    if not job.has("AGGREGATE_LOSS_CURVE"):
        LOG.debug("AGGREGATE_LOSS_CURVE parameter not specified, " \
                "skipping aggregate loss curve computation...")

        return

    epsilon_provider = risk_job.EpsilonProvider(job.params)
    aggregate_loss_curve = prob.AggregateLossCurve.from_kvs(job.id, epsilon_provider)

    path = os.path.join(job.params["BASE_PATH"],
            job.params["OUTPUT_DIR"], _filename(job.id))

    plotter = curve.CurvePlot(path)
    plotter.write(_for_plotting(
            aggregate_loss_curve.compute(),
            job.params["INVESTIGATION_TIME"]), autoscale_y=False)

    plotter.close()
    LOG.debug("Aggregate loss curve stored at %s" % path)
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
Script to run FIR analysis

EXAMPLE:
python run_fir.py -experiment ap_memory_raw -ts_experiment mvpa_raw -mask rh-hippocampus-tail -len_et 7
"""

from __future__ import division
import os.path as op
import itertools
import numpy as np
import scipy as sp
import pandas as pd
import nibabel as nib
from scipy import stats
import statsmodels.api as sm
import matplotlib as mpl
import matplotlib.pyplot as plt
import nitime as nit

import sys
import argparse
import shutil
from textwrap import dedent

import os
from glob import glob
import hashlib
from scipy.interpolate import interp1d
from lyman import gather_project_info

import lyman
from lyman import mvpa, evoked
from lyman import tools
import seaborn as sns
import moss

# Define some functions

def evoked_extract_subject(subj, mask_name, n_runs, summary_func=np.mean,
                           exp_name=None, ts_exp=None, residual=False):
    """Extract timeseries from within a mask, summarizing flexibly.
    Parameters
    ----------
    subj : string
        subject name
    mask_name : string
        name of mask in data hierarchy
    summary_func : callable or None
        callable to reduce data over voxel dimensions. can take an
        ``axis`` argument to operate over each frame, if this
        argument does not exist the function will be called on the
        n_tr x n_voxel array. if None, simply returns all voxels.
    residual : boolean
        If True, extract from the registered residual timecourse.
    exp_name : string
        experiment name, if not using the default experiment
    Returns
    -------
    data : dict with ndarray
        datta array is n_runs x n_timepoint x n_dimension,
        data are not otherwise altered
    """
    project = gather_project_info()
    if exp_name is None:
        exp_name = project["default_exp"]

    if ts_exp is None:
        ts_exp = exp_name

    # Get a path to the file where
    cache_dir = op.join(project["analysis_dir"],
                        exp_name, subj, "evoked")

    try:
        os.makedirs(cache_dir)
    except OSError:
        pass

    if summary_func is None:
        func_name = ""
    else:
        func_name = summary_func.__name__
    cache_fname = mask_name + "_" + func_name
    cache_fname = cache_fname.strip("_") + ".npz"
    cache_file = op.join(cache_dir, cache_fname)

    # Get paths to the relevant files
    mask_file = op.join(project["data_dir"], subj, "masks",
                        "%s.nii.gz" % mask_name)
    ts_dir = op.join(project["analysis_dir"], ts_exp, subj,
                     "reg", "epi", "unsmoothed")

    ftemp = op.join(ts_dir, "run_{:d}/{}_xfm.nii.gz")
    fstem = "res4d" if residual else "timeseries"
    ts_files = [ftemp.format(r_i, fstem) for r_i in range(1,n_runs+1)]
    print ts_files

    # Get the hash value for this extraction
    cache_hash = hashlib.sha1()
    cache_hash.update(mask_name)
    cache_hash.update(str(op.getmtime(mask_file)))
    for ts_file in ts_files:
        cache_hash.update(str(op.getmtime(ts_file)))
    cache_hash = cache_hash.hexdigest()

    # If the file exists and the hash matches, return the data
    if op.exists(cache_file):
        with np.load(cache_file) as cache_obj:
            if cache_hash == str(cache_obj["hash"]):
                return dict(cache_obj.items())

    # Otherwise, do the extraction
    data = []
    mask = nib.load(mask_file).get_data().astype(bool)
    for run, ts_file in enumerate(ts_files):
        ts_data = nib.load(ts_file).get_data()
        roi_data = ts_data[mask].T

        if summary_func is None:
            data.append(roi_data)
            continue

        # Try to use the axis argument to summarize over voxels
        try:
            roi_data = summary_func(roi_data, axis=1)

            print "Taking mean for run " + str(run +1)
        # Catch a TypeError and just call the function
        # This lets us do e.g. a PCA
        except TypeError:
            roi_data = summary_func(roi_data)

        data.append(roi_data)

    data = np.array(list(map(np.squeeze, data)))

    # Save the results and return them
    data_dict = dict(data=data, subj=subj, hash=cache_hash)
    np.savez(cache_file, **data_dict)

    return data_dict

def evoked_extract_group(mask_name, n_runs, summary_func=np.mean,
                         exp_name=None, ts_exp=None, subjects=None, dv=None):
    """Extract timeseries from within a mask, summarizing flexibly.
    Parameters
    ----------
    mask_name : string
        name of mask in data hierarchy
    summary_func : callable or None
        callable to reduce data over voxel dimensions. can take an
        ``axis`` argument to operate over each frame, if this
        argument does not exist the function will be called on the
        n_tr x n_voxel array. if None, simply returns all voxels.
    exp_name : string
        experiment name, if not using the default experiment
    subjects : sequence of strings
        subjects to operate over if not using default subject list
    dv : IPython cluster direct view
        if provided with view on cluster, executes in parallel over
        subjects
    Returns
    -------
    data : list of dicts with ndarrays
        each array is squeezed n_runs x n_timepoint x n_dimension
        data is not otherwise altered
    """
    if dv is None:
        import __builtin__
        _map = __builtin__.map
    else:
        _map = dv.map_sync

    if subjects is None:
        subj_file = op.join(os.environ["LYMAN_DIR"], "subjects.txt")
        subjects = np.loadtxt(subj_file, str).tolist()

    mask_name = [mask_name for s in subjects]
    summary_func = [summary_func for s in subjects]
    exp_name = [exp_name for s in subjects]
    ts_exp = [ts_exp for s in subjects]
    n_runs = [n_runs for s in subjects]

    data = _map(evoked_extract_subject, subjects, mask_name, n_runs,
                summary_func, exp_name, ts_exp)
    for d in data:
        d["data"] = np.asarray(d["data"])

    return data

def _evoked_1d(data, events, n_bins, tr, calc_method, correct_baseline):

    events_ts = nit.TimeSeries(events, sampling_interval=tr)
    data_ts = nit.TimeSeries(data, sampling_interval=tr)

    analyzer = nit.analysis.EventRelatedAnalyzer(data_ts, events_ts, n_bins)

    evoked_data = getattr(analyzer, calc_method)
    evoked_data = np.asarray(evoked_data).T.astype(float)

    if evoked_data.ndim == 1:
        evoked_data = np.array([evoked_data])
    if correct_baseline:
        evoked_data = evoked_data - evoked_data[:, 0, None]
    return evoked_data

def _evoked_2d(data, events, n_bins, tr, calc_method, correct_baseline):

    evoked_data = []
    for data_i in data.T:
        evoked_data_i = _evoked_1d(data_i, events, n_bins, tr,
                                   calc_method, correct_baseline)
        evoked_data.append(evoked_data_i)

    evoked_data = np.transpose(evoked_data, (1, 2, 0))
    return evoked_data

def integrate_evoked(evoked, axis=-1):
    """Integrate a peristumulus timecourse.
    Parameters
    ----------
    evoked : list of 2D arrays or 2D array
        values of evoked datapoints
    Returns
    -------
    int_evoked : squeezed array
        evoked values integrated over the time dimension
    """
    return sp.trapz(evoked, axis=axis)

def calculate_evoked_ts(data, n_bins, problem=None, events=None, tr=2,
                        calc_method="FIR", offset=0, upsample=1,
                        percent_change=True, correct_baseline=True,
                        event_names=None):
    """Calcuate an evoked response for a list of datapoints.
    Parameters
    ----------
    data : sequence of n_run x n_tp arrays
        timeseries data
    n_bins : int
        number of bins for the peristumulus trace
    problem : string
        problem name for event file in data hierarchy
        overrides `events` if both are passed
    events : dataframe or list of dataframes
        one dataframe describing event information for each subj.
        must contain `onset`, `run`, and `condition` columns
        caution: `run` should be 1-based
    tr : int
        original time resolution of the data (seconds)
    upsample : int
        factor to upsample the data with using cubic splines
    calc_method : string
        name of method on nitime EventRelatedAnalyzer object to
        calculate the evoked response.
    offset : float
        value to adjust onset times by
    percent_change : boolean
        if True, convert signal to percent change by run
    correct_baseline : boolean
        if True, adjust evoked trace to be 0 in first bin
    event_names : list of strings
        names of conditions, otherwise uses sorted unique
        values for the condition field in the event dataframe
    Returns
    -------
    evoked_ds :
    """

    project = lyman.gather_project_info()
    design_template = op.join(project["data_dir"], "%s",
                              "design/%s.csv" % problem)

    col_names = ['subj', 'event', 'time', 'response']
    evoked_ds = pd.DataFrame(columns=col_names)

    for i, data_i in enumerate(data):

        # Get event information
        subj = data_i["subj"]
        events_i = pd.read_csv(design_template % subj)

        # Map from event names to integer index values
        event_names = sorted(events_i.condition.unique())
        event_map = pd.Series(range(1, len(event_names) + 1),
                              index=event_names)

        # Create the timeseries of event occurances
        calc_tr = float(tr) / upsample

        event_list = []
        data_list = []
        for run, run_data in enumerate(data_i["data"], 1):

            # Possibly upsample the data
            if upsample != 1:
                time_points = len(run_data)
                x = np.linspace(0, time_points - 1, time_points)
                xx = np.linspace(0, time_points,
                                 time_points * upsample + 1)[:-upsample]
                interpolator = interp1d(x, run_data, "cubic", axis=0)
                run_data = interpolator(xx)

            run_events = events_i[events_i.run == run]
            run_events.onset += offset

            event_id = np.zeros(len(run_data), int)
            event_index = np.array(run_events.onset / calc_tr).astype(int)
            event_id[event_index] = run_events.condition.map(event_map)
            event_list.append(event_id)

            if percent_change:
                run_data = nit.utils.percent_change(run_data, ax=0)
            data_list.append(run_data)

        # Set up the Nitime objects
        event_info = np.concatenate(event_list)
        data = np.concatenate(data_list, axis=0)

        # Do the calculations
        calc_bins = n_bins * upsample

        print 'Data dimensions: ' + str(data.ndim)
        if data.ndim == 1:
            evoked_data = _evoked_1d(data, event_info, calc_bins, calc_tr,
                                     calc_method, correct_baseline)
        elif data.ndim == 2:
            evoked_data = _evoked_2d(data, event_info, n_bins, calc_tr,
                                     calc_method, correct_baseline)

        for event_num, d in enumerate(evoked_data):
            evoked_ds = evoked_ds.append(pd.DataFrame({'subj': subj,
                                                       'event': event_names[event_num],
                                                       'time': np.arange(1, n_bins+1),
                                                       'response': pd.Series(evoked_data[event_num])}))

    return evoked_ds

def main(arglist):

    args = parse_args(arglist)

    project = lyman.gather_project_info()
    exp = lyman.gather_experiment_info(args.experiment)
    anal_dir = project["analysis_dir"]
    data_dir = project["data_dir"]

    subjects = pd.Series(lyman.determine_subjects(), name="subj")

    all_rois = ['lh-hippocampus-tail', 'rh-hippocampus-tail',
                'lh-hippocampus-head', 'rh-hippocampus-head']

    def extract_group(subjects, args, exp):
        exp_name = args.experiment
        d = evoked_extract_group(args.mask, exp['n_runs'], subjects=subjects,
                                 exp_name=exp_name, ts_exp=args.ts_experiment)

        return d

    # Extract the timeseries from within a mask
    data = extract_group(subjects, args, exp)

    # Calculate evoked response (FIR model)
    signal = calculate_evoked_ts(data, args.len_et,
                                 problem=exp['design_name'],
                                 tr=int(exp['TR']))

    signal.time = signal.time-1
    signal['time'] = (signal.time * 2).astype(int)

    # save csv of data
    filepath = op.join(project['analysis_dir'], args.experiment, 'group/roi', 'evoked_' + mask + '.csv')
    signal.to_csv(filepath, index=False)

def parse_args(arglist):
    """Take an arglist and return an argparse Namespace."""
    help = dedent("""
    Run FIR model on subject data
    """)
    parser = tools.parser
    parser.description = help
    parser.formatter_class = argparse.RawDescriptionHelpFormatter
    parser.add_argument("-experiment", help="experimental paradigm (include -altmodel id relevant)")
    parser.add_argument("-ts_experiment", help="registered timeseries data")
    parser.add_argument("-mask", help="mask name")
    parser.add_argument("-len_et", help="The expected length of the event-triggered quantity (in the same time-units as the events are represented (presumably number of TRs, for fMRI data). For example, the size of the block dedicated in the fir_matrix to each type of event")
    return parser.parse_args(arglist)

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>Add code for FIR analysis<commit_after>#! /usr/bin/env python
"""
Script to run FIR analysis

EXAMPLE:
python run_fir.py -experiment ap_memory_raw -ts_experiment mvpa_raw -mask rh-hippocampus-tail -len_et 7
"""

from __future__ import division
import os.path as op
import itertools
import numpy as np
import scipy as sp
import pandas as pd
import nibabel as nib
from scipy import stats
import statsmodels.api as sm
import matplotlib as mpl
import matplotlib.pyplot as plt
import nitime as nit

import sys
import argparse
import shutil
from textwrap import dedent

import os
from glob import glob
import hashlib
from scipy.interpolate import interp1d
from lyman import gather_project_info

import lyman
from lyman import mvpa, evoked
from lyman import tools
import seaborn as sns
import moss

# Define some functions

def evoked_extract_subject(subj, mask_name, n_runs, summary_func=np.mean,
                           exp_name=None, ts_exp=None, residual=False):
    """Extract timeseries from within a mask, summarizing flexibly.
    Parameters
    ----------
    subj : string
        subject name
    mask_name : string
        name of mask in data hierarchy
    summary_func : callable or None
        callable to reduce data over voxel dimensions. can take an
        ``axis`` argument to operate over each frame, if this
        argument does not exist the function will be called on the
        n_tr x n_voxel array. if None, simply returns all voxels.
    residual : boolean
        If True, extract from the registered residual timecourse.
    exp_name : string
        experiment name, if not using the default experiment
    Returns
    -------
    data : dict with ndarray
        datta array is n_runs x n_timepoint x n_dimension,
        data are not otherwise altered
    """
    project = gather_project_info()
    if exp_name is None:
        exp_name = project["default_exp"]

    if ts_exp is None:
        ts_exp = exp_name

    # Get a path to the file where
    cache_dir = op.join(project["analysis_dir"],
                        exp_name, subj, "evoked")

    try:
        os.makedirs(cache_dir)
    except OSError:
        pass

    if summary_func is None:
        func_name = ""
    else:
        func_name = summary_func.__name__
    cache_fname = mask_name + "_" + func_name
    cache_fname = cache_fname.strip("_") + ".npz"
    cache_file = op.join(cache_dir, cache_fname)

    # Get paths to the relevant files
    mask_file = op.join(project["data_dir"], subj, "masks",
                        "%s.nii.gz" % mask_name)
    ts_dir = op.join(project["analysis_dir"], ts_exp, subj,
                     "reg", "epi", "unsmoothed")

    ftemp = op.join(ts_dir, "run_{:d}/{}_xfm.nii.gz")
    fstem = "res4d" if residual else "timeseries"
    ts_files = [ftemp.format(r_i, fstem) for r_i in range(1,n_runs+1)]
    print ts_files

    # Get the hash value for this extraction
    cache_hash = hashlib.sha1()
    cache_hash.update(mask_name)
    cache_hash.update(str(op.getmtime(mask_file)))
    for ts_file in ts_files:
        cache_hash.update(str(op.getmtime(ts_file)))
    cache_hash = cache_hash.hexdigest()

    # If the file exists and the hash matches, return the data
    if op.exists(cache_file):
        with np.load(cache_file) as cache_obj:
            if cache_hash == str(cache_obj["hash"]):
                return dict(cache_obj.items())

    # Otherwise, do the extraction
    data = []
    mask = nib.load(mask_file).get_data().astype(bool)
    for run, ts_file in enumerate(ts_files):
        ts_data = nib.load(ts_file).get_data()
        roi_data = ts_data[mask].T

        if summary_func is None:
            data.append(roi_data)
            continue

        # Try to use the axis argument to summarize over voxels
        try:
            roi_data = summary_func(roi_data, axis=1)

            print "Taking mean for run " + str(run +1)
        # Catch a TypeError and just call the function
        # This lets us do e.g. a PCA
        except TypeError:
            roi_data = summary_func(roi_data)

        data.append(roi_data)

    data = np.array(list(map(np.squeeze, data)))

    # Save the results and return them
    data_dict = dict(data=data, subj=subj, hash=cache_hash)
    np.savez(cache_file, **data_dict)

    return data_dict

def evoked_extract_group(mask_name, n_runs, summary_func=np.mean,
                         exp_name=None, ts_exp=None, subjects=None, dv=None):
    """Extract timeseries from within a mask, summarizing flexibly.
    Parameters
    ----------
    mask_name : string
        name of mask in data hierarchy
    summary_func : callable or None
        callable to reduce data over voxel dimensions. can take an
        ``axis`` argument to operate over each frame, if this
        argument does not exist the function will be called on the
        n_tr x n_voxel array. if None, simply returns all voxels.
    exp_name : string
        experiment name, if not using the default experiment
    subjects : sequence of strings
        subjects to operate over if not using default subject list
    dv : IPython cluster direct view
        if provided with view on cluster, executes in parallel over
        subjects
    Returns
    -------
    data : list of dicts with ndarrays
        each array is squeezed n_runs x n_timepoint x n_dimension
        data is not otherwise altered
    """
    if dv is None:
        import __builtin__
        _map = __builtin__.map
    else:
        _map = dv.map_sync

    if subjects is None:
        subj_file = op.join(os.environ["LYMAN_DIR"], "subjects.txt")
        subjects = np.loadtxt(subj_file, str).tolist()

    mask_name = [mask_name for s in subjects]
    summary_func = [summary_func for s in subjects]
    exp_name = [exp_name for s in subjects]
    ts_exp = [ts_exp for s in subjects]
    n_runs = [n_runs for s in subjects]

    data = _map(evoked_extract_subject, subjects, mask_name, n_runs,
                summary_func, exp_name, ts_exp)
    for d in data:
        d["data"] = np.asarray(d["data"])

    return data

def _evoked_1d(data, events, n_bins, tr, calc_method, correct_baseline):

    events_ts = nit.TimeSeries(events, sampling_interval=tr)
    data_ts = nit.TimeSeries(data, sampling_interval=tr)

    analyzer = nit.analysis.EventRelatedAnalyzer(data_ts, events_ts, n_bins)

    evoked_data = getattr(analyzer, calc_method)
    evoked_data = np.asarray(evoked_data).T.astype(float)

    if evoked_data.ndim == 1:
        evoked_data = np.array([evoked_data])
    if correct_baseline:
        evoked_data = evoked_data - evoked_data[:, 0, None]
    return evoked_data

def _evoked_2d(data, events, n_bins, tr, calc_method, correct_baseline):

    evoked_data = []
    for data_i in data.T:
        evoked_data_i = _evoked_1d(data_i, events, n_bins, tr,
                                   calc_method, correct_baseline)
        evoked_data.append(evoked_data_i)

    evoked_data = np.transpose(evoked_data, (1, 2, 0))
    return evoked_data

def integrate_evoked(evoked, axis=-1):
    """Integrate a peristumulus timecourse.
    Parameters
    ----------
    evoked : list of 2D arrays or 2D array
        values of evoked datapoints
    Returns
    -------
    int_evoked : squeezed array
        evoked values integrated over the time dimension
    """
    return sp.trapz(evoked, axis=axis)

def calculate_evoked_ts(data, n_bins, problem=None, events=None, tr=2,
                        calc_method="FIR", offset=0, upsample=1,
                        percent_change=True, correct_baseline=True,
                        event_names=None):
    """Calcuate an evoked response for a list of datapoints.
    Parameters
    ----------
    data : sequence of n_run x n_tp arrays
        timeseries data
    n_bins : int
        number of bins for the peristumulus trace
    problem : string
        problem name for event file in data hierarchy
        overrides `events` if both are passed
    events : dataframe or list of dataframes
        one dataframe describing event information for each subj.
        must contain `onset`, `run`, and `condition` columns
        caution: `run` should be 1-based
    tr : int
        original time resolution of the data (seconds)
    upsample : int
        factor to upsample the data with using cubic splines
    calc_method : string
        name of method on nitime EventRelatedAnalyzer object to
        calculate the evoked response.
    offset : float
        value to adjust onset times by
    percent_change : boolean
        if True, convert signal to percent change by run
    correct_baseline : boolean
        if True, adjust evoked trace to be 0 in first bin
    event_names : list of strings
        names of conditions, otherwise uses sorted unique
        values for the condition field in the event dataframe
    Returns
    -------
    evoked_ds :
    """

    project = lyman.gather_project_info()
    design_template = op.join(project["data_dir"], "%s",
                              "design/%s.csv" % problem)

    col_names = ['subj', 'event', 'time', 'response']
    evoked_ds = pd.DataFrame(columns=col_names)

    for i, data_i in enumerate(data):

        # Get event information
        subj = data_i["subj"]
        events_i = pd.read_csv(design_template % subj)

        # Map from event names to integer index values
        event_names = sorted(events_i.condition.unique())
        event_map = pd.Series(range(1, len(event_names) + 1),
                              index=event_names)

        # Create the timeseries of event occurances
        calc_tr = float(tr) / upsample

        event_list = []
        data_list = []
        for run, run_data in enumerate(data_i["data"], 1):

            # Possibly upsample the data
            if upsample != 1:
                time_points = len(run_data)
                x = np.linspace(0, time_points - 1, time_points)
                xx = np.linspace(0, time_points,
                                 time_points * upsample + 1)[:-upsample]
                interpolator = interp1d(x, run_data, "cubic", axis=0)
                run_data = interpolator(xx)

            run_events = events_i[events_i.run == run]
            run_events.onset += offset

            event_id = np.zeros(len(run_data), int)
            event_index = np.array(run_events.onset / calc_tr).astype(int)
            event_id[event_index] = run_events.condition.map(event_map)
            event_list.append(event_id)

            if percent_change:
                run_data = nit.utils.percent_change(run_data, ax=0)
            data_list.append(run_data)

        # Set up the Nitime objects
        event_info = np.concatenate(event_list)
        data = np.concatenate(data_list, axis=0)

        # Do the calculations
        calc_bins = n_bins * upsample

        print 'Data dimensions: ' + str(data.ndim)
        if data.ndim == 1:
            evoked_data = _evoked_1d(data, event_info, calc_bins, calc_tr,
                                     calc_method, correct_baseline)
        elif data.ndim == 2:
            evoked_data = _evoked_2d(data, event_info, n_bins, calc_tr,
                                     calc_method, correct_baseline)

        for event_num, d in enumerate(evoked_data):
            evoked_ds = evoked_ds.append(pd.DataFrame({'subj': subj,
                                                       'event': event_names[event_num],
                                                       'time': np.arange(1, n_bins+1),
                                                       'response': pd.Series(evoked_data[event_num])}))

    return evoked_ds

def main(arglist):

    args = parse_args(arglist)

    project = lyman.gather_project_info()
    exp = lyman.gather_experiment_info(args.experiment)
    anal_dir = project["analysis_dir"]
    data_dir = project["data_dir"]

    subjects = pd.Series(lyman.determine_subjects(), name="subj")

    all_rois = ['lh-hippocampus-tail', 'rh-hippocampus-tail',
                'lh-hippocampus-head', 'rh-hippocampus-head']

    def extract_group(subjects, args, exp):
        exp_name = args.experiment
        d = evoked_extract_group(args.mask, exp['n_runs'], subjects=subjects,
                                 exp_name=exp_name, ts_exp=args.ts_experiment)

        return d

    # Extract the timeseries from within a mask
    data = extract_group(subjects, args, exp)

    # Calculate evoked response (FIR model)
    signal = calculate_evoked_ts(data, int(args.len_et),
                                 problem=exp['design_name'],
                                 tr=int(exp['TR']))

    signal.time = signal.time-1
    signal['time'] = (signal.time * 2).astype(int)

    # save csv of data
    filepath = op.join(project['analysis_dir'], args.experiment, 'group/roi', 'evoked_' + args.mask + '.csv')
    signal.to_csv(filepath, index=False)

def parse_args(arglist):
    """Take an arglist and return an argparse Namespace."""
    help = dedent("""
    Run FIR model on subject data
    """)
    parser = tools.parser
    parser.description = help
    parser.formatter_class = argparse.RawDescriptionHelpFormatter
    parser.add_argument("-experiment", help="experimental paradigm (include -altmodel id relevant)")
    parser.add_argument("-ts_experiment", help="registered timeseries data")
    parser.add_argument("-mask", help="mask name")
    parser.add_argument("-len_et", help="The expected length of the event-triggered quantity (in the same time-units as the events are represented (presumably number of TRs, for fMRI data). For example, the size of the block dedicated in the fir_matrix to each type of event")
    return parser.parse_args(arglist)

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
Python version by Stuart Colville http://muffinresearch.co.uk
License: http://www.opensource.org/licenses/mit-license.php
"""

import argparse
import logging
logger = logging.getLogger(__name__)
import os
import string
import sys

try:
    import regex
except ImportError:
    import re as regex
    REGEX_AVAILABLE = False
else:
    REGEX_AVAILABLE = True

__all__ = ['titlecase']
__version__ = '2.2.0'

SMALL = r'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
PUNCT = r"""!"“#$%&'‘()*+,\-–‒—―./:;?@[\\\]_`{|}~"""

SMALL_WORDS = regex.compile(r'^(%s)$' % SMALL, regex.I)

SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), regex.I)
SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), regex.I)
SUBPHRASE = regex.compile(r'([:.;?!\-–‒—―][ ])(%s)' % SMALL)
MAC_MC = regex.compile(r"^([Mm]c|MC)(\w.+)")
MR_MRS_MS_DR = regex.compile(r"^((m((rs?)|s))|Dr)$", regex.I)

if REGEX_AVAILABLE:
    INLINE_PERIOD = regex.compile(r'[\p{Letter}][.][\p{Letter}]', regex.I)
    UC_ELSEWHERE = regex.compile(r'[%s]*?[\p{Letter}]+[\p{Uppercase_Letter}]+?' % PUNCT)
    CAPFIRST = regex.compile(r"^[%s]*?([\p{Letter}])" % PUNCT)
    APOS_SECOND = regex.compile(r"^[dol]{1}['‘]{1}[\p{Letter}]+(?:['s]{2})?$", regex.I)
    UC_INITIALS = regex.compile(r"^(?:[\p{Uppercase_Letter}]{1}\.{1}|[\p{Uppercase_Letter}]{1}\.{1}[\p{Uppercase_Letter}]{1})+$")
else:
    INLINE_PERIOD = regex.compile(r'[\w][.][\w]', regex.I)
    UC_ELSEWHERE = regex.compile(r'[%s]*?[a-zA-Z]+[A-Z]+?' % PUNCT)
    CAPFIRST = regex.compile(r"^[%s]*?([\w])" % PUNCT)
    APOS_SECOND = regex.compile(r"^[dol]['‘][\w]+(?:['s]{2})?$", regex.I)
    UC_INITIALS = regex.compile(r"^(?:[A-Z]\.|[A-Z]\.[A-Z])+$")

class Immutable(object):
    pass

class ImmutableString(str, Immutable):
    pass

class ImmutableBytes(bytes, Immutable):
    pass

def _mark_immutable(text):
    if isinstance(text, bytes):
        return ImmutableBytes(text)
    return ImmutableString(text)

def set_small_word_list(small=SMALL):
    global SMALL_WORDS
    global SMALL_FIRST
    global SMALL_LAST
    global SUBPHRASE
    SMALL_WORDS = regex.compile(r'^(%s)$' % small, regex.I)
    SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), regex.I)
    SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (small, PUNCT), regex.I)
    SUBPHRASE = regex.compile(r'([:.;?!][ ])(%s)' % small)

def titlecase(text, callback=None, small_first_last=True):
    """
    :param text: Titlecases input text
    :param callback: Callback function that returns the titlecase version of a specific word
    :param small_first_last: Capitalize small words (e.g. 'A') at the beginning; disabled when recursing
    :type text: str
    :type callback: function
    :type small_first_last: bool

    This filter changes all words to Title Caps, and attempts to be clever
    about *un*capitalizing SMALL words like a/an/the in the input.

    The list of "SMALL words" which are not capped comes from
    the New York Times Manual of Style, plus 'vs' and 'v'.

    """
    lines = regex.split('[\r\n]+', text)
    processed = []
    for line in lines:
        all_caps = line.upper() == line
        words = regex.split('[\t ]', line)
        tc_line = []
        for word in words:
            if callback:
                new_word = callback(word, all_caps=all_caps)
                if new_word:
                    # Address #22: If a callback has done something
                    # specific, leave this string alone from now on
                    tc_line.append(_mark_immutable(new_word))
                    continue

            if all_caps:
                if UC_INITIALS.match(word):
                    tc_line.append(word)
                    continue

            if APOS_SECOND.match(word):
                if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU':
                    word = word[0].lower() + word[1] + word[2].upper() + word[3:]
                else:
                    word = word[0].upper() + word[1] + word[2].upper() + word[3:]
                tc_line.append(word)
                continue

            match = MAC_MC.match(word)
            if match:
                tc_line.append("%s%s" % (match.group(1).capitalize(),
                                         titlecase(match.group(2), callback, True)))
                continue

            match = MR_MRS_MS_DR.match(word)
            if match:
                word = word[0].upper() + word[1:]
                tc_line.append(word)
                continue

            if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)):
                tc_line.append(word)
                continue
            if SMALL_WORDS.match(word):
                tc_line.append(word.lower())
                continue

            if "/" in word and "//" not in word:
                slashed = map(
                    lambda t: titlecase(t,callback,False),
                    word.split('/')
                )
                tc_line.append("/".join(slashed))
                continue

            if '-' in word:
                hyphenated = map(
                    lambda t: titlecase(t, callback, False),
                    word.split('-')
                )
                tc_line.append("-".join(hyphenated))
                continue

            if all_caps:
                word = word.lower()

            # A term with all consonants should be considered an acronym.  But if it's
            # too short (like "St", don't apply this)
            CONSONANTS = ''.join(set(string.ascii_lowercase)
                                 - {'a', 'e', 'i', 'o', 'u', 'y'})
            is_all_consonants = regex.search('\A[' + CONSONANTS + ']+\Z', word,
                                             flags=regex.IGNORECASE)
            if is_all_consonants and len(word) > 2:
                tc_line.append(word.upper())
                continue

            # Just a normal word that needs to be capitalized
            tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word))

        if small_first_last and tc_line:
            if not isinstance(tc_line[0], Immutable):
                tc_line[0] = SMALL_FIRST.sub(lambda m: '%s%s' % (
                    m.group(1),
                    m.group(2).capitalize()
                ), tc_line[0])

            if not isinstance(tc_line[-1], Immutable):
                tc_line[-1] = SMALL_LAST.sub(
                    lambda m: m.group(0).capitalize(), tc_line[-1]
                )

        result = " ".join(tc_line)

        result = SUBPHRASE.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        processed.append(result)

    result = "\n".join(processed)
    logger.debug(result)
    return result

def create_wordlist_filter_from_file(file_path):
    '''
    Load a list of abbreviations from the file with the provided path,
    reading one abbreviation from each line, and return a callback to
    be passed to the `titlecase` function for preserving their given
    canonical capitalization during title-casing.
    '''
    if file_path is None:
        logger.debug('No abbreviations file path given')
        return lambda word, **kwargs: None
    file_path_str = str(file_path)
    if not os.path.isfile(file_path_str):
        logger.debug('No abbreviations file found at ' + file_path_str)
        return lambda word, **kwargs: None
    with open(file_path_str) as f:
        logger.debug('Reading abbreviations from file ' + file_path_str)
        abbrevs_gen = (line.strip() for line in f.read().splitlines() if line)
        abbrevs = {abbr.upper(): abbr for abbr in abbrevs_gen}
        if logger.isEnabledFor(logging.DEBUG):
            for abbr in abbrevs.values():
                logger.debug('Registered abbreviation: ' + abbr)
        return lambda word, **kwargs: abbrevs.get(word.upper())

def cmd():
    '''Handler for command line invocation'''

    # Try to handle any reasonable thing thrown at this.
    # Consume '-f' and '-o' as input/output, allow '-' for stdin/stdout
    # and treat any subsequent arguments as a space separated string to
    # be titlecased (so it still works if people forget quotes)
    parser = argparse.ArgumentParser()
    in_group = parser.add_mutually_exclusive_group()
    in_group.add_argument('string', nargs='*', default=[],
            help='String to titlecase')
    in_group.add_argument('-f', '--input-file',
            help='File to read from to titlecase')
    parser.add_argument('-o', '--output-file',
            help='File to write titlecased output to')
    parser.add_argument('-w', '--wordlist',
            help='Wordlist for acronyms')

    args = parser.parse_args()

    if args.input_file is not None:
        if args.input_file == '-':
            ifile = sys.stdin
        else:
            ifile = open(args.input_file)
    else:
        ifile = sys.stdin

    if args.output_file is not None:
        if args.output_file == '-':
            ofile = sys.stdout
        else:
            ofile = open(args.output_file, 'w')
    else:
        ofile = sys.stdout

    if len(args.string) > 0:
        in_string = ' '.join(args.string)
    else:
        with ifile:
            in_string = ifile.read()

    if args.wordlist is not None:
        wordlist_file = args.wordlist
    else:
        wordlist_file = os.path.join(os.path.expanduser('~'), '.titlecase.txt')
    wordlist_filter = create_wordlist_filter_from_file(wordlist_file)

    with ofile:
        ofile.write(titlecase(in_string, callback=wordlist_filter))
<commit_msg>v2.3<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
Python version by Stuart Colville http://muffinresearch.co.uk
License: http://www.opensource.org/licenses/mit-license.php
"""

import argparse
import logging
logger = logging.getLogger(__name__)
import os
import string
import sys

try:
    import regex
except ImportError:
    import re as regex
    REGEX_AVAILABLE = False
else:
    REGEX_AVAILABLE = True

__all__ = ['titlecase']
__version__ = '2.3.0'

SMALL = r'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
PUNCT = r"""!"“#$%&'‘()*+,\-–‒—―./:;?@[\\\]_`{|}~"""

SMALL_WORDS = regex.compile(r'^(%s)$' % SMALL, regex.I)

SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), regex.I)
SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), regex.I)
SUBPHRASE = regex.compile(r'([:.;?!\-–‒—―][ ])(%s)' % SMALL)
MAC_MC = regex.compile(r"^([Mm]c|MC)(\w.+)")
MR_MRS_MS_DR = regex.compile(r"^((m((rs?)|s))|Dr)$", regex.I)

if REGEX_AVAILABLE:
    INLINE_PERIOD = regex.compile(r'[\p{Letter}][.][\p{Letter}]', regex.I)
    UC_ELSEWHERE = regex.compile(r'[%s]*?[\p{Letter}]+[\p{Uppercase_Letter}]+?' % PUNCT)
    CAPFIRST = regex.compile(r"^[%s]*?([\p{Letter}])" % PUNCT)
    APOS_SECOND = regex.compile(r"^[dol]{1}['‘]{1}[\p{Letter}]+(?:['s]{2})?$", regex.I)
    UC_INITIALS = regex.compile(r"^(?:[\p{Uppercase_Letter}]{1}\.{1}|[\p{Uppercase_Letter}]{1}\.{1}[\p{Uppercase_Letter}]{1})+$")
else:
    INLINE_PERIOD = regex.compile(r'[\w][.][\w]', regex.I)
    UC_ELSEWHERE = regex.compile(r'[%s]*?[a-zA-Z]+[A-Z]+?' % PUNCT)
    CAPFIRST = regex.compile(r"^[%s]*?([\w])" % PUNCT)
    APOS_SECOND = regex.compile(r"^[dol]['‘][\w]+(?:['s]{2})?$", regex.I)
    UC_INITIALS = regex.compile(r"^(?:[A-Z]\.|[A-Z]\.[A-Z])+$")

class Immutable(object):
    pass

class ImmutableString(str, Immutable):
    pass

class ImmutableBytes(bytes, Immutable):
    pass

def _mark_immutable(text):
    if isinstance(text, bytes):
        return ImmutableBytes(text)
    return ImmutableString(text)

def set_small_word_list(small=SMALL):
    global SMALL_WORDS
    global SMALL_FIRST
    global SMALL_LAST
    global SUBPHRASE
    SMALL_WORDS = regex.compile(r'^(%s)$' % small, regex.I)
    SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), regex.I)
    SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (small, PUNCT), regex.I)
    SUBPHRASE = regex.compile(r'([:.;?!][ ])(%s)' % small)

def titlecase(text, callback=None, small_first_last=True):
    """
    :param text: Titlecases input text
    :param callback: Callback function that returns the titlecase version of a specific word
    :param small_first_last: Capitalize small words (e.g. 'A') at the beginning; disabled when recursing
    :type text: str
    :type callback: function
    :type small_first_last: bool

    This filter changes all words to Title Caps, and attempts to be clever
    about *un*capitalizing SMALL words like a/an/the in the input.

    The list of "SMALL words" which are not capped comes from
    the New York Times Manual of Style, plus 'vs' and 'v'.

    """
    lines = regex.split('[\r\n]+', text)
    processed = []
    for line in lines:
        all_caps = line.upper() == line
        words = regex.split('[\t ]', line)
        tc_line = []
        for word in words:
            if callback:
                new_word = callback(word, all_caps=all_caps)
                if new_word:
                    # Address #22: If a callback has done something
                    # specific, leave this string alone from now on
                    tc_line.append(_mark_immutable(new_word))
                    continue

            if all_caps:
                if UC_INITIALS.match(word):
                    tc_line.append(word)
                    continue

            if APOS_SECOND.match(word):
                if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU':
                    word = word[0].lower() + word[1] + word[2].upper() + word[3:]
                else:
                    word = word[0].upper() + word[1] + word[2].upper() + word[3:]
                tc_line.append(word)
                continue

            match = MAC_MC.match(word)
            if match:
                tc_line.append("%s%s" % (match.group(1).capitalize(),
                                         titlecase(match.group(2), callback, True)))
                continue

            match = MR_MRS_MS_DR.match(word)
            if match:
                word = word[0].upper() + word[1:]
                tc_line.append(word)
                continue

            if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)):
                tc_line.append(word)
                continue
            if SMALL_WORDS.match(word):
                tc_line.append(word.lower())
                continue

            if "/" in word and "//" not in word:
                slashed = map(
                    lambda t: titlecase(t,callback,False),
                    word.split('/')
                )
                tc_line.append("/".join(slashed))
                continue

            if '-' in word:
                hyphenated = map(
                    lambda t: titlecase(t, callback, False),
                    word.split('-')
                )
                tc_line.append("-".join(hyphenated))
                continue

            if all_caps:
                word = word.lower()

            # A term with all consonants should be considered an acronym.  But if it's
            # too short (like "St", don't apply this)
            CONSONANTS = ''.join(set(string.ascii_lowercase)
                                 - {'a', 'e', 'i', 'o', 'u', 'y'})
            is_all_consonants = regex.search('\A[' + CONSONANTS + ']+\Z', word,
                                             flags=regex.IGNORECASE)
            if is_all_consonants and len(word) > 2:
                tc_line.append(word.upper())
                continue

            # Just a normal word that needs to be capitalized
            tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word))

        if small_first_last and tc_line:
            if not isinstance(tc_line[0], Immutable):
                tc_line[0] = SMALL_FIRST.sub(lambda m: '%s%s' % (
                    m.group(1),
                    m.group(2).capitalize()
                ), tc_line[0])

            if not isinstance(tc_line[-1], Immutable):
                tc_line[-1] = SMALL_LAST.sub(
                    lambda m: m.group(0).capitalize(), tc_line[-1]
                )

        result = " ".join(tc_line)

        result = SUBPHRASE.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        processed.append(result)

    result = "\n".join(processed)
    logger.debug(result)
    return result

def create_wordlist_filter_from_file(file_path):
    '''
    Load a list of abbreviations from the file with the provided path,
    reading one abbreviation from each line, and return a callback to
    be passed to the `titlecase` function for preserving their given
    canonical capitalization during title-casing.
    '''
    if file_path is None:
        logger.debug('No abbreviations file path given')
        return lambda word, **kwargs: None
    file_path_str = str(file_path)
    if not os.path.isfile(file_path_str):
        logger.debug('No abbreviations file found at ' + file_path_str)
        return lambda word, **kwargs: None
    with open(file_path_str) as f:
        logger.debug('Reading abbreviations from file ' + file_path_str)
        abbrevs_gen = (line.strip() for line in f.read().splitlines() if line)
        abbrevs = {abbr.upper(): abbr for abbr in abbrevs_gen}
        if logger.isEnabledFor(logging.DEBUG):
            for abbr in abbrevs.values():
                logger.debug('Registered abbreviation: ' + abbr)
        return lambda word, **kwargs: abbrevs.get(word.upper())

def cmd():
    '''Handler for command line invocation'''

    # Try to handle any reasonable thing thrown at this.
    # Consume '-f' and '-o' as input/output, allow '-' for stdin/stdout
    # and treat any subsequent arguments as a space separated string to
    # be titlecased (so it still works if people forget quotes)
    parser = argparse.ArgumentParser()
    in_group = parser.add_mutually_exclusive_group()
    in_group.add_argument('string', nargs='*', default=[],
            help='String to titlecase')
    in_group.add_argument('-f', '--input-file',
            help='File to read from to titlecase')
    parser.add_argument('-o', '--output-file',
            help='File to write titlecased output to')
    parser.add_argument('-w', '--wordlist',
            help='Wordlist for acronyms')

    args = parser.parse_args()

    if args.input_file is not None:
        if args.input_file == '-':
            ifile = sys.stdin
        else:
            ifile = open(args.input_file)
    else:
        ifile = sys.stdin

    if args.output_file is not None:
        if args.output_file == '-':
            ofile = sys.stdout
        else:
            ofile = open(args.output_file, 'w')
    else:
        ofile = sys.stdout

    if len(args.string) > 0:
        in_string = ' '.join(args.string)
    else:
        with ifile:
            in_string = ifile.read()

    if args.wordlist is not None:
        wordlist_file = args.wordlist
    else:
        wordlist_file = os.path.join(os.path.expanduser('~'), '.titlecase.txt')
    wordlist_filter = create_wordlist_filter_from_file(wordlist_file)

    with ofile:
        ofile.write(titlecase(in_string, callback=wordlist_filter))
<|endoftext|>


<commit_before>from BaseHTTPServer import BaseHTTPRequestHandler
import urlparse
import traceback
import os
import mysql.connector

class Ec2MetaHandler(BaseHTTPRequestHandler):

    def __init__(self, req, ca, huh):
        self.cnx = mysql.connector.connect(user='tmcd',
                              database='tbdb', unix_socket='/tmp/mysql.sock')
        BaseHTTPRequestHandler.__init__(self,req,ca,huh)

    def do_GET(self):
        parsed_path = urlparse.urlparse(self.path)

        only_path = parsed_path.path
        folders=[]
        while 1:
            only_path,folder=os.path.split(only_path)

            if folder != "":
                folders.append(folder)
            if only_path=="/":
                break;

        if len(folders) > 0:
            folders.pop() #Ignore version
        folders.reverse()

        try:
            message = self.handle_req(folders, self.metas)
            message = message + "\n"
        except Exception as e:
            print traceback.format_exc()
            self.send_response(404)
            self.end_headers()
            return

        self.send_response(200)
        self.end_headers()
        self.wfile.write(message)
        return

    def listmetas(self, metas):
        message = "\n".join(map(lambda x: x + "/" if (x == "public-keys" or not(callable(metas[x])))  else x,
                        metas.keys()));
        return message

    def handle_req(self, arg, metas):
        if callable(metas):
            return metas(self, arg)
        elif len(arg) == 0:
            return self.listmetas(metas);
        else:
            return self.handle_req(arg[1:], metas[arg[0]])

    def do_userdata(self):
        #TODO
        return "\n";

    def doamiid(self, arg):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select osname from os_info "
            "join nodes on os_info.osid = nodes.osid "
            "join interfaces on nodes.node_id=interfaces.node_id "
            "where interfaces.ip=%s", (ip,));
        if cursor.with_rows:
            ami_id = cursor.fetchone()
            ami_id = ami_id[0]
        else:
            ami_id = ""
        cursor.close()
        return ami_id;

    def dolocal_hostname(self, args):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select vname,eid,pid from reserved join interfaces on interfaces.node_id=reserved.node_id"
            " where interfaces.ip=%s",(ip,))
        if cursor.with_rows:
            node_id = cursor.fetchone()
        else:
            cursor.close()
            return ""

        cursor.close()
        return node_id[0] + "." + node_id[1] + "." + node_id[2] + "." + "emulab.net"

    def doavail(self, args):
        #TODO
        return "emulab"

    def domacs(self, args):
        #TODO
        return "324AF"

    def domac(self, args):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select mac from interfaces"
            " where interfaces.ip=%s",(ip,))
        if cursor.with_rows:
            mac = cursor.fetchone()
        else:
            cursor.close()
            return ""

        cursor.close()
        split = [mac[0][i:i+2] for i in range(0, len(mac[0]),2)]
        return ":".join(split)

    def doinstance_id(self, args):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select uuid from interfaces"
            " where interfaces.ip=%s",(ip,))
        if cursor.with_rows:
            uuid = cursor.fetchone()
        else:
            cursor.close()
            return ""

        cursor.close()
        return uuid[0]

    def dopublic_keys(self, args):
        if len(args) == 0:
            #Throw out all the users. Hope the stuff don't change between queries
            cursor = self.cnx.cursor()
            ip = self.client_address[0]
            cursor.execute("select user_pubkeys.uid,user_pubkeys.idx from user_pubkeys "
                "join group_membership on group_membership.uid = user_pubkeys.uid "
                "join experiments on experiments.pid=group_membership.pid AND experiments.gid=group_membership.gid "
                "join reserved on reserved.exptidx=experiments.idx "
                "join interfaces on reserved.node_id=interfaces.node_id "
                "where interfaces.ip=%s;", (ip,));

            list = ""
            ctr = 0
            if cursor.with_rows:
                for (user,uid) in cursor:
                    list = list + str(ctr) + "=" + str(user) + str(uid) + "\n"
                    ctr = ctr+1
            else:
                cursor.close()
                return ""

            cursor.close()
            return list
        elif len(args) == 1:
            #TODO: Verify ig idx is within limits
            return "openssh-key"
        elif len(args) == 2:
            val = int(args[0])
            cursor = self.cnx.cursor()
            ip = self.client_address[0]
            cursor.execute("select user_pubkeys.pubkey from user_pubkeys "
                "join group_membership on group_membership.uid = user_pubkeys.uid "
                "join experiments on experiments.pid=group_membership.pid AND experiments.gid=group_membership.gid "
                "join reserved on reserved.exptidx=experiments.idx "
                "join interfaces on reserved.node_id=interfaces.node_id "
                "where interfaces.ip=%s limit " + str(val) +", 1;", (ip,));

            if cursor.with_rows:
                key = cursor.fetchone()
            else:
                cursor.close()
                return ""

            cursor.close()
            return key[0]

    metas = {
        "meta-data" : {
            "placement" : {"availability-zone" : doavail},
            "ami-id": doamiid,
            "local-hostname" : dolocal_hostname,
            "public-hostname":dolocal_hostname,
            "network": {"interfaces": {"macs" : domacs}},
            "mac":domac,
            "instance-id":doinstance_id,
            "public-keys": dopublic_keys },
        "user-data" : do_userdata
    }

if __name__ == '__main__':
    from BaseHTTPServer import HTTPServer
    import socket
    server = HTTPServer((socket.gethostbyname(socket.gethostname()), 8787), Ec2MetaHandler)
    pid = str(os.getpid())
    pidfile = "/var/run/tmcd-meta.pid"

    if os.path.isfile(pidfile):
        print "%s already exists, exiting" % pidfile
        sys.exit()
    else:
        file(pidfile, 'w').write(pid)

    print 'Starting server, use <Ctrl-C> to stop'
    server.serve_forever()

    os.unlink(pidfile)

<commit_msg>Make the users pubkey be first<commit_after>from BaseHTTPServer import BaseHTTPRequestHandler
import urlparse
import traceback
import os
import mysql.connector

class Ec2MetaHandler(BaseHTTPRequestHandler):

    def __init__(self, req, ca, huh):
        self.cnx = mysql.connector.connect(user='tmcd',
                              database='tbdb', unix_socket='/tmp/mysql.sock')
        BaseHTTPRequestHandler.__init__(self,req,ca,huh)

    def do_GET(self):
        parsed_path = urlparse.urlparse(self.path)

        only_path = parsed_path.path
        folders=[]
        while 1:
            only_path,folder=os.path.split(only_path)

            if folder != "":
                folders.append(folder)
            if only_path=="/":
                break;

        if len(folders) > 0:
            folders.pop() #Ignore version
        folders.reverse()

        try:
            message = self.handle_req(folders, self.metas)
            message = message + "\n"
        except Exception as e:
            print traceback.format_exc()
            self.send_response(404)
            self.end_headers()
            return

        self.send_response(200)
        self.end_headers()
        self.wfile.write(message)
        return

    def listmetas(self, metas):
        message = "\n".join(map(lambda x: x + "/" if (x == "public-keys" or not(callable(metas[x])))  else x,
                        metas.keys()));
        return message

    def handle_req(self, arg, metas):
        if callable(metas):
            return metas(self, arg)
        elif len(arg) == 0:
            return self.listmetas(metas);
        else:
            return self.handle_req(arg[1:], metas[arg[0]])

    def do_userdata(self):
        #TODO
        return "\n";

    def doamiid(self, arg):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select osname from os_info "
            "join nodes on os_info.osid = nodes.osid "
            "join interfaces on nodes.node_id=interfaces.node_id "
            "where interfaces.ip=%s", (ip,));
        if cursor.with_rows:
            ami_id = cursor.fetchone()
            ami_id = ami_id[0]
        else:
            ami_id = ""
        cursor.close()
        return ami_id;

    def dolocal_hostname(self, args):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select vname,eid,pid from reserved join interfaces on interfaces.node_id=reserved.node_id"
            " where interfaces.ip=%s",(ip,))
        if cursor.with_rows:
            node_id = cursor.fetchone()
        else:
            cursor.close()
            return ""

        cursor.close()
        return node_id[0] + "." + node_id[1] + "." + node_id[2] + "." + "emulab.net"

    def doavail(self, args):
        #TODO
        return "emulab"

    def domacs(self, args):
        #TODO
        return "324AF"

    def domac(self, args):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select mac from interfaces"
            " where interfaces.ip=%s",(ip,))
        if cursor.with_rows:
            mac = cursor.fetchone()
        else:
            cursor.close()
            return ""

        cursor.close()
        split = [mac[0][i:i+2] for i in range(0, len(mac[0]),2)]
        return ":".join(split)

    def doinstance_id(self, args):
        cursor = self.cnx.cursor()
        ip = self.client_address[0]
        cursor.execute("select uuid from interfaces"
            " where interfaces.ip=%s",(ip,))
        if cursor.with_rows:
            uuid = cursor.fetchone()
        else:
            cursor.close()
            return ""

        cursor.close()
        return uuid[0]

    def dopublic_keys(self, args):
        if len(args) == 0:
            #Throw out all the users. Hope the stuff don't change between queries
            cursor = self.cnx.cursor()
            ip = self.client_address[0]
            cursor.execute("select user_pubkeys.uid,user_pubkeys.idx from user_pubkeys "
                "join group_membership on group_membership.uid = user_pubkeys.uid "
                "join experiments on experiments.pid=group_membership.pid AND experiments.gid=group_membership.gid "
                "join reserved on reserved.exptidx=experiments.idx "
                "join interfaces on reserved.node_id=interfaces.node_id "
                "where interfaces.ip=%s and user_pubkeys.uid=experiments.expu_swap_uid "
                "UNION "
                "select user_pubkeys.uid,user_pubkeys.idx from user_pubkeys "
                "join group_membership on group_membership.uid = user_pubkeys.uid "
                "join experiments on experiments.pid=group_membership.pid AND experiments.gid=group_membership.gid "
                "join reserved on reserved.exptidx=experiments.idx "
                "join interfaces on reserved.node_id=interfaces.node_id "
                "where interfaces.ip=%s and user_pubkeys.uid!=experiments.expt_swap_uid;"
                , (ip,ip,))

            list = ""
            ctr = 0
            if cursor.with_rows:
                for (user,uid) in cursor:
                    list = list + str(ctr) + "=" + str(user) + str(uid) + "\n"
                    ctr = ctr+1
            else:
                cursor.close()
                return ""

            cursor.close()
            return list
        elif len(args) == 1:
            #TODO: Verify ig idx is within limits
            return "openssh-key"
        elif len(args) == 2:
            val = int(args[0])
            cursor = self.cnx.cursor()
            ip = self.client_address[0]
            cursor.execute("((select user_pubkeys.pubkey from user_pubkeys "
                "join group_membership on group_membership.uid = user_pubkeys.uid "
                "join experiments on experiments.pid=group_membership.pid AND experiments.gid=group_membership.gid "
                "join reserved on reserved.exptidx=experiments.idx "
                "join interfaces on reserved.node_id=interfaces.node_id "
                "where interfaces.ip=%s and user_pubkeys.uid=experiments.expt_swap_uid) "
                "UNION "
                "(select user_pubkeys.pubkey from user_pubkeys "
                "join group_membership on group_membership.uid = user_pubkeys.uid "
                "join experiments on experiments.pid=group_membership.pid AND experiments.gid=group_membership.gid "
                "join reserved on reserved.exptidx=experiments.idx "
                "join interfaces on reserved.node_id=interfaces.node_id "
                "where interfaces.ip=%s and user_pubkeys.uid!=experiments.expt_swap_uid)) limit " + str(val) + ", 1;",
                (ip, ip,))

            if cursor.with_rows:
                key = cursor.fetchone()
            else:
                cursor.close()
                return ""

            cursor.close()
            return key[0]

    metas = {
        "meta-data" : {
            "placement" : {"availability-zone" : doavail},
            "ami-id": doamiid,
            "local-hostname" : dolocal_hostname,
            "public-hostname":dolocal_hostname,
            "network": {"interfaces": {"macs" : domacs}},
            "mac":domac,
            "instance-id":doinstance_id,
            "public-keys": dopublic_keys },
        "user-data" : do_userdata
    }

if __name__ == '__main__':
    from BaseHTTPServer import HTTPServer
    import socket
    server = HTTPServer((socket.gethostbyname(socket.gethostname()), 8787), Ec2MetaHandler)
    pid = str(os.getpid())
    pidfile = "/var/run/tmcd-meta.pid"

    if os.path.isfile(pidfile):
        print "%s already exists, exiting" % pidfile
        sys.exit()
    else:
        file(pidfile, 'w').write(pid)

    print 'Starting server, use <Ctrl-C> to stop'
    server.serve_forever()

    os.unlink(pidfile)

<|endoftext|>


<commit_before>"""
    distutilazy
    -----------

    Extra distutils command classes.

    :license: MIT, see LICENSE for more details.
"""

import os
import sys

__version__ = '0.2.0'
__all__ = ['clean', 'pyinstaller', 'command']

base_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
if not base_dir in sys.path:
    if len(sys.path):
        sys.path.insert(1, base_dir)
    else:
        sys.path.append(base_dir)
<commit_msg>Updated version from 0.2.0 to 0.2.1<commit_after>"""
    distutilazy
    -----------

    Extra distutils command classes.

    :license: MIT, see LICENSE for more details.
"""

import os
import sys

__version__ = '0.2.1'
__all__ = ['clean', 'pyinstaller', 'command']

base_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
if not base_dir in sys.path:
    if len(sys.path):
        sys.path.insert(1, base_dir)
    else:
        sys.path.append(base_dir)
<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

@testing.parameterize(*testing.product({
    'axis': [None, 0, 1, 2, -1, (0, 1), (1, 0), (0, -1), (-2, 0)],
    'keepdims': [True, False],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'contain_zero': [True, False],
}))
class TestProd(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(self.dtype)
        if self.contain_zero:
            index = numpy.random.choice(self.x.size)
            self.x.ravel()[index] = 0
        g_shape = self.x.prod(axis=self.axis, keepdims=self.keepdims).shape
        self.gy = numpy.random.uniform(-1, 1, g_shape).astype(self.dtype)
        self.ggx = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(self.dtype)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.prod(x, axis=self.axis, keepdims=self.keepdims)
        self.assertEqual(y.data.dtype, self.dtype)
        y_expect = self.x.prod(axis=self.axis, keepdims=self.keepdims)

        if self.dtype == numpy.float16:
            options = {'atol': 1e-3, 'rtol': 1e-3}
        else:
            options = {}

        testing.assert_allclose(y_expect, y.data, **options)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.prod(x, self.axis, self.keepdims)

        gradient_check.check_backward(
            f, x_data, y_grad, atol=1e-3, dtype=numpy.float64)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    def check_double_backward(self, x_data, y_grad, x_grad_grad):
        def f(x):
            x = functions.prod(x, self.axis, self.keepdims)
            return x * x

        gradient_check.check_double_backward(
            f, x_data, y_grad, x_grad_grad, atol=1e-3, dtype=numpy.float64)

    @condition.retry(3)
    def test_double_backward_cpu(self):
        self.check_double_backward(self.x, self.gy, self.ggx)

    @attr.gpu
    @condition.retry(3)
    def test_double_backward_gpu(self):
        self.check_double_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), cuda.to_gpu(self.ggx))

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestProdError(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(self.dtype)

    def test_invalid_axis_type(self):
        with self.assertRaises(TypeError):
            functions.Prod([0])

    def test_invalid_axis_type_in_tuple(self):
        with self.assertRaises(TypeError):
            functions.Prod((1, 'x'))

    def test_duplicate_axis(self):
        with self.assertRaises(ValueError):
            functions.Prod((0, 0))

    def test_pos_neg_duplicate_axis(self):
        with self.assertRaises(ValueError):
            self.x.prod(axis=(1, -2))

testing.run_module(__name__, __file__)
<commit_msg>remove condition.retry decorator<commit_after>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product({
    'axis': [None, 0, 1, 2, -1, (0, 1), (1, 0), (0, -1), (-2, 0)],
    'keepdims': [True, False],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'contain_zero': [True, False],
}))
class TestProd(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(self.dtype)
        if self.contain_zero:
            index = numpy.random.choice(self.x.size)
            self.x.ravel()[index] = 0
        g_shape = self.x.prod(axis=self.axis, keepdims=self.keepdims).shape
        self.gy = numpy.random.uniform(-1, 1, g_shape).astype(self.dtype)
        self.ggx = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(self.dtype)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.prod(x, axis=self.axis, keepdims=self.keepdims)
        self.assertEqual(y.data.dtype, self.dtype)
        y_expect = self.x.prod(axis=self.axis, keepdims=self.keepdims)

        if self.dtype == numpy.float16:
            options = {'atol': 1e-3, 'rtol': 1e-3}
        else:
            options = {}

        testing.assert_allclose(y_expect, y.data, **options)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.prod(x, self.axis, self.keepdims)

        gradient_check.check_backward(
            f, x_data, y_grad, atol=1e-3, dtype=numpy.float64)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    def check_double_backward(self, x_data, y_grad, x_grad_grad):
        def f(x):
            x = functions.prod(x, self.axis, self.keepdims)
            return x * x

        gradient_check.check_double_backward(
            f, x_data, y_grad, x_grad_grad, atol=1e-3, dtype=numpy.float64)

    def test_double_backward_cpu(self):
        self.check_double_backward(self.x, self.gy, self.ggx)

    @attr.gpu
    def test_double_backward_gpu(self):
        self.check_double_backward(
            cuda.to_gpu(self.x), cuda.to_gpu(self.gy), cuda.to_gpu(self.ggx))

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestProdError(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(self.dtype)

    def test_invalid_axis_type(self):
        with self.assertRaises(TypeError):
            functions.Prod([0])

    def test_invalid_axis_type_in_tuple(self):
        with self.assertRaises(TypeError):
            functions.Prod((1, 'x'))

    def test_duplicate_axis(self):
        with self.assertRaises(ValueError):
            functions.Prod((0, 0))

    def test_pos_neg_duplicate_axis(self):
        with self.assertRaises(ValueError):
            self.x.prod(axis=(1, -2))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
import chainer.functions as F
from chainer import testing

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'shape': [(), (3, 2)]
}))
@testing.inject_backend_tests(
    None,
    # CPU tests
    [
        {},
    ]
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'cuda_device': [0, 1],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0', 'cuda:1']
    })
)
@testing.with_requires('scipy')
class TestZeta(testing.FunctionTestCase):

    def setUp(self):
        self.check_forward_options = {'atol': 1e-3, 'rtol': 1e-3}
        self.check_backward_options = {'atol': 1e-3, 'rtol': 1e-3}
        self.check_double_backward_options = {'atol': 1e-3, 'rtol': 1e-3}

        self._x = numpy.random.uniform(low=2, high=50, size=self.shape).\
            astype(self.dtype)

    def generate_inputs(self):
        q = numpy.random.uniform(low=1, high=50, size=self.shape).\
            astype(self.dtype)
        return q,

    def forward_expected(self, inputs):
        q, = inputs
        import scipy
        y_expect = scipy.special.zeta(self._x, q)
        return numpy.array(y_expect, dtype=self.dtype),

    def forward(self, inputs, device):
        q, = inputs
        y = F.zeta(device.send(self._x.astype(q.dtype)), q)
        return y,

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'shape': [(), (3, 2)]
}))
@testing.without_requires('scipy')
class TestZetaExceptions(unittest.TestCase):
    def setUp(self):
        self._x = numpy.random.uniform(low=-5, high=1, size=self.shape).\
            astype(self.dtype)
        self.q = numpy.random.uniform(low=-5, high=1, size=self.shape).\
            astype(self.dtype)
        self.func = F.zeta

    def check_forward(self, q_data):
        q = chainer.Variable(q_data)
        with self.assertRaises(ImportError):
            self.func(q, self._x)

    def test_zeta_forward_cpu(self):
        self.check_forward(self.q)

testing.run_module(__name__, __file__)
<commit_msg>Fix flaky test: TestZeta<commit_after>import unittest

import numpy

import chainer
import chainer.functions as F
from chainer import testing

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'shape': [(), (3, 2)],
    'x_range': [(1.1, 2), (2, 50)],
    'q_range': [(1.1, 2), (2, 50)],
}))
@testing.inject_backend_tests(
    None,
    # CPU tests
    [
        {},
    ]
    # GPU tests
    + testing.product({
        'use_cuda': [True],
        'cuda_device': [0, 1],
    })
    # ChainerX tests
    + testing.product({
        'use_chainerx': [True],
        'chainerx_device': ['native:0', 'cuda:0', 'cuda:1']
    })
)
@testing.with_requires('scipy')
class TestZeta(testing.FunctionTestCase):

    def setUp(self):
        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 1e-2, 'rtol': 1e-2}
            self.check_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
            self.check_double_backward_options = {'atol': 1e-2, 'rtol': 1e-2}
        else:
            self.check_forward_options = {'atol': 1e-3, 'rtol': 1e-3}
            self.check_backward_options = {'atol': 1e-3, 'rtol': 1e-3}
            self.check_double_backward_options = {'atol': 1e-3, 'rtol': 1e-3}

        low, high = self.x_range
        self._x = numpy.random.uniform(
            low=low, high=high, size=self.shape).astype(self.dtype)

    def generate_inputs(self):
        low, high = self.q_range
        q = numpy.random.uniform(
            low=low, high=high, size=self.shape).astype(self.dtype)
        return q,

    def forward_expected(self, inputs):
        q, = inputs
        import scipy
        y_expect = scipy.special.zeta(self._x, q)
        return numpy.array(y_expect, dtype=self.dtype),

    def forward(self, inputs, device):
        q, = inputs
        y = F.zeta(device.send(self._x.astype(q.dtype)), q)
        return y,

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'shape': [(), (3, 2)]
}))
@testing.without_requires('scipy')
class TestZetaExceptions(unittest.TestCase):
    def setUp(self):
        self._x = numpy.random.uniform(low=-5, high=1, size=self.shape).\
            astype(self.dtype)
        self.q = numpy.random.uniform(low=-5, high=1, size=self.shape).\
            astype(self.dtype)
        self.func = F.zeta

    def check_forward(self, q_data):
        q = chainer.Variable(q_data)
        with self.assertRaises(ImportError):
            self.func(q, self._x)

    def test_zeta_forward_cpu(self):
        self.check_forward(self.q)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>__version__ = '0.4.1'
<commit_msg>Increase minor version number<commit_after>__version__ = '0.5.0'
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from setuptools import find_packages, setup

with open("README.md") as f:
  long_description = f.read()

version = None
with open(os.path.join(os.getcwd(), "jupyterlab_gitsync",
                       "version.py")) as f:
  for l in f:
    if l.startswith("VERSION"):
      version = l.rstrip().split(" = ")[1].replace("'", "")

if not version:
  raise RuntimeError("Unable to determine version")

npm_package = "jupyterlab_gitsync-{}.tgz".format(version)
if not os.path.exists(os.path.join(os.getcwd(), npm_package)):
  raise FileNotFoundError("Cannot find NPM package. Did you run `npm pack`?")

data_files = [
    ("share/jupyter/lab/extensions", (npm_package,)),
    ("etc/jupyter/jupyter_notebook_config.d",
     ("jupyter-config/jupyter_notebook_config.d/jupyterlab_gitsync.json",
     )),
]

setup(
    name="jupyterlab_gitsync",
    version=version,
    description="JupyterLab Git Sync",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/GoogleCloudPlatform/jupyter-extensions",
    data_files=data_files,
    license="Apache License 2.0",
    packages=find_packages(),
    python_requires=">=3.6",
    install_requires=[
        "google-cloud-storage>=1.24.1",
        "jupyterlab~=1.2.0",
        "gcp_jupyterlab_shared>=1.0.0",
    ],
)
<commit_msg>remove unnecessary dependencies<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from setuptools import find_packages, setup

with open("README.md") as f:
  long_description = f.read()

version = None
with open(os.path.join(os.getcwd(), "jupyterlab_gitsync",
                       "version.py")) as f:
  for l in f:
    if l.startswith("VERSION"):
      version = l.rstrip().split(" = ")[1].replace("'", "")

if not version:
  raise RuntimeError("Unable to determine version")

npm_package = "jupyterlab_gitsync-{}.tgz".format(version)
if not os.path.exists(os.path.join(os.getcwd(), npm_package)):
  raise FileNotFoundError("Cannot find NPM package. Did you run `npm pack`?")

data_files = [
    ("share/jupyter/lab/extensions", (npm_package,)),
    ("etc/jupyter/jupyter_notebook_config.d",
     ("jupyter-config/jupyter_notebook_config.d/jupyterlab_gitsync.json",
     )),
]

setup(
    name="jupyterlab_gitsync",
    version=version,
    description="JupyterLab Git Sync",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/GoogleCloudPlatform/jupyter-extensions",
    data_files=data_files,
    license="Apache License 2.0",
    packages=find_packages(),
    python_requires=">=3.6",
    install_requires=[
        "jupyterlab~=1.2.0",
        "gcp_jupyterlab_shared>=1.0.0",
    ],
)
<|endoftext|>