<commit_before>#!/usr/bin/env python3

r"""
Make waterfall plots of a file set, view from top to bottom.
"""

import os
import sys
from os.path import dirname, abspath, isdir
import gc
from argparse import ArgumentParser
import logging
logger_name = "stax"
logger = logging.getLogger(logger_name)
logger.setLevel(logging.INFO)

# Plotting packages import
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use("agg")

# Math/Science package imports
import numpy as np

# Blimpy imports
import blimpy as bl
from blimpy.utils import rebin

# Plotting constants
fontsize=16
font = {"family" : "DejaVu Sans",
"size" : fontsize}
MAX_IMSHOW_POINTS = (4096, 1268)

def plot_waterfall(wf, f_start=None, f_stop=None, **kwargs):
    r"""
    Plot waterfall of data in a .fil or .h5 file.

    Parameters
    ----------
    wf : blimpy.Waterfall object
        Waterfall object of an H5 or Filterbank file containing the dynamic spectrum data.
    f_start : float
        Start frequency, in MHz.
    f_stop : float
        Stop frequency, in MHz.
    kwargs : dict
        Keyword args to be passed to matplotlib imshow().

    Notes
    -----
    Plot a single-panel waterfall plot (frequency vs. time vs. intensity)
    for one of the files in the set of interest, at the
    frequency of the expected event.
    """

    # Extract source name.
    source_name = wf.header["source_name"]

    # prepare font
    matplotlib.rc("font", **font)

    # Load in the data from fil
    plot_f, plot_data = wf.grab_data(f_start=f_start, f_stop=f_stop)

    # Make sure waterfall plot is under 4k*4k
    dec_fac_x, dec_fac_y = 1, 1

    # rebinning data to plot correctly with fewer points
    try:
        if plot_data.shape[0] > MAX_IMSHOW_POINTS[0]:
            dec_fac_x = plot_data.shape[0] / MAX_IMSHOW_POINTS[0]
        if plot_data.shape[1] > MAX_IMSHOW_POINTS[1]:
            dec_fac_y =  int(np.ceil(plot_data.shape[1] /  MAX_IMSHOW_POINTS[1]))
        plot_data = rebin(plot_data, dec_fac_x, dec_fac_y)
    except Exception as ex:
        print("\n*** Oops, grab_data returned plot_data.shape={}, plot_f.shape={}"
              .format(plot_data.shape, plot_f.shape))
        print("Waterfall info for {}:".format(wf.filename))
        wf.info()
        raise ValueError("*** Something is wrong with the grab_data output!") from ex

    # Rolled back PR #82

    # determine extent of the plotting panel for imshow
    extent=(plot_f[0], plot_f[-1], (wf.timestamps[-1]-wf.timestamps[0])*24.*60.*60, 0.0)

    # plot and scale intensity (log vs. linear)
    kwargs["cmap"] = kwargs.get("cmap", "viridis")
    plot_data = 10.0 * np.log10(plot_data)

    # get normalization parameters
    vmin = plot_data.min()
    vmax = plot_data.max()
    normalized_plot_data = (plot_data - vmin) / (vmax - vmin)

    # display the waterfall plot
    this_plot = plt.imshow(normalized_plot_data,
        aspect="auto",
        rasterized=True,
        interpolation="nearest",
        extent=extent,
        **kwargs
    )

    # add plot labels
    plt.xlabel("Frequency [Hz]",fontdict=font)
    plt.ylabel("Time [s]",fontdict=font)

    # add source name
    ax = plt.gca()
    plt.text(0.03, 0.8, source_name, transform=ax.transAxes, bbox=dict(facecolor="white"))

    return this_plot

def sort2(x, y):
    r""" Return lowest value, highest value"""
    if y < x:
        return y, x
    return x, y

def make_waterfall_plots(file_list, plot_dir, f_start=None, f_stop=None, **kwargs):
    r"""
    Make waterfall plots of a file set, view from top to bottom.

    Parameters
    ----------
    file_list : str
        List of filterbank files to plot in a stacked mode.
    f_start : float
        Start frequency, in MHz.
    f_stop : float
        Stop frequency, in MHz.
    source_name_list : list
        List of source names in the set, in order.
    kwargs : dict
        Keyword args to be passed to matplotlib imshow().
    """

    # prepare for plotting
    matplotlib.rc("font", **font)

    # set up the sub-plots
    n_plots = len(file_list)
    fig = plt.subplots(n_plots, sharex=True, sharey=True,figsize=(10, 2*n_plots))

    # get directory path for storing PNG files
    if plot_dir is None:
        dirpath = dirname(abspath(file_list[0])) + "/"
    else:
        if not isdir(plot_dir):
            os.mkdir(plot_dir)
        dirpath = plot_dir

    # read in data for the first panel
    max_load = bl.calcload.calc_max_load(file_list[0])
    #print("plot_event make_waterfall_plots: max_load={} is required for {}".format(max_load, file_list[0]))
    wf1 = bl.Waterfall(file_list[0], max_load=max_load)
    t0 = wf1.header["tstart"]
    source_name = wf1.header["source_name"]

    # Fix frequency boundaries if required.
    freqs = wf1.container.populate_freqs()
    if f_start is None:
        f_start = freqs[0]
    if f_stop is None:
        f_stop = freqs[-1]
    the_lowest, the_highest = sort2(f_start, f_stop)

    # rebin data to plot correctly with fewer points
    plot_f1, plot_data1 = wf1.grab_data(f_start=the_lowest, f_stop=the_highest)
    dec_fac_x, dec_fac_y = 1, 1
    if plot_data1.shape[0] > MAX_IMSHOW_POINTS[0]:
        dec_fac_x = plot_data1.shape[0] / MAX_IMSHOW_POINTS[0]
    if plot_data1.shape[1] > MAX_IMSHOW_POINTS[1]:
        dec_fac_y =  int(np.ceil(plot_data1.shape[1] /  MAX_IMSHOW_POINTS[1]))
    plot_data1 = rebin(plot_data1, dec_fac_x, dec_fac_y)

    # Compute the midpoint.
    the_midpoint = np.abs(the_lowest + the_highest) / 2.

    # Protect RAM utilisation.
    del wf1, freqs, plot_f1, plot_data1
    gc.collect()

    # Fill in each subplot for the full plot
    subplots = []
    for ii, filename in enumerate(file_list):
        logger.debug("make_waterfall_plots: file {} in list: {}".format(ii, filename))
        # identify panel
        subplot = plt.subplot(n_plots, 1, ii + 1)
        subplots.append(subplot)

        # read in data
        max_load = bl.calcload.calc_max_load(filename)
        wf = bl.Waterfall(filename, max_load=max_load)

        # Validate frequency range.
        freqs = wf.container.populate_freqs()
        ii_lowest, ii_highest = sort2(freqs[0], freqs[-1])
        logger.info("Processing: {}, freq lowest={}, highest={}".format(filename, ii_lowest, ii_highest))
        if the_lowest < ii_lowest or the_highest > ii_highest:
            logger.warning("Frequency range not compatible!  Ignoring this file.")
            # Protect RAM utilisation.
            del wf, freqs
            gc.collect()
            continue # Skip this file.

        # make plot with plot_waterfall
        source_name = wf.header["source_name"]
        this_plot = plot_waterfall(wf,
                                   f_start=the_lowest,
                                   f_stop=the_highest,
                                   **kwargs)

        # Title the full plot if processing the first file.
        if ii == 0:
            plot_title = "%s \n MJD:%5.5f (first file)" % (source_name, t0)
            plt.title(plot_title)

        # Format full plot.
        if ii < len(file_list)-1:
            plt.xticks(np.linspace(the_lowest, the_highest, num=4), ["","","",""])

        # Protect RAM utilisation.
        del wf, freqs
        gc.collect()

    # More overall plot formatting, axis labelling.
    factor = 1e6
    units = "Hz"
    xloc = np.linspace(the_lowest, the_highest, 5)
    xticks = [round(loc_freq) for loc_freq in (xloc - the_midpoint) * factor]
    if np.max(xticks) > 1000:
        xticks = [xt / 1000 for xt in xticks]
        units = "kHz"
    plt.xticks(xloc, xticks)
    plt.xlabel("Relative Frequency [%s] from %f MHz" % (units, the_midpoint), fontdict=font)

    # Add colorbar.
    cax = fig[0].add_axes([0.94, 0.11, 0.03, 0.77])
    fig[0].colorbar(this_plot, cax=cax, label="Normalized Power (Arbitrary Units)")

    # Adjust plots
    plt.subplots_adjust(hspace=0,wspace=0)

    # Save the figures.
    path_png = dirpath + source_name + "_fstart_{:0.6f}".format(f_start) \
                + "_fstop_{:0.6f}".format(f_stop) + ".png"
    plt.savefig(path_png, bbox_inches="tight")
    logger.info("Saved plot: {}".format(path_png))

    # show figure before closing if this is an interactive context
    mplbe = matplotlib.get_backend()
    logger.debug("make_waterfall_plots: backend = {}".format(mplbe))
    if mplbe != "agg":
        plt.show()

    # close all figure windows
    plt.close("all")

def cmd_line(args=None):
    r"""Coomand line parser"""
    parser = ArgumentParser(description='Make waterfall plots from a set of files, viewed from top to bottom.')
    parser.add_argument('file_list', type=str, nargs='+', help='List of files to plot')
    parser.add_argument('-f_start', type=float, default=None, help='Start frequency.  Default: None.')
    parser.add_argument('-f_stop', type=float, default=None, help='Stop frequency.  Default: None.')
    parser.add_argument('-plot_dir', type=str, default=".", help='Directory to receive the plot (.png).  Default: current directory.')
    if args is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(args)
    if args.file_list == []:
        os.system("stax -h")
        sys.exit(0)

    # Make the plots.
    args.plot_dir += "/"
    args.file_list = sorted(args.file_list)
    logger.info("Begin")
    make_waterfall_plots(args.file_list, args.plot_dir, args.f_start, args.f_stop)
    logger.info("End")

if __name__ == "__main__":
    cmd_line()
<commit_msg>Correct entry point name for stax<commit_after>#!/usr/bin/env python3

r"""
Make waterfall plots of a file set, view from top to bottom.
"""

import os
import sys
from os.path import dirname, abspath, isdir
import gc
from argparse import ArgumentParser
import logging
logger_name = "stax"
logger = logging.getLogger(logger_name)
logger.setLevel(logging.INFO)

# Plotting packages import
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use("agg")

# Math/Science package imports
import numpy as np

# Blimpy imports
import blimpy as bl
from blimpy.utils import rebin

# Plotting constants
fontsize=16
font = {"family" : "DejaVu Sans",
"size" : fontsize}
MAX_IMSHOW_POINTS = (4096, 1268)

def plot_waterfall(wf, f_start=None, f_stop=None, **kwargs):
    r"""
    Plot waterfall of data in a .fil or .h5 file.

    Parameters
    ----------
    wf : blimpy.Waterfall object
        Waterfall object of an H5 or Filterbank file containing the dynamic spectrum data.
    f_start : float
        Start frequency, in MHz.
    f_stop : float
        Stop frequency, in MHz.
    kwargs : dict
        Keyword args to be passed to matplotlib imshow().

    Notes
    -----
    Plot a single-panel waterfall plot (frequency vs. time vs. intensity)
    for one of the files in the set of interest, at the
    frequency of the expected event.
    """

    # Extract source name.
    source_name = wf.header["source_name"]

    # prepare font
    matplotlib.rc("font", **font)

    # Load in the data from fil
    plot_f, plot_data = wf.grab_data(f_start=f_start, f_stop=f_stop)

    # Make sure waterfall plot is under 4k*4k
    dec_fac_x, dec_fac_y = 1, 1

    # rebinning data to plot correctly with fewer points
    try:
        if plot_data.shape[0] > MAX_IMSHOW_POINTS[0]:
            dec_fac_x = plot_data.shape[0] / MAX_IMSHOW_POINTS[0]
        if plot_data.shape[1] > MAX_IMSHOW_POINTS[1]:
            dec_fac_y =  int(np.ceil(plot_data.shape[1] /  MAX_IMSHOW_POINTS[1]))
        plot_data = rebin(plot_data, dec_fac_x, dec_fac_y)
    except Exception as ex:
        print("\n*** Oops, grab_data returned plot_data.shape={}, plot_f.shape={}"
              .format(plot_data.shape, plot_f.shape))
        print("Waterfall info for {}:".format(wf.filename))
        wf.info()
        raise ValueError("*** Something is wrong with the grab_data output!") from ex

    # Rolled back PR #82

    # determine extent of the plotting panel for imshow
    extent=(plot_f[0], plot_f[-1], (wf.timestamps[-1]-wf.timestamps[0])*24.*60.*60, 0.0)

    # plot and scale intensity (log vs. linear)
    kwargs["cmap"] = kwargs.get("cmap", "viridis")
    plot_data = 10.0 * np.log10(plot_data)

    # get normalization parameters
    vmin = plot_data.min()
    vmax = plot_data.max()
    normalized_plot_data = (plot_data - vmin) / (vmax - vmin)

    # display the waterfall plot
    this_plot = plt.imshow(normalized_plot_data,
        aspect="auto",
        rasterized=True,
        interpolation="nearest",
        extent=extent,
        **kwargs
    )

    # add plot labels
    plt.xlabel("Frequency [Hz]",fontdict=font)
    plt.ylabel("Time [s]",fontdict=font)

    # add source name
    ax = plt.gca()
    plt.text(0.03, 0.8, source_name, transform=ax.transAxes, bbox=dict(facecolor="white"))

    return this_plot

def sort2(x, y):
    r""" Return lowest value, highest value"""
    if y < x:
        return y, x
    return x, y

def make_waterfall_plots(file_list, plot_dir, f_start=None, f_stop=None, **kwargs):
    r"""
    Make waterfall plots of a file set, view from top to bottom.

    Parameters
    ----------
    file_list : str
        List of filterbank files to plot in a stacked mode.
    f_start : float
        Start frequency, in MHz.
    f_stop : float
        Stop frequency, in MHz.
    source_name_list : list
        List of source names in the set, in order.
    kwargs : dict
        Keyword args to be passed to matplotlib imshow().
    """

    # prepare for plotting
    matplotlib.rc("font", **font)

    # set up the sub-plots
    n_plots = len(file_list)
    fig = plt.subplots(n_plots, sharex=True, sharey=True,figsize=(10, 2*n_plots))

    # get directory path for storing PNG files
    if plot_dir is None:
        dirpath = dirname(abspath(file_list[0])) + "/"
    else:
        if not isdir(plot_dir):
            os.mkdir(plot_dir)
        dirpath = plot_dir

    # read in data for the first panel
    max_load = bl.calcload.calc_max_load(file_list[0])
    #print("plot_event make_waterfall_plots: max_load={} is required for {}".format(max_load, file_list[0]))
    wf1 = bl.Waterfall(file_list[0], max_load=max_load)
    t0 = wf1.header["tstart"]
    source_name = wf1.header["source_name"]

    # Fix frequency boundaries if required.
    freqs = wf1.container.populate_freqs()
    if f_start is None:
        f_start = freqs[0]
    if f_stop is None:
        f_stop = freqs[-1]
    the_lowest, the_highest = sort2(f_start, f_stop)

    # rebin data to plot correctly with fewer points
    plot_f1, plot_data1 = wf1.grab_data(f_start=the_lowest, f_stop=the_highest)
    dec_fac_x, dec_fac_y = 1, 1
    if plot_data1.shape[0] > MAX_IMSHOW_POINTS[0]:
        dec_fac_x = plot_data1.shape[0] / MAX_IMSHOW_POINTS[0]
    if plot_data1.shape[1] > MAX_IMSHOW_POINTS[1]:
        dec_fac_y =  int(np.ceil(plot_data1.shape[1] /  MAX_IMSHOW_POINTS[1]))
    plot_data1 = rebin(plot_data1, dec_fac_x, dec_fac_y)

    # Compute the midpoint.
    the_midpoint = np.abs(the_lowest + the_highest) / 2.

    # Protect RAM utilisation.
    del wf1, freqs, plot_f1, plot_data1
    gc.collect()

    # Fill in each subplot for the full plot
    subplots = []
    for ii, filename in enumerate(file_list):
        logger.debug("make_waterfall_plots: file {} in list: {}".format(ii, filename))
        # identify panel
        subplot = plt.subplot(n_plots, 1, ii + 1)
        subplots.append(subplot)

        # read in data
        max_load = bl.calcload.calc_max_load(filename)
        wf = bl.Waterfall(filename, max_load=max_load)

        # Validate frequency range.
        freqs = wf.container.populate_freqs()
        ii_lowest, ii_highest = sort2(freqs[0], freqs[-1])
        logger.info("Processing: {}, freq lowest={}, highest={}".format(filename, ii_lowest, ii_highest))
        if the_lowest < ii_lowest or the_highest > ii_highest:
            logger.warning("Frequency range not compatible!  Ignoring this file.")
            # Protect RAM utilisation.
            del wf, freqs
            gc.collect()
            continue # Skip this file.

        # make plot with plot_waterfall
        source_name = wf.header["source_name"]
        this_plot = plot_waterfall(wf,
                                   f_start=the_lowest,
                                   f_stop=the_highest,
                                   **kwargs)

        # Title the full plot if processing the first file.
        if ii == 0:
            plot_title = "%s \n MJD:%5.5f (first file)" % (source_name, t0)
            plt.title(plot_title)

        # Format full plot.
        if ii < len(file_list)-1:
            plt.xticks(np.linspace(the_lowest, the_highest, num=4), ["","","",""])

        # Protect RAM utilisation.
        del wf, freqs
        gc.collect()

    # More overall plot formatting, axis labelling.
    factor = 1e6
    units = "Hz"
    xloc = np.linspace(the_lowest, the_highest, 5)
    xticks = [round(loc_freq) for loc_freq in (xloc - the_midpoint) * factor]
    if np.max(xticks) > 1000:
        xticks = [xt / 1000 for xt in xticks]
        units = "kHz"
    plt.xticks(xloc, xticks)
    plt.xlabel("Relative Frequency [%s] from %f MHz" % (units, the_midpoint), fontdict=font)

    # Add colorbar.
    cax = fig[0].add_axes([0.94, 0.11, 0.03, 0.77])
    fig[0].colorbar(this_plot, cax=cax, label="Normalized Power (Arbitrary Units)")

    # Adjust plots
    plt.subplots_adjust(hspace=0,wspace=0)

    # Save the figures.
    path_png = dirpath + source_name + "_fstart_{:0.6f}".format(f_start) \
                + "_fstop_{:0.6f}".format(f_stop) + ".png"
    plt.savefig(path_png, bbox_inches="tight")
    logger.info("Saved plot: {}".format(path_png))

    # show figure before closing if this is an interactive context
    mplbe = matplotlib.get_backend()
    logger.debug("make_waterfall_plots: backend = {}".format(mplbe))
    if mplbe != "agg":
        plt.show()

    # close all figure windows
    plt.close("all")

def cmd_tool(args=None):
    r"""Coomand line parser"""
    parser = ArgumentParser(description='Make waterfall plots from a set of files, viewed from top to bottom.')
    parser.add_argument('file_list', type=str, nargs='+', help='List of files to plot')
    parser.add_argument('-f_start', type=float, default=None, help='Start frequency.  Default: None.')
    parser.add_argument('-f_stop', type=float, default=None, help='Stop frequency.  Default: None.')
    parser.add_argument('-plot_dir', type=str, default=".", help='Directory to receive the plot (.png).  Default: current directory.')
    if args is None:
        args = parser.parse_args()
    else:
        args = parser.parse_args(args)
    if args.file_list == []:
        os.system("stax -h")
        sys.exit(0)

    # Make the plots.
    args.plot_dir += "/"
    args.file_list = sorted(args.file_list)
    logger.info("Begin")
    make_waterfall_plots(args.file_list, args.plot_dir, args.f_start, args.f_stop)
    logger.info("End")

if __name__ == "__main__":
    cmd_tool()
<|endoftext|>


<commit_before>import os
import subprocess
import re
import psutil
import platform

class SystemCollector(object):
    def get_uptime(self):

        with open('/proc/uptime', 'r') as line:
            contents = line.read().split()

        total_seconds = float(contents[0])

        MINUTE = 60
        HOUR = MINUTE * 60
        DAY = HOUR * 24

        days = int(total_seconds / DAY)
        hours = int((total_seconds % DAY) / HOUR)
        minutes = int((total_seconds % HOUR) / MINUTE)
        seconds = int(total_seconds % MINUTE)

        uptime = "{0} days {1} hours {2} minutes {3} seconds".format(days, hours, minutes, seconds)

        return uptime

    def get_system_info(self):
        distro = {}
        system_info = {}
        dist = platform.dist()
        if filter(None, dist):
            distro['distribution'] = dist[0]
            distro['release'] = '{0}/{1}'.format(dist[1], dist[2])
        else:
            distro['distribution'] = platform.system()
            distro['release'] = platform.release()

        system_info["distro"] = distro

        processor = {
            'cpu-cores': psutil.NUM_CPUS
        }

        try:
            if os.path.exists('/proc/cpuinfo'):
                model_name_raw = subprocess.Popen(['grep', '"model name"', '-m', '1', '/proc/cpuinfo'],
                                                  stdout=subprocess.PIPE,
                                                  close_fds=True).communicate()[0]
                cpu_model_name = model_name_raw.replace('model name', '').strip().lstrip(':').strip()
            elif os.path.exists('/var/run/dmesg.boot'):
                model_name_raw = subprocess.Popen(['grep', 'CPU:', '/var/run/dmesg.boot'],
                                                  stdout=subprocess.PIPE,
                                                  close_fds=True).communicate()[0]
                cpu_model_name = model_name_raw.replace('CPU:', '').strip()
            else:
                cpu_model_name = 'unknown'
        except Exception:
            cpu_model_name = 'unknown'

        processor['model-name'] = cpu_model_name

        system_info["processor"] = processor

        return system_info

    def get_memory_info(self):
        _swap_columns = ('swap_total', 'swap_used', 'swap_free', 'swap_percent_used')
        swap_values = dict(zip(_swap_columns, psutil.swap_memory()))
        memory_usage = psutil.virtual_memory()

        raw_data = {
            "total": memory_usage.total,
            "free": memory_usage.free,
            "used": memory_usage.used,
            "percent_used": int(memory_usage.percent)
        }

        raw_data.update(swap_values)

        data = {}
        for name, value in raw_data.iteritems():
            value = int(value)
            if name.find('percent') is -1:
                value /= 1024 * 1024  # Convert to MB

            data[name] = value

        return data

    def get_disk_usage(self, path_list):
        _columns = ('total', 'used', 'free')
        data = {}

        partition_list = psutil.disk_partitions(all=False)

        for path in path_list:
            try:
                part = filter(lambda x: x.mountpoint == path, partition_list)[0]
                if os.name == 'nt':
                    if 'cdrom' in part.opts or part.fstype == '':
                        # skip cd-rom drives with no disk in it; they may raise
                        # ENOENT, pop-up a Windows GUI error for a non-ready
                        # partition or just hang.
                        continue
                usage = psutil.disk_usage(part.mountpoint)
                row = dict(zip(_columns, map(lambda x: x / (1024 * 1024), usage)))  # Convert to MB
                row['volume'] = part.device
                row['path'] = part.mountpoint
                row['percent'] = int(usage.percent)

                data[part.device.replace('/dev/', '')] = row
            except IndexError:
                pass

        return data

    def get_disk_io(self):
        raw_data = psutil.disk_io_counters(True)
        data = {}
        for disk_name, disk_data in raw_data.iteritems():
            row_data = dict(disk_data._asdict())
            row_data['read_kb'] = row_data['read_bytes'] / 1024  # Convert to KB
            row_data['write_kb'] = row_data['write_bytes'] / 1024  # Convert to KB
            del(row_data['read_bytes'])
            del(row_data['write_bytes'])

            data[disk_name] = row_data

        return data

    def get_network_traffic(self):
        data = {}
        for device, stat in psutil.net_io_counters(pernic=True).iteritems():
            data[device] = {'kb_received': stat.bytes_recv / 1024, 'kb_transmitted': stat.bytes_sent / 1024}

        return data

    def get_load_average(self):
        _loadavg_columns = ('minute', 'five_minutes', 'fifteen_minutes')
        load_dict = dict(zip(_loadavg_columns, os.getloadavg()))

        cores = psutil.NUM_CPUS
        load_dict['cores'] = cores

        return load_dict

    def get_cpu_utilization(self):
        _columns = ('user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq', 'steal', 'quest', 'quest_nice')
        cpu_time_percent = psutil.cpu_times_percent(interval=3)
        data = dict(zip(_columns, cpu_time_percent))
        return data

system_info_collector = SystemCollector()

class ProcessInfoCollector(object):
    def __init__(self):
        memory = system_info_collector.get_memory_info()
        self.total_memory = memory['total']

    def process_list(self):
        stats = subprocess.Popen(['pidstat', '-ruht'],
                                 stdout=subprocess.PIPE, close_fds=True) \
            .communicate()[0]

        stats_data = stats.splitlines()
        del stats_data[0:2]  # Deletes Unix system data

        converted_data = []
        for line in stats_data:
            if re.search('command', line, re.IGNORECASE):  # Matches the first line
                header = line.split()
                del header[0]  # Deletes the # symbol
            else:
                command = line.split()
                data_dict = dict(zip(header, command))

                process_memory_mb = float(self.total_memory / 100) * float(
                    data_dict["%MEM"].replace(",", "."))  # Convert the % in MB
                memory = "{0:.3}".format(process_memory_mb)
                memory = memory.replace(",", ".")

                cpu = "{0:.2f}".format(float(data_dict["%CPU"].replace(",", ".")))
                cpu = cpu.replace(",", ".")

                command = data_dict["Command"]

                if not re.search('_', command, re.IGNORECASE):
                    extracted_data = {"cpu:%": cpu,
                                      "memory:mb": memory,
                                      "command": command}
                    converted_data.append(extracted_data)

        return converted_data

process_info_collector = ProcessInfoCollector()
<commit_msg>Sanitize path<commit_after>import os
import subprocess
import re
import psutil
import platform

class SystemCollector(object):
    def get_uptime(self):

        with open('/proc/uptime', 'r') as line:
            contents = line.read().split()

        total_seconds = float(contents[0])

        MINUTE = 60
        HOUR = MINUTE * 60
        DAY = HOUR * 24

        days = int(total_seconds / DAY)
        hours = int((total_seconds % DAY) / HOUR)
        minutes = int((total_seconds % HOUR) / MINUTE)
        seconds = int(total_seconds % MINUTE)

        uptime = "{0} days {1} hours {2} minutes {3} seconds".format(days, hours, minutes, seconds)

        return uptime

    def get_system_info(self):
        distro = {}
        system_info = {}
        dist = platform.dist()
        if filter(None, dist):
            distro['distribution'] = dist[0]
            distro['release'] = '{0}/{1}'.format(dist[1], dist[2])
        else:
            distro['distribution'] = platform.system()
            distro['release'] = platform.release()

        system_info["distro"] = distro

        processor = {
            'cpu-cores': psutil.NUM_CPUS
        }

        try:
            if os.path.exists('/proc/cpuinfo'):
                model_name_raw = subprocess.Popen(['grep', '"model name"', '-m', '1', '/proc/cpuinfo'],
                                                  stdout=subprocess.PIPE,
                                                  close_fds=True).communicate()[0]
                cpu_model_name = model_name_raw.replace('model name', '').strip().lstrip(':').strip()
            elif os.path.exists('/var/run/dmesg.boot'):
                model_name_raw = subprocess.Popen(['grep', 'CPU:', '/var/run/dmesg.boot'],
                                                  stdout=subprocess.PIPE,
                                                  close_fds=True).communicate()[0]
                cpu_model_name = model_name_raw.replace('CPU:', '').strip()
            else:
                cpu_model_name = 'unknown'
        except Exception:
            cpu_model_name = 'unknown'

        processor['model-name'] = cpu_model_name

        system_info["processor"] = processor

        return system_info

    def get_memory_info(self):
        _swap_columns = ('swap_total', 'swap_used', 'swap_free', 'swap_percent_used')
        swap_values = dict(zip(_swap_columns, psutil.swap_memory()))
        memory_usage = psutil.virtual_memory()

        raw_data = {
            "total": memory_usage.total,
            "free": memory_usage.free,
            "used": memory_usage.used,
            "percent_used": int(memory_usage.percent)
        }

        raw_data.update(swap_values)

        data = {}
        for name, value in raw_data.iteritems():
            value = int(value)
            if name.find('percent') is -1:
                value /= 1024 * 1024  # Convert to MB

            data[name] = value

        return data

    def get_disk_usage(self, path_list):
        _columns = ('total', 'used', 'free')
        data = {}

        partition_list = psutil.disk_partitions(all=True)

        def _sanitize(p):
            if p is not "/":
                return p.rstrip("/")
            return p
        path_list = map(_sanitize, path_list)

        for path in path_list:
            try:
                part = filter(lambda x: x.mountpoint == path, partition_list)[0]
                if os.name == 'nt':
                    if 'cdrom' in part.opts or part.fstype == '':
                        # skip cd-rom drives with no disk in it; they may raise
                        # ENOENT, pop-up a Windows GUI error for a non-ready
                        # partition or just hang.
                        continue
                usage = psutil.disk_usage(part.mountpoint)
                row = dict(zip(_columns, map(lambda x: x / (1024 * 1024), usage)))  # Convert to MB
                row['volume'] = part.device
                row['path'] = part.mountpoint
                row['percent'] = int(usage.percent)

                data[part.device.replace('/dev/', '')] = row
            except IndexError:
                pass

        return data

    def get_disk_io(self):
        raw_data = psutil.disk_io_counters(True)
        data = {}
        for disk_name, disk_data in raw_data.iteritems():
            row_data = dict(disk_data._asdict())
            row_data['read_kb'] = row_data['read_bytes'] / 1024  # Convert to KB
            row_data['write_kb'] = row_data['write_bytes'] / 1024  # Convert to KB
            del(row_data['read_bytes'])
            del(row_data['write_bytes'])

            data[disk_name] = row_data

        return data

    def get_network_traffic(self):
        data = {}
        for device, stat in psutil.net_io_counters(pernic=True).iteritems():
            data[device] = {'kb_received': stat.bytes_recv / 1024, 'kb_transmitted': stat.bytes_sent / 1024}

        return data

    def get_load_average(self):
        _loadavg_columns = ('minute', 'five_minutes', 'fifteen_minutes')
        load_dict = dict(zip(_loadavg_columns, os.getloadavg()))

        cores = psutil.NUM_CPUS
        load_dict['cores'] = cores

        return load_dict

    def get_cpu_utilization(self):
        _columns = ('user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq', 'steal', 'quest', 'quest_nice')
        cpu_time_percent = psutil.cpu_times_percent(interval=3)
        data = dict(zip(_columns, cpu_time_percent))
        return data

system_info_collector = SystemCollector()

class ProcessInfoCollector(object):
    def __init__(self):
        memory = system_info_collector.get_memory_info()
        self.total_memory = memory['total']

    def process_list(self):
        stats = subprocess.Popen(['pidstat', '-ruht'],
                                 stdout=subprocess.PIPE, close_fds=True) \
            .communicate()[0]

        stats_data = stats.splitlines()
        del stats_data[0:2]  # Deletes Unix system data

        converted_data = []
        for line in stats_data:
            if re.search('command', line, re.IGNORECASE):  # Matches the first line
                header = line.split()
                del header[0]  # Deletes the # symbol
            else:
                command = line.split()
                data_dict = dict(zip(header, command))

                process_memory_mb = float(self.total_memory / 100) * float(
                    data_dict["%MEM"].replace(",", "."))  # Convert the % in MB
                memory = "{0:.3}".format(process_memory_mb)
                memory = memory.replace(",", ".")

                cpu = "{0:.2f}".format(float(data_dict["%CPU"].replace(",", ".")))
                cpu = cpu.replace(",", ".")

                command = data_dict["Command"]

                if not re.search('_', command, re.IGNORECASE):
                    extracted_data = {"cpu:%": cpu,
                                      "memory:mb": memory,
                                      "command": command}
                    converted_data.append(extracted_data)

        return converted_data

process_info_collector = ProcessInfoCollector()
<|endoftext|>


<commit_before># Copyright (c) 2016 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from collections import namedtuple
from collections.abc import MutableMapping

class URI (str):
    """URI string

    This is exactly like a str object (and will even test as equal to
    strings of identical value). It is a literal instance of str.

    The only difference is that these also return true as instances of
    URI, which is a distinct type of XMP value.

        >>> isinstance(URI("hello"), str)
        True
        >>> isinstance(URI("hello"), URI)
        True
        >>> isinstance("hello", URI)
        False
        >>> URI("hello") == "hello"
        True
        >>> "hello" == URI("hello")
        True

    Other than pass the "is this a URI" test, URI objects can't do
    anything that str objects can't do.
    """

    def __new__ (cls, *args, **kwargs):
        """Return a new str object."""
        return str.__new__(cls, *args, **kwargs)

    def __repr__ (self):
        """Return a str representing a URI (rather than a str)."""
        return "<{} {}>".format(self.__class__.__name__, self)

URI_RDF = URI("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
URI_X   = URI("adobe:ns:meta/")
URI_XML = URI("http://www.w3.org/XML/1998/namespace")

UriTag = namedtuple("UriTag", ("uri", "name"))

class XmpBaseValue:
    """Abstract XMP Value"""

    def __init__ (self):
        """Raise NotImplementedError.

        I don't want these to ever be created directly.
        """
        self.__not_implemented()

    @property
    def py_value (self):
        """Raise NotImplementedError.

        This should be overwritten to return a pythonic value that
        behaves however a user would expect without needing to know
        about XMP's more advanced features.

        For example, if this is an XmpInteger, this property should
        return an int object.
        """
        self.__not_implemented()

    def compare (self, rhs):
        """Raise NotImplementedError.

        This should be overwritten to return an integer: 0 if this
        instance is "equal" to the right-hand side, a negative number if
        this instance is "less," or a positive number if this instance
        is "greater."

        What "equal," "less," and "greater" mean will be up to you.
        """
        self.__not_implemented()

    def is_equal (self, rhs):
        """Return true if the compare method returns 0.

        It can usually be simpler to check equality than to run a full
        compare, so you should probably overwrite this, but you don't
        have to.
        """
        return self.compare(rhs) == 0

    def __eq__ (self, rhs):
        """Return true if this is equal to the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only actually test equality if the right hand side
            # is a valid type.
            return self.is_equal(rhs)

        else:
            # If it's not a valid type, it's not equal. No need to raise
            # TypeError.
            return False

    def __ne__ (self, rhs):
        """Return false if this is equal to the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only actually test equality if the right hand side
            # is a valid type.
            return not self.is_equal(rhs)

        else:
            # If it's not a valid type, it's not equal. No need to raise
            # TypeError.
            return True

    def __lt__ (self, rhs):
        """Return true if this is less than the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) < 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, "<")

    def __gt__ (self, rhs):
        """Return true if this is greater than the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) > 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, ">")

    def __le__ (self, rhs):
        """Return true if this is less than or equal to the right-hand
        side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) <= 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, "<=")

    def __ge__ (self, rhs):
        """Return true if this is greater than or equal to the
        right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) >= 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, ">=")

    def raise_invalid_init (self, expected, actual):
        """Raise an error that the init value is invalid.

        Both arguments should be str objects.
        """

        raise TypeError("Init value must be {}, not {}".format(
                expected, actual))

    def __is_same_class (self, rhs):
        return self.__class__ is rhs.__class__

    def __raise_cant_compare (self, rhs, op):
        raise TypeError("unorderable types: {}() {} {}()".format(
                self.__class__.__name__, op, rhs.__class__.__name__))

    def __not_implemented (self):
        raise NotImplementedError("{} isn't meant to ever be " \
                "implemented directly".format(self.__class__.__name__))

class XmpBaseSimpleValue (XmpBaseValue):

    @property
    def py_value (self):
        """Return a pythonic value."""
        return self.value

    def compare (self, rhs):
        """Return a comparison integer."""
        if self.is_equal(rhs):
            return 0

        elif self.value < rhs.value:
            return -1

        else:
            return 1

    def is_equal (self, rhs):
        """Return true if both values are equal."""
        return self.value == rhs.value

    def __repr__ (self):
        """Return an unambiguous representation."""
        if str(self):
            return "<{} {}>".format(self.__class__.__name__, str(self))

        else:
            return "<{}>".format(self.__class__.__name__)

class XmpURI (XmpBaseSimpleValue):
    """XMP URI value"""

    def __init__ (self, uri):
        """Set the internal value."""

        if isinstance(uri, str):
            # We expect a str object.
            self.__init_from_str(uri)

        else:
            # It's also possible that the value is an XMP value. Or
            # maybe it's just not a valid init value at all.
            self.__init_maybe_from_xmp_value(uri)

    def __init_from_str (self, uri):
        if isinstance(uri, URI):
            # We know we have a str object, and if that str is a URI, we
            # don't even need to init a new object. We can just set it.
            self.value = uri

        else:
            # Ok it's some other sort of str object, so we need to
            # create an actual URI object from it.
            self.value = URI(uri)

    def __init_maybe_from_xmp_value (self, uri):
        if isinstance(uri, XmpURI):
            # Cool! It's already an XMP URI. We can just take its value
            # as-is.
            self.value = uri.value

        else:
            # Oh no! It's some other thing that I can't deal with.
            self.raise_invalid_init("URI", uri.__class__.__name__)

    def __str__ (self):
        """Return the URI value itself."""
        return self.value

class XmpText (XmpBaseSimpleValue):
    """Simple XMP Value"""

    # This is the basic type I expect for the internal value. Also, the
    # py_value property should return something of this type.
    py_type = str

    def __init__ (self, value):
        """Set the internal value."""

        if isinstance(value, self.__class__):
            self.value = value.value

        else:
            self.__init_from_py_value(value)

    def __str__ (self):
        """Return a string representation as would appear in XML."""
        return str(self.value)

    def is_valid_init_value (self, value):
        """Return true if the value is of a valid init format."""
        return isinstance(value, self.py_type)

    def format_valid_init_value (self, value):
        """Return the value we should store."""
        return value

    def __init_from_py_value (self, value):
        # Check that the init value is something we can work with.
        if not self.is_valid_init_value(value):
            # If it's not valid, raise TypeError.
            self.raise_invalid_init(self.py_type.__name__,
                    value.__class__.__name__)

        # Cool! Set our internal value.
        self.value = self.format_valid_init_value(value)

class XmpInteger (XmpText):
    """XMP Integer"""

    py_type = int

    def __str__ (self):
        return "{:d}".format(self.value)

class XmpBaseCollection (XmpBaseValue):
    """XMP Collection"""

    @property
    def py_value (self):
        return self

class XmpBaseArray (XmpBaseCollection):
    """XMP Array"""

    # This must be a named UriTag duple.
    xmp_tag = None

    # This must be a subclass of XmpBaseValue.
    xmp_type = None

    def __init__ (self, iterable = ( )):
        assert isinstance(self.xmp_tag, UriTag) \
                and issubclass(self.xmp_type, XmpBaseValue)

        self.value = [ ]
        self.extend(iterable)

    def __getitem__ (self, index):
        return self.value[index].py_value

    def __len__ (self):
        return len(self.value)

    def __contains__ (self, value):
        for wrapper in self.value:
            if wrapper.loose_equal(value):
                return True

        return False

    def __iter__ (self):
        for wrapper in self.value:
            yield wrapper.py_value

    def __reversed__ (self):
        for wrapper in reversed(self.value):
            yield wrapper.py_value

    def index (self, value, i = 0, j = None):
        if j is None:
            j = len(self)

        for pos in range(i, j):
            if self.value[pos].loose_equal(value):
                return pos

        raise ValueError("{} is not in array".format(repr(value)))

    def count (self, value):
        count = 0

        for wrapper in self.value:
            if wrapper.loose_equal(value):
                count += 1

        return count

class VanillaXMP (MutableMapping):

    def __delitem__ (self, key):
        pass

    def __getitem__ (self, key):
        pass

    def __iter__ (self):
        pass

    def __len__ (self):
        pass

    def __setitem__ (self, key, value):
        pass
<commit_msg>Add truthiness to XMP values<commit_after># Copyright (c) 2016 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
from collections import namedtuple
from collections.abc import MutableMapping

class URI (str):
    """URI string

    This is exactly like a str object (and will even test as equal to
    strings of identical value). It is a literal instance of str.

    The only difference is that these also return true as instances of
    URI, which is a distinct type of XMP value.

        >>> isinstance(URI("hello"), str)
        True
        >>> isinstance(URI("hello"), URI)
        True
        >>> isinstance("hello", URI)
        False
        >>> URI("hello") == "hello"
        True
        >>> "hello" == URI("hello")
        True

    Other than pass the "is this a URI" test, URI objects can't do
    anything that str objects can't do.
    """

    def __new__ (cls, *args, **kwargs):
        """Return a new str object."""
        return str.__new__(cls, *args, **kwargs)

    def __repr__ (self):
        """Return a str representing a URI (rather than a str)."""
        return "<{} {}>".format(self.__class__.__name__, self)

URI_RDF = URI("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
URI_X   = URI("adobe:ns:meta/")
URI_XML = URI("http://www.w3.org/XML/1998/namespace")

UriTag = namedtuple("UriTag", ("uri", "name"))

class XmpBaseValue:
    """Abstract XMP Value"""

    def __init__ (self):
        """Raise NotImplementedError.

        I don't want these to ever be created directly.
        """
        self.__not_implemented()

    @property
    def py_value (self):
        """Raise NotImplementedError.

        This should be overwritten to return a pythonic value that
        behaves however a user would expect without needing to know
        about XMP's more advanced features.

        For example, if this is an XmpInteger, this property should
        return an int object.
        """
        self.__not_implemented()

    def compare (self, rhs):
        """Raise NotImplementedError.

        This should be overwritten to return an integer: 0 if this
        instance is "equal" to the right-hand side, a negative number if
        this instance is "less," or a positive number if this instance
        is "greater."

        What "equal," "less," and "greater" mean will be up to you.
        """
        self.__not_implemented()

    def is_equal (self, rhs):
        """Return true if the compare method returns 0.

        It can usually be simpler to check equality than to run a full
        compare, so you should probably overwrite this, but you don't
        have to.
        """
        return self.compare(rhs) == 0

    def __eq__ (self, rhs):
        """Return true if this is equal to the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only actually test equality if the right hand side
            # is a valid type.
            return self.is_equal(rhs)

        else:
            # If it's not a valid type, it's not equal. No need to raise
            # TypeError.
            return False

    def __ne__ (self, rhs):
        """Return false if this is equal to the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only actually test equality if the right hand side
            # is a valid type.
            return not self.is_equal(rhs)

        else:
            # If it's not a valid type, it's not equal. No need to raise
            # TypeError.
            return True

    def __lt__ (self, rhs):
        """Return true if this is less than the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) < 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, "<")

    def __gt__ (self, rhs):
        """Return true if this is greater than the right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) > 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, ">")

    def __le__ (self, rhs):
        """Return true if this is less than or equal to the right-hand
        side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) <= 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, "<=")

    def __ge__ (self, rhs):
        """Return true if this is greater than or equal to the
        right-hand side."""
        if self.__is_same_class(rhs):
            # We'll only compare the values if the right hand side is a
            # valid type.
            return self.compare(rhs) >= 0

        else:
            # If it's not a valid type, we'll raise an exception.
            self.__raise_cant_compare(rhs, ">=")

    def __bool__ (self):
        """Return truthiness."""
        self.__not_implemented()

    def raise_invalid_init (self, expected, actual):
        """Raise an error that the init value is invalid.

        Both arguments should be str objects.
        """

        raise TypeError("Init value must be {}, not {}".format(
                expected, actual))

    def __is_same_class (self, rhs):
        return self.__class__ is rhs.__class__

    def __raise_cant_compare (self, rhs, op):
        raise TypeError("unorderable types: {}() {} {}()".format(
                self.__class__.__name__, op, rhs.__class__.__name__))

    def __not_implemented (self):
        raise NotImplementedError("{} isn't meant to ever be " \
                "implemented directly".format(self.__class__.__name__))

class XmpBaseSimpleValue (XmpBaseValue):

    @property
    def py_value (self):
        """Return a pythonic value."""
        return self.value

    def compare (self, rhs):
        """Return a comparison integer."""
        if self.is_equal(rhs):
            return 0

        elif self.value < rhs.value:
            return -1

        else:
            return 1

    def is_equal (self, rhs):
        """Return true if both values are equal."""
        return self.value == rhs.value

    def __repr__ (self):
        """Return an unambiguous representation."""
        if str(self):
            return "<{} {}>".format(self.__class__.__name__, str(self))

        else:
            return "<{}>".format(self.__class__.__name__)

    def __bool__ (self):
        """Return the same truthiness as our value."""
        return bool(self.value)

class XmpURI (XmpBaseSimpleValue):
    """XMP URI value"""

    def __init__ (self, uri):
        """Set the internal value."""

        if isinstance(uri, str):
            # We expect a str object.
            self.__init_from_str(uri)

        else:
            # It's also possible that the value is an XMP value. Or
            # maybe it's just not a valid init value at all.
            self.__init_maybe_from_xmp_value(uri)

    def __init_from_str (self, uri):
        if isinstance(uri, URI):
            # We know we have a str object, and if that str is a URI, we
            # don't even need to init a new object. We can just set it.
            self.value = uri

        else:
            # Ok it's some other sort of str object, so we need to
            # create an actual URI object from it.
            self.value = URI(uri)

    def __init_maybe_from_xmp_value (self, uri):
        if isinstance(uri, XmpURI):
            # Cool! It's already an XMP URI. We can just take its value
            # as-is.
            self.value = uri.value

        else:
            # Oh no! It's some other thing that I can't deal with.
            self.raise_invalid_init("URI", uri.__class__.__name__)

    def __str__ (self):
        """Return the URI value itself."""
        return self.value

class XmpText (XmpBaseSimpleValue):
    """Simple XMP Value"""

    # This is the basic type I expect for the internal value. Also, the
    # py_value property should return something of this type.
    py_type = str

    def __init__ (self, value):
        """Set the internal value."""

        if isinstance(value, self.__class__):
            self.value = value.value

        else:
            self.__init_from_py_value(value)

    def __str__ (self):
        """Return a string representation as would appear in XML."""
        return str(self.value)

    def is_valid_init_value (self, value):
        """Return true if the value is of a valid init format."""
        return isinstance(value, self.py_type)

    def format_valid_init_value (self, value):
        """Return the value we should store."""
        return value

    def __init_from_py_value (self, value):
        # Check that the init value is something we can work with.
        if not self.is_valid_init_value(value):
            # If it's not valid, raise TypeError.
            self.raise_invalid_init(self.py_type.__name__,
                    value.__class__.__name__)

        # Cool! Set our internal value.
        self.value = self.format_valid_init_value(value)

class XmpInteger (XmpText):
    """XMP Integer"""

    py_type = int

    def __str__ (self):
        return "{:d}".format(self.value)

class XmpBaseCollection (XmpBaseValue):
    """XMP Collection"""

    @property
    def py_value (self):
        return self

class XmpBaseArray (XmpBaseCollection):
    """XMP Array"""

    # This must be a named UriTag duple.
    xmp_tag = None

    # This must be a subclass of XmpBaseValue.
    xmp_type = None

    def __init__ (self, iterable = ( )):
        assert isinstance(self.xmp_tag, UriTag) \
                and issubclass(self.xmp_type, XmpBaseValue)

        self.value = [ ]
        self.extend(iterable)

    def __getitem__ (self, index):
        return self.value[index].py_value

    def __len__ (self):
        return len(self.value)

    def __contains__ (self, value):
        for wrapper in self.value:
            if wrapper.loose_equal(value):
                return True

        return False

    def __iter__ (self):
        for wrapper in self.value:
            yield wrapper.py_value

    def __reversed__ (self):
        for wrapper in reversed(self.value):
            yield wrapper.py_value

    def index (self, value, i = 0, j = None):
        if j is None:
            j = len(self)

        for pos in range(i, j):
            if self.value[pos].loose_equal(value):
                return pos

        raise ValueError("{} is not in array".format(repr(value)))

    def count (self, value):
        count = 0

        for wrapper in self.value:
            if wrapper.loose_equal(value):
                count += 1

        return count

class VanillaXMP (MutableMapping):

    def __delitem__ (self, key):
        pass

    def __getitem__ (self, key):
        pass

    def __iter__ (self):
        pass

    def __len__ (self):
        pass

    def __setitem__ (self, key, value):
        pass
<|endoftext|>


<commit_before>from SimPEG import Survey, Problem, Utils, np, sp
from simpegEM.Utils.EMUtils import omega

class FieldsFDEM(Problem.Fields):
    """Fancy Field Storage for a FDEM survey."""
    knownFields = {}
    dtype = complex

class FieldsFDEM_e(FieldsFDEM):
    knownFields = {'e_sol':'E'}
    aliasFields = {
                    'e' : ['e_sol','E','_e'],
                    'b' : ['e_sol','F','_b']
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl

    def _e(self, e_sol, srcList):
        e = e_sol
        for i, src in enumerate(srcList):
            ePrimary = src.ePrimary(self.survey.prob)
            if ePrimary is not None:
                e[:,i] += ePrimary
        return e

    def _b(self, e_sol, srcList):
        C = self._edgeCurl
        b = (C * e_sol)
        for i, src in enumerate(srcList):
            b[:,i] *= - 1./(1j*omega(src.freq))
            S_m, _ = src.eval(self.survey.prob)
            if S_m is not None:
                b[:,i] += 1./(1j*omega(src.freq)) * S_m

            bPrimary = src.bPrimary(self.survey.prob)
            if bPrimary is not None:
                b[:,i] += bPrimary

        return b

    def _bDeriv(self, e, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        # S_mDeriv,_ = src.getSourceDeriv(self.survey.prob, v, adjoint)
        # if S_mDeriv is None:
        #     return None
        # else:
        #     return 1./(1j*omega(src.freq)) * S_mDeriv

class FieldsFDEM_b(FieldsFDEM):
    knownFields = {'b_sol':'F'}
    aliasFields = {
                    'b' : ['b_sol','F','_b'],
                    'e' : ['b_sol','E','_e']
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl
        self._MeSigmaI = self.survey.prob.MeSigmaI
        self._MfMui = self.survey.prob.MfMui

    def _b(self, b_sol, srcList):
        b = b_sol

        for i, src in enumerate(srcList):
            bPrimary = src.bPrimary(self.survey.prob)
            if bPrimary is not None:
                b[:,i] += bPrimary
        return b

    def _e(self, b_sol, srcList):
        e = self._MeSigmaI * ( self._edgeCurl.T * ( self._MfMui * b_sol))

        for i,src in enumerate(srcList):
            _,S_e = src.eval(self.survey.prob)
            if S_e is not None:
                e += -self._MeSigmaI*S_e

            ePrimary = src.ePrimary(self.survey.prob)
            if ePrimary is not None:
                e[:,i] += ePrimary

        return e

    def _eDeriv(self, b_sol, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        _,S_eDeriv = src.evalDeriv(self.survey.prob, v, adjoint)

        if S_eDeriv is None:
            return None
        else:
            return -S_eDeriv

class FieldsFDEM_j(FieldsFDEM):
    knownFields = {'j_sol':'F'}
    aliasFields = {
                    'j' : ['j_sol','F','_j'],
                    'h' : ['j_sol','E','_h']
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl
        self._MeMuI = self.survey.prob.MeMuI
        self._MfRho = self.survey.prob.MfRho
        self._curModel = self.survey.prob.curModel

    def _j(self, j_sol, srcList):
        j = j_sol
        for i, src in enumerate(srcList):
            jPrimary = src.jPrimary(self.survey.prob)
            if jPrimary is not None:
                j[:,i] += jPrimary
        return j

    def _h(self, j_sol, srcList):
        MeMuI = self._MeMuI
        C = self._edgeCurl
        MfRho = self._MfRho

        h =  MeMuI * (C.T * (MfRho * j_sol) )

        for i, src in enumerate(srcList):
            h[:,i] *= -1./(1j*omega(src.freq))
            S_m,_ = src.eval(self.survey.prob)
            if S_m is not None:
                h[:,i] += 1./(1j*omega(src.freq)) * MeMuI * S_m

            hPrimary = src.hPrimary(self.survey.prob)
            if hPrimary is not None:
                h[:,i] += hPrimary

        return h

    def _hDeriv(self, j_sol, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        sig = self._curModel.transform
        sigi = 1/sig
        dsig_dm = self._curModel.transformDeriv
        dsigi_dsig = -Utils.sdiag(sigi)**2
        dMf_dsigi = self.mesh.getFaceInnerProductDeriv(sigi)(j)
        sigi = self._MfRho

        S_mDeriv,_ = src.getSourceDeriv(self.survey.prob, v, adjoint)

        if not adjoint:
            h_Deriv= -(1./(1j*omega(freq))) * MeMuI * ( C.T * ( dMf_dsigi * ( dsigi_dsig * ( dsig_dm * v ) ) ) )
        else:
            h_Deriv= -(1./(1j*omega(freq))) * dsig_dm.T * ( dsigi_dsig.T * ( dMf_dsigi.T * ( C * ( MeMuI.T * v ) ) ) )

        if S_mDeriv is not None:
            return 1./(1j*omega(src.freq)) * S_mDeriv + h_Deriv

class FieldsFDEM_h(FieldsFDEM):
    knownFields = {'h_sol':'E'}
    aliasFields = {
                    'h' : ['h_sol','E','_h'],
                    'j' : ['h_sol','F','_j']
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl
        self._MeMuI = self.survey.prob.MeMuI
        self._MfRho = self.survey.prob.MfRho

    def _h(self, h_sol, srcList):
        h = h_sol
        for i, src in enumerate(srcList):
            hPrimary = src.hPrimary(self.survey.prob)
            if hPrimary is not None:
                h[:,i] += hPrimary
            return h

    def _j(self, h_sol, srcList):
        j = self._edgeCurl*h_sol
        for i, src in enumerate(srcList):
            _,S_e = src.eval(self.survey.prob)
            if S_e is not None:
                j[:,i] += -S_e

            jPrimary = src.jPrimary(self.survey.prob)
            if jPrimary is not None:
                j[:,i] += jPrimary
        return j

    def _jDeriv(self, h_sol, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        _,S_eDeriv = src.getSourceDeriv(self.survey.prob, v, adjoint)
        if S_eDeriv is None:
            return None
        else:
            return - S_eDeriv

    # def calcFields(self, sol, freq, fieldType, adjoint=False):
    #     j = sol
    #     if fieldType == 'j':
    #         return j
    #     elif fieldType == 'h':
    #         MeMuI = self._MeMuI
    #         C = self.mesh.edgeCurl
    #         MfRho = self._MfRho
    #         if not adjoint:
    #             h = -(1./(1j*omega(freq))) * MeMuI * ( C.T * ( MfRho * j ) )
    #         else:
    #             h = -(1./(1j*omega(freq))) * MfRho.T * ( C * ( MeMuI.T * j ) )
    #         return h
    #     raise NotImplementedError('fieldType "%s" is not implemented.' % fieldType)

    # def calcFieldsDeriv(self, sol, freq, fieldType, v, adjoint=False):
    #     j = sol
    #     if fieldType == 'j':
    #         return None
    #     elif fieldType == 'h':
    #         MeMuI = self._MeMuI
    #         C = self.mesh.edgeCurl
    #         sig = self._curModel.transform
    #         sigi = 1/sig
    #         dsig_dm = self._curModel.transformDeriv
    #         dsigi_dsig = -Utils.sdiag(sigi)**2
    #         dMf_dsigi = self.mesh.getFaceInnerProductDeriv(sigi)(j)
    #         sigi = self._MfRho
    #         if not adjoint:
    #             return -(1./(1j*omega(freq))) * MeMuI * ( C.T * ( dMf_dsigi * ( dsigi_dsig * ( dsig_dm * v ) ) ) )
    #         else:
    #             return -(1./(1j*omega(freq))) * dsig_dm.T * ( dsigi_dsig.T * ( dMf_dsigi.T * ( C * ( MeMuI.T * v ) ) ) )
    #     raise NotImplementedError('fieldType "%s" is not implemented.' % fieldType)

    # def calcFields(self, sol, freq, fieldType, adjoint=False):
    #     h = sol
    #     if fieldType == 'j':
    #         C = self.mesh.edgeCurl
    #         if adjoint:
    #             return C.T*h
    #         return C*h
    #     elif fieldType == 'h':
    #         return h
    #     raise NotImplementedError('fieldType "%s" is not implemented.' % fieldType)

    # def calcFieldsDeriv(self, sol, freq, fieldType, v, adjoint=False):
    #     return None<commit_msg>broke apart primary-secondary<commit_after>from SimPEG import Survey, Problem, Utils, np, sp
from simpegEM.Utils.EMUtils import omega

class FieldsFDEM(Problem.Fields):
    """Fancy Field Storage for a FDEM survey."""
    knownFields = {}
    dtype = complex

class FieldsFDEM_e(FieldsFDEM):
    knownFields = {'e_sol':'E'}
    aliasFields = {
                    'e' : ['e_sol','E','_e'],
                    'ePrimary' : ['e_sol','E','_ePrimary'],
                    'eSecondary' : ['e_sol','E','_eSecondary'],
                    'b' : ['e_sol','F','_b'],
                    'bPrimary' : ['e_sol','F','_bPrimary'],
                    'bSecondary' : ['e_sol','F','_bSecondary']
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl

    def _ePrimary(self, e_sol, srcList):
        ePrimary = np.zeros_like(e_sol)
        for i, src in enumerate(srcList):
            ep = src.ePrimary(self.survey.prob)
            if ep is not None:
                ePrimary[:,i] = ep
        return ePrimary

    def _eSecondary(self, e_sol, srcList):
        return e_sol

    def _e(self, e_sol, srcList):
        return self._ePrimary(e_sol,srcList) + self._eSecondary(e_sol,srcList)

    def _bPrimary(self, e_sol, srcList):
        bPrimary = np.zeros([self._edgeCurl.shape[0],e_sol.shape[1]],dtype = complex)
        for i, src in enumerate(srcList):
            bp = src.bPrimary(self.survey.prob)
            if bp is not None:
                bPrimary[:,i] += bp
        return bPrimary

    def _bSecondary(self, e_sol, srcList):
        C = self._edgeCurl
        b = (C * e_sol)
        for i, src in enumerate(srcList):
            b[:,i] *= - 1./(1j*omega(src.freq))
            S_m, _ = src.eval(self.survey.prob)
            if S_m is not None:
                b[:,i] += 1./(1j*omega(src.freq)) * S_m
        return b

    def _b(self, e_sol, srcList):
        return self._bPrimary(e_sol, srcList) + self._bSecondary(e_sol, srcList)

    def _bDeriv(self, e, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        # S_mDeriv,_ = src.getSourceDeriv(self.survey.prob, v, adjoint)
        # if S_mDeriv is None:
        #     return None
        # else:
        #     return 1./(1j*omega(src.freq)) * S_mDeriv

class FieldsFDEM_b(FieldsFDEM):
    knownFields = {'b_sol':'F'}
    aliasFields = {
                    'b' : ['b_sol','F','_b'],
                    'bPrimary' : ['b_sol','F','_bPrimary'],
                    'bSecondary' : ['b_sol','F','_bSecondary'],
                    'e' : ['b_sol','E','_e'],
                    'ePrimary' : ['b_sol','E','_ePrimary'],
                    'eSecondary' : ['b_sol','E','_eSecondary'],
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl
        self._MeSigmaI = self.survey.prob.MeSigmaI
        self._MfMui = self.survey.prob.MfMui

    def _bPrimary(self, b_sol, srcList):
        bPrimary = np.zeros_like(b_sol)
        for i, src in enumerate(srcList):
            bp = src.bPrimary(self.survey.prob)
            if bp is not None:
                bPrimary[:,i] = bp
        return bPrimary

    def _bSecondary(self, b_sol, srcList):
        return b_sol

    def _b(self, b_sol, srcList):
        return self._bPrimary(b_sol, srcList) + self._bSecondary(b_sol, srcList)

    def _ePrimary(self, b_sol, srcList):
        ePrimary = np.zeros([self._edgeCurl.shape[1],b_sol.shape[1]],dtype = complex)
        for i,src in enumerate(srcList):
            ep = src.ePrimary(self.survey.prob)
            if ep is not None:
                ePrimary[:,i] = ep
        return ePrimary

    def _eSecondary(self, b_sol, srcList):
        e = self._MeSigmaI * ( self._edgeCurl.T * ( self._MfMui * b_sol))
        for i,src in enumerate(srcList):
            _,S_e = src.eval(self.survey.prob)
            if S_e is not None:
                e += -self._MeSigmaI*S_e

        return e

    def _e(self, b_sol, srcList):
        return self._ePrimary(b_sol, srcList) + self._eSecondary(b_sol, srcList)

    def _eDeriv(self, b_sol, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        _,S_eDeriv = src.evalDeriv(self.survey.prob, v, adjoint)

        if S_eDeriv is None:
            return None
        else:
            return -S_eDeriv

class FieldsFDEM_j(FieldsFDEM):
    knownFields = {'j_sol':'F'}
    aliasFields = {
                    'j' : ['j_sol','F','_j'],
                    'jPrimary' : ['j_sol','F','_jPrimary'],
                    'jSecondary' : ['j_sol','F','_jSecondary'],
                    'h' : ['j_sol','E','_h'],
                    'hPrimary' : ['j_sol','E','_hPrimary'],
                    'hSecondary' : ['j_sol','E','_hSecondary'],
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl
        self._MeMuI = self.survey.prob.MeMuI
        self._MfRho = self.survey.prob.MfRho
        self._curModel = self.survey.prob.curModel

    def _jPrimary(self, j_sol, srcList):
        jPrimary = np.zeros_like(j_sol)
        for i, src in enumerate(srcList):
            jp = src.jPrimary(self.survey.prob)
            if jp is not None:
                jPrimary[:,i] += jp
        return jPrimary

    def _jSecondary(self, j_sol, srcList):
        return j_sol

    def _j(self, j_sol, srcList):
        return self._jPrimary(j_sol, srcList) + self._jSecondary(j_sol, srcList)

    def _hPrimary(self, j_sol, srcList):
        hPrimary = np.zeros([self._edgeCurl.shape[1],j_sol.shape[1]],dtype = complex)
        for i, src in enumerate(srcList):
            hp = src.hPrimary(self.survey.prob)
            if hp is not None:
                hPrimary[:,i] = hp
        return hPrimary

    def _hSecondary(self, j_sol, srcList):
        MeMuI = self._MeMuI
        C = self._edgeCurl
        MfRho = self._MfRho
        h =  MeMuI * (C.T * (MfRho * j_sol) )
        for i, src in enumerate(srcList):
            h[:,i] *= -1./(1j*omega(src.freq))
            S_m,_ = src.eval(self.survey.prob)
            if S_m is not None:
                h[:,i] += 1./(1j*omega(src.freq)) * MeMuI * S_m
        return h

    def _h(self, j_sol, srcList):
        return self._hPrimary(j_sol, srcList) + self._hSecondary(j_sol, srcList)

    def _hDeriv(self, j_sol, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        sig = self._curModel.transform
        sigi = 1/sig
        dsig_dm = self._curModel.transformDeriv
        dsigi_dsig = -Utils.sdiag(sigi)**2
        dMf_dsigi = self.mesh.getFaceInnerProductDeriv(sigi)(j)
        sigi = self._MfRho

        S_mDeriv,_ = src.getSourceDeriv(self.survey.prob, v, adjoint)

        if not adjoint:
            h_Deriv= -(1./(1j*omega(freq))) * MeMuI * ( C.T * ( dMf_dsigi * ( dsigi_dsig * ( dsig_dm * v ) ) ) )
        else:
            h_Deriv= -(1./(1j*omega(freq))) * dsig_dm.T * ( dsigi_dsig.T * ( dMf_dsigi.T * ( C * ( MeMuI.T * v ) ) ) )

        if S_mDeriv is not None:
            return 1./(1j*omega(src.freq)) * S_mDeriv + h_Deriv

class FieldsFDEM_h(FieldsFDEM):
    knownFields = {'h_sol':'E'}
    aliasFields = {
                    'h' : ['h_sol','E','_h'],
                    'hPrimary' : ['h_sol','E','_hPrimary'],
                    'hSecondary' : ['h_sol','E','_hSecondary'],
                    'j' : ['h_sol','F','_j'],
                    'jPrimary' : ['h_sol','F','_jPrimary'],
                    'jSecondary' : ['h_sol','F','_jSecondary']
                  }

    def __init__(self,mesh,survey,**kwargs):
        FieldsFDEM.__init__(self,mesh,survey,**kwargs)

    def startup(self):
        self._edgeCurl = self.survey.prob.mesh.edgeCurl
        self._MeMuI = self.survey.prob.MeMuI
        self._MfRho = self.survey.prob.MfRho

    def _hPrimary(self, h_sol, srcList):
        hPrimary = np.zeros_like(h_sol)
        for i, src in enumerate(srcList):
            hp = src.hPrimary(self.survey.prob)
            if hp is not None:
                hPrimary[:,i] += hp
            return hPrimary

    def _hSecondary(self, h_sol, srcList):
        return h_sol

    def _h(self, h_sol, srcList):
        return self._hPrimary(h_sol, srcList) + self._hSecondary(h_sol, srcList)

    def _jPrimary(self, h_sol, srcList):
        jPrimary = np.zeros([self._edgeCurl.shape[0], h_sol.shape[1]])
        for i, src in enumerate(srcList):
            jp = src.jPrimary(self.survey.prob)
            if jp is not None:
                jPrimary[:,i] = jp
        return jPrimary

    def _jSecondary(self, h_sol, srcList):
        j = self._edgeCurl*h_sol
        for i, src in enumerate(srcList):
            _,S_e = src.eval(self.survey.prob)
            if S_e is not None:
                j[:,i] += -S_e
        return j

    def _j(self, h_sol, srcList):
        return self._jPrimary(h_sol, srcList) + self._jSecondary(h_sol, srcList)

    def _jDeriv(self, h_sol, srcList, v, adjoint=False):
        raise NotImplementedError('Fields Derivs Not Implemented Yet')
        _,S_eDeriv = src.getSourceDeriv(self.survey.prob, v, adjoint)
        if S_eDeriv is None:
            return None
        else:
            return - S_eDeriv

    # def calcFields(self, sol, freq, fieldType, adjoint=False):
    #     j = sol
    #     if fieldType == 'j':
    #         return j
    #     elif fieldType == 'h':
    #         MeMuI = self._MeMuI
    #         C = self.mesh.edgeCurl
    #         MfRho = self._MfRho
    #         if not adjoint:
    #             h = -(1./(1j*omega(freq))) * MeMuI * ( C.T * ( MfRho * j ) )
    #         else:
    #             h = -(1./(1j*omega(freq))) * MfRho.T * ( C * ( MeMuI.T * j ) )
    #         return h
    #     raise NotImplementedError('fieldType "%s" is not implemented.' % fieldType)

    # def calcFieldsDeriv(self, sol, freq, fieldType, v, adjoint=False):
    #     j = sol
    #     if fieldType == 'j':
    #         return None
    #     elif fieldType == 'h':
    #         MeMuI = self._MeMuI
    #         C = self.mesh.edgeCurl
    #         sig = self._curModel.transform
    #         sigi = 1/sig
    #         dsig_dm = self._curModel.transformDeriv
    #         dsigi_dsig = -Utils.sdiag(sigi)**2
    #         dMf_dsigi = self.mesh.getFaceInnerProductDeriv(sigi)(j)
    #         sigi = self._MfRho
    #         if not adjoint:
    #             return -(1./(1j*omega(freq))) * MeMuI * ( C.T * ( dMf_dsigi * ( dsigi_dsig * ( dsig_dm * v ) ) ) )
    #         else:
    #             return -(1./(1j*omega(freq))) * dsig_dm.T * ( dsigi_dsig.T * ( dMf_dsigi.T * ( C * ( MeMuI.T * v ) ) ) )
    #     raise NotImplementedError('fieldType "%s" is not implemented.' % fieldType)

    # def calcFields(self, sol, freq, fieldType, adjoint=False):
    #     h = sol
    #     if fieldType == 'j':
    #         C = self.mesh.edgeCurl
    #         if adjoint:
    #             return C.T*h
    #         return C*h
    #     elif fieldType == 'h':
    #         return h
    #     raise NotImplementedError('fieldType "%s" is not implemented.' % fieldType)

    # def calcFieldsDeriv(self, sol, freq, fieldType, v, adjoint=False):
    #     return None<|endoftext|>


<commit_before>"""
This test is for testing the NanGuardMode.
"""
from pylearn2.devtools.nan_guard import NanGuardMode
import numpy
import theano
import theano.tensor as T

def test_NanGuardMode():
    """
    Tests if NanGuardMode is working by feeding in numpy.inf and numpy.nans
    intentionally. A working implementation should be able to capture all
    the abnormalties.
    """
    x = T.matrix()
    w = theano.shared(numpy.random.randn(5, 7).astype(theano.config.floatX))
    y = T.dot(x, w)

    fun = theano.function(
        [x], y,
        mode=NanGuardMode(nan_is_error=True, inf_is_error=True)
    )
    a = numpy.random.randn(3, 5).astype(theano.config.floatX)
    infa = numpy.tile(
        (numpy.asarray(100.) ** 1000000).astype(theano.config.floatX), (3, 5))
    nana = numpy.tile(
        numpy.asarray(numpy.nan).astype(theano.config.floatX), (3, 5))
    biga = numpy.tile(
        numpy.asarray(1e20).astype(theano.config.floatX), (3, 5))

    work = [False, False, False]

    fun(a)  # normal values
    try:
        fun(infa)  # INFs
    except AssertionError:
        work[0] = True
    try:
        fun(nana)  # NANs
    except AssertionError:
        work[1] = True
    try:
        fun(biga)  # big values
    except AssertionError:
        work[2] = True

    if not reduce(lambda x, y: x and y, work):
        raise AssertionError("NanGuardMode not working.")
<commit_msg>remove reduce() in test code<commit_after>"""
This test is for testing the NanGuardMode.
"""
from pylearn2.devtools.nan_guard import NanGuardMode
import numpy
import theano
import theano.tensor as T

def test_NanGuardMode():
    """
    Tests if NanGuardMode is working by feeding in numpy.inf and numpy.nans
    intentionally. A working implementation should be able to capture all
    the abnormalties.
    """
    x = T.matrix()
    w = theano.shared(numpy.random.randn(5, 7).astype(theano.config.floatX))
    y = T.dot(x, w)

    fun = theano.function(
        [x], y,
        mode=NanGuardMode(nan_is_error=True, inf_is_error=True)
    )
    a = numpy.random.randn(3, 5).astype(theano.config.floatX)
    infa = numpy.tile(
        (numpy.asarray(100.) ** 1000000).astype(theano.config.floatX), (3, 5))
    nana = numpy.tile(
        numpy.asarray(numpy.nan).astype(theano.config.floatX), (3, 5))
    biga = numpy.tile(
        numpy.asarray(1e20).astype(theano.config.floatX), (3, 5))

    work = [False, False, False]

    fun(a)  # normal values
    try:
        fun(infa)  # INFs
    except AssertionError:
        work[0] = True
    try:
        fun(nana)  # NANs
    except AssertionError:
        work[1] = True
    try:
        fun(biga)  # big values
    except AssertionError:
        work[2] = True

    if not (work[0] and work[1] and work[2]):
        raise AssertionError("NanGuardMode not working.")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse

import json, time, datetime
from datetime import date

import logging
logger = logging.getLogger(__name__)

# GET ~/keyboard/ 요청에 반응
def keyboard(request):
    return JsonResponse({
        'type': 'buttons',
        'buttons': ['조식', '중식', '석식', '내일의 조식', '내일의 중식', '내일의 석식']
    })

# csrf 토큰 에러 방지, POST 요청에 message response
@csrf_exempt
def message(request):
    json_str = ((request.body).decode('utf-8'))
    received_json_data = json.loads(json_str)
    meal = received_json_data['content']

    dayString = ["월", "화", "수", "목", "금", "토", "일"]
    today = datetime.datetime.today().weekday()

    nextdayString = ["화", "수", "목", "금", "토", "일", "월"]
    tomorrow = datetime.datetime.today().weekday()

    today_date = datetime.date.today().strftime("%m월 %d일 ")
    tomorrow_date = date.fromtimestamp(time.time() + 60*60*24).strftime("%m월 %d일 ")

    if meal == '조식' or meal == '중식' or meal == '석식':
        return JsonResponse({
            'message': {
                'text': today_date + str(dayString[today]) + '요일 ' + meal + ' 메뉴입니다. \n \n' + str(crawl(request))
            },
            'keyboard': {
                'type': 'buttons',
                'buttons': ['조식', '중식', '석식', '내일의 조식', '내일의 중식', '내일의 석식']
            }
     })
    if meal == '내일의 조식' or meal == '내일의 중식' or meal == '내일의 석식':
        return JsonResponse({
            'message': {
                'text': '['+ meal + '] \n' + tomorrow_date + str(nextdayString[tomorrow]) + '요일 급식 메뉴입니다. \n \n' + str(crawl(request))
            },
            'keyboard': {
                'type': 'buttons',
                'buttons': ['내일의 조식', '내일의 중식', '내일의 석식', '조식', '중식', '석식']
            }
     })

# message 요청 받을시 크롤링 실시
def crawl(request):
    from bs4 import BeautifulSoup
    from urllib.request import urlopen

    json_str = ((request.body).decode('utf-8'))
    received_json_data = json.loads(json_str)
    meal = received_json_data['content']

    #타학교에서 이용시 수정
    regionCode = 'gne.go.kr'
    schulCode = 'S100000747'

    ScCode = 1

    if meal == '조식' or meal == '내일의 조식':
        ScCode = 1
    if meal == '중식' or meal == '내일의 중식':
        ScCode = 2
    if meal == '석식' or meal == '내일의 석식':
        ScCode = 3

    # NEIS에서 파싱
    html = urlopen('http://stu.' + regionCode + '/sts_sci_md01_001.do?schulCode='+ schulCode + '&schulCrseScCode=4&schulKndScCode=04&schMmealScCode=' + str(ScCode))
    source = html.read()
    html.close()

    # beautifulsoup4를 이용해 utf-8, lxml으로 파싱
    soup = BeautifulSoup(source, "lxml", from_encoding='utf-8')

    # div_id="contents"안의 table을 모두 검색 후 td태그만 추출
    table_div = soup.find(id="contents")
    tables = table_div.find_all("table")
    menu_table = tables[0]
    td = menu_table.find_all('td')

    # 요일 import, 월요일 ~ 일요일 = 0~6
    today = datetime.datetime.today().weekday()

    # 월요일 ~ 일요일 = td[8] ~ td[14]
    if meal == '조식' or meal == '중식' or meal == '석식':
        if today == 0:
            menu = td[8]
        elif today == 1:
            menu = td[9]
        elif today == 2:
            menu = td[10]
        elif today == 3:
            menu = td[11]
        elif today == 5:
            menu = td[12]
        elif today == 6:
            menu = td[13]

    elif meal == '내일의 조식' or meal == '내일의 중식' or meal == '내일의 석식':
        if today == 0:
            menu = td[9]
        elif today == 1:
            menu = td[10]
        elif today == 2:
            menu = td[11]
        elif today == 3:
            menu = td[12]
        elif today == 5:
            menu = td[13]
        elif today == 6:
            menu = td[14]

    # 파싱 후 불필요한 태그 잔해물 제거
    menu = str(menu).replace(' ', '').replace('*', '').replace('', '').replace('	', '').replace('1.', '').replace('2.', '').replace('3.', '').replace('4.', '').replace('5.', '').replace('6.', '').replace('7.', '').replace('8.', '').replace('9.', '').replace('10.', '').replace('11.', '').replace('12.', '').replace('13.', '').replace('14.', '').replace('15.', '').replace('1', '')

    return menu
<commit_msg>코드 정리<commit_after># -*- coding: utf-8 -*-

from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse

import json, time, datetime
from datetime import date

import logging
logger = logging.getLogger(__name__)

# GET ~/keyboard/ 요청에 반응
def keyboard(request):
    return JsonResponse({
        'type': 'buttons',
        'buttons': ['조식', '중식', '석식', '내일의 조식', '내일의 중식', '내일의 석식']
    })

# csrf 토큰 에러 방지, POST 요청에 message response
@csrf_exempt
def message(request):
    json_str = ((request.body).decode('utf-8'))
    received_json_data = json.loads(json_str)
    meal = received_json_data['content']

    daystring = ["월", "화", "수", "목", "금", "토", "일"]
    today = datetime.datetime.today().weekday()

    nextdaystring = ["화", "수", "목", "금", "토", "일", "월"]

    today_date = datetime.date.today().strftime("%m월 %d일 ")
    tomorrow_date = date.fromtimestamp(time.time() + 60 * 60 * 24).strftime("%m월 %d일 ")

    if meal == '조식' or meal == '중식' or meal == '석식':
        return JsonResponse({
            'message': {
                'text': today_date + daystring[today] + '요일 ' + meal + ' 메뉴입니다. \n \n' + crawl(request)
            },
            'keyboard': {
                'type': 'buttons',
                'buttons': ['조식', '중식', '석식', '내일의 조식', '내일의 중식', '내일의 석식']
            }
        })
    if meal == '내일의 조식' or meal == '내일의 중식' or meal == '내일의 석식':
        return JsonResponse({
            'message': {
                'text': '[' + meal + '] \n' + tomorrow_date + nextdaystring[today] + '요일 급식 메뉴입니다. \n \n' + crawl(request)
            },
            'keyboard': {
                'type': 'buttons',
                'buttons': ['내일의 조식', '내일의 중식', '내일의 석식', '조식', '중식', '석식']
            }
        })

# message 요청 받을시 크롤링 실시
def crawl(request):
    from bs4 import BeautifulSoup
    from urllib.request import urlopen

    json_str = ((request.body).decode('utf-8'))
    received_json_data = json.loads(json_str)
    meal = received_json_data['content']

    # 타학교에서 이용시 수정
    regioncode = 'gne.go.kr'
    schulcode = 'S100000747'

    if meal == '조식' or meal == '내일의 조식':
        sccode = 1
    if meal == '중식' or meal == '내일의 중식':
        sccode = 2
    if meal == '석식' or meal == '내일의 석식':
        sccode = 3

    # NEIS에서 파싱
    html = urlopen('http://stu.' + regioncode + '/sts_sci_md01_001.do?schulCode=' + schulcode + '&schulCrseScCode=4&schulKndScCode=04&schMmealScCode=' + str(sccode))
    source = html.read()
    html.close()

    # beautifulsoup4를 이용해 utf-8, lxml으로 파싱
    soup = BeautifulSoup(source, "lxml", from_encoding='utf-8')

    # div_id="contents"안의 table을 모두 검색 후 td태그만 추출
    table_div = soup.find(id="contents")
    tables = table_div.find_all("table")
    menu_table = tables[0]
    td = menu_table.find_all('td')

    # 요일 import, 월요일 ~ 일요일 = 0~6
    today = datetime.datetime.today().weekday()

    # 월요일 ~ 일요일 = td[8] ~ td[14]
    if meal == '조식' or meal == '중식' or meal == '석식':
        if today == 0:
            menu = td[8]
        elif today == 1:
            menu = td[9]
        elif today == 2:
            menu = td[10]
        elif today == 3:
            menu = td[11]
        elif today == 5:
            menu = td[12]
        elif today == 6:
            menu = td[13]
    if meal == '내일의 조식' or meal == '내일의 중식' or meal == '내일의 석식':
        if today == 0:
            menu = td[9]
        elif today == 1:
            menu = td[10]
        elif today == 2:
            menu = td[11]
        elif today == 3:
            menu = td[12]
        elif today == 5:
            menu = td[13]
        elif today == 6:
            menu = td[8]

    # 파싱 후 불필요한 태그 잔해물 제거
    menu = str(menu).replace(' ', '').replace('*', '').replace('', '').replace('	', '').replace('1.', '').replace('2.', '').replace('3.', '').replace('4.','').replace('5.', '').replace('6.', '').replace('7.', '').replace('8.', '').replace('9.', '').replace('10.', '').replace('11.', '').replace('12.', '').replace('13.', '').replace('14.', '').replace('15.', '').replace('1', '')

    return menu<|endoftext|>


<commit_before>import numpy as np
from astropy import units as u
from astropy import constants

kb_cgs = constants.k_B.cgs
h_cgs = constants.h.cgs
eightpicubed = 8 * np.pi**3
threehc = 3 * constants.h.cgs * constants.c.cgs

def line_tau(tex, total_column, partition_function, degeneracy, frequency,
             energy_upper, einstein_A):
    # don't use dipole moment, because there are extra hidden dependencies

    N_upper = (total_column * degeneracy / partition_function *
               np.exp(-energy_upper / (kb_cgs * tex)))

    # equation 29 in Mangum 2015
    #tauperdnu = (eightpicubed * frequency * dipole_moment**2 / threehc *
    #             (np.exp(frequency*h_cgs/(kb_cgs*tex))-1) * N_upper)
    tauperdnu = ((constants.c**2/(8*np.pi*frequency**2) * einstein_A * N_upper)*
                 (np.exp(frequency*h_cgs/(kb_cgs*tex))-1))

    return tauperdnu.decompose()

def Jnu(nu, T):
    """RJ equivalent temperature (eqn 24)"""
    return constants.h*nu/constants.k_B / (np.exp(constants.h*nu/(constants.k_B*T))-1)

def line_brightness(tex, dnu, frequency, tbg=2.73*u.K, *args, **kwargs):
    tau = line_tau(tex=tex, frequency=frequency, *args, **kwargs) / dnu
    tau = tau.decompose()
    assert tau.unit == u.dimensionless_unscaled
    return (Jnu(frequency, tex)-Jnu(frequency, tbg)).decompose() * (1 - np.exp(-tau))

# url = 'http://cdms.ph1.uni-koeln.de/cdms/tap/'
# rslt = requests.post(url+"/sync", data={'REQUEST':"doQuery", 'LANG': 'VSS2', 'FORMAT':'XSAMS', 'QUERY':"SELECT SPECIES WHERE MoleculeStoichiometricFormula='CH2O'"})

if __name__ == "__main__":
    # example

    # 303
    J = 3
    gI = 0.25
    gJ = 2*J+1
    gK = 1

    # 321 has same parameters for g

    ph2co = {'tex':18.75*u.K,
             'total_column': 1e12*u.cm**-2,
             'partition_function': 44.6812, # splatalogue's 18.75
             'degeneracy': gI*gJ*gK,
             #'dipole_moment': 2.331e-18*u.esu*u.cm, #2.331*u.debye,
            }

    ph2co_303 = {
             'frequency': 218.22219*u.GHz,
             'energy_upper': kb_cgs*20.95582*u.K,
             'einstein_A': 10**-3.55007/u.s,
    }
    ph2co_303.update(ph2co)
    ph2co_303['dnu'] = (1*u.km/u.s/constants.c * ph2co_303['frequency'])

    ph2co_321 = {
             'frequency': 218.76007*u.GHz,
             'energy_upper': kb_cgs*68.11081*u.K,
             'einstein_A': 10**-3.80235/u.s,
    }
    ph2co_321.update(ph2co)
    ph2co_321['dnu'] = (1*u.km/u.s/constants.c * ph2co_321['frequency'])

    ph2co_322 = {
             'frequency': 218.47563*u.GHz,
             'energy_upper': kb_cgs*68.0937*u.K,
             'einstein_A': 10**-3.80373/u.s,
    }
    ph2co_322.update(ph2co)
    ph2co_322['dnu'] = (1*u.km/u.s/constants.c * ph2co_322['frequency'])

    print("tau303 = {0}".format(line_tau(**ph2co_303)))
    print("tau321 = {0}".format(line_tau(**ph2co_321)))
    print("tau322 = {0}".format(line_tau(**ph2co_322)))
    print("r303/r321 = {0}".format(line_brightness(**ph2co_321)/line_brightness(**ph2co_303)))
    print("r303/r322 = {0}".format(line_brightness(**ph2co_322)/line_brightness(**ph2co_303)))

    # CDMS Q
    import requests
    import bs4
    url = 'http://cdms.ph1.uni-koeln.de/cdms/tap/'
    rslt = requests.post(url+"/sync", data={'REQUEST':"doQuery", 'LANG': 'VSS2', 'FORMAT':'XSAMS', 'QUERY':"SELECT SPECIES WHERE MoleculeStoichiometricFormula='CH2O'"})
    bb = bs4.BeautifulSoup(rslt.content, 'html5lib')
    h = [x for x in bb.findAll('molecule') if x.ordinarystructuralformula.value.text=='H2CO'][0]
    tem_, Q_ = h.partitionfunction.findAll('datalist')
    tem = [float(x) for x in tem_.text.split()]
    Q = [float(x) for x in Q_.text.split()]

    del ph2co_303['tex']
    del ph2co_303['partition_function']
    T_303 = np.array([line_brightness(tex=tex*u.K, partition_function=pf,
                                      **ph2co_303).value for tex,pf in
                      zip(tem,Q)])

    del ph2co_321['tex']
    del ph2co_321['partition_function']
    T_321 = np.array([line_brightness(tex=tex*u.K, partition_function=pf,
                                      **ph2co_321).value for tex,pf in
                      zip(tem,Q)])

    del ph2co_322['tex']
    del ph2co_322['partition_function']
    T_322 = np.array([line_brightness(tex=tex*u.K, partition_function=pf,
                                      **ph2co_322).value for tex,pf in
                      zip(tem,Q)])

    import pylab as pl

    pl.clf()
    pl.subplot(2,1,1)
    pl.plot(tem, T_321, label='$3_{2,1}-2_{2,0}$')
    pl.plot(tem, T_322, label='$3_{2,2}-2_{2,1}$')
    pl.plot(tem, T_303, label='$3_{0,3}-2_{0,2}$')
    pl.xlim(0,200)
    pl.subplot(2,1,2)
    pl.plot(tem, T_321/T_303, label='321/303')
    pl.plot(tem, T_322/T_303, label='322/303')
    pl.xlim(0,200)

    pl.draw(); pl.show()
<commit_msg>improve efficiency and robustness of LTE model<commit_after>import numpy as np
from astropy import units as u
from astropy import constants

kb_cgs = constants.k_B.cgs.value
h_cgs = constants.h.cgs.value
eightpicubed = 8 * np.pi**3
threehc = 3 * constants.h.cgs * constants.c.cgs
hoverk_cgs = (h_cgs/kb_cgs)
c_cgs = constants.c.cgs.value

def line_tau(tex, total_column, partition_function, degeneracy, frequency,
             energy_upper, einstein_A):
    # don't use dipole moment, because there are extra hidden dependencies

    assert einstein_A.unit.is_equivalent(u.Hz)
    assert frequency.unit.is_equivalent(u.Hz)
    assert energy_upper.unit.is_equivalent(u.erg)
    assert total_column.unit.is_equivalent(u.cm**-2)
    assert tex.unit.is_equivalent(u.K)

    N_upper = (total_column * degeneracy / partition_function *
               np.exp(-energy_upper / (constants.k_B * tex)))

    # equation 29 in Mangum 2015
    #taudnu = (eightpicubed * frequency * dipole_moment**2 / threehc *
    #             (np.exp(frequency*h_cgs/(kb_cgs*tex))-1) * N_upper)
    taudnu = ((constants.c**2/(8*np.pi*frequency**2) * einstein_A * N_upper)*
              (np.exp(frequency*constants.h/(constants.k_B*tex))-1))

    return taudnu.decompose()

def line_tau_cgs(tex, total_column, partition_function, degeneracy, frequency,
                 energy_upper, einstein_A):

    N_upper = (total_column * degeneracy / partition_function *
               np.exp(-energy_upper / (kb_cgs * tex)))

    # equation 29 in Mangum 2015
    #taudnu = (eightpicubed * frequency * dipole_moment**2 / threehc *
    #             (np.exp(frequency*h_cgs/(kb_cgs*tex))-1) * N_upper)
    taudnu = ((c_cgs**2/(8*np.pi*frequency**2) * einstein_A * N_upper)*
              (np.exp(frequency*h_cgs/(kb_cgs*tex))-1))

    return taudnu

def Jnu(nu, T):
    """RJ equivalent temperature (eqn 24)"""
    return constants.h*nu/constants.k_B / (np.exp(constants.h*nu/(constants.k_B*T))-1)

def Jnu_cgs(nu, T):
    """RJ equivalent temperature (eqn 24)
    (use cgs constants for speed)
    """
    return hoverk_cgs*nu / (np.exp(hoverk_cgs*nu/T)-1)

def line_brightness(tex, dnu, frequency, tbg=2.73*u.K, *args, **kwargs):
    tau = line_tau(tex=tex, frequency=frequency, *args, **kwargs) / dnu
    tau = tau.decompose()
    assert tau.unit == u.dimensionless_unscaled
    return (Jnu(frequency, tex)-Jnu(frequency, tbg)).decompose() * (1 - np.exp(-tau))

def line_brightness_cgs(tex, dnu, frequency, tbg=2.73, *args, **kwargs):
    tau = line_tau(tex=tex, frequency=frequency, *args, **kwargs) / dnu
    return (Jnu(frequency, tex)-Jnu(frequency, tbg)) * (1 - np.exp(-tau))

# url = 'http://cdms.ph1.uni-koeln.de/cdms/tap/'
# rslt = requests.post(url+"/sync", data={'REQUEST':"doQuery", 'LANG': 'VSS2', 'FORMAT':'XSAMS', 'QUERY':"SELECT SPECIES WHERE MoleculeStoichiometricFormula='CH2O'"})

if __name__ == "__main__":
    # example

    # 303
    J = 3
    gI = 0.25
    gJ = 2*J+1
    gK = 1

    # 321 has same parameters for g

    ph2co = {'tex':18.75*u.K,
             'total_column': 1e12*u.cm**-2,
             'partition_function': 44.6812, # splatalogue's 18.75
             'degeneracy': gI*gJ*gK,
             #'dipole_moment': 2.331e-18*u.esu*u.cm, #2.331*u.debye,
            }

    ph2co_303 = {
             'frequency': 218.22219*u.GHz,
             'energy_upper': kb_cgs*20.95582*u.K,
             'einstein_A': 10**-3.55007/u.s,
    }
    ph2co_303.update(ph2co)
    ph2co_303['dnu'] = (1*u.km/u.s/constants.c * ph2co_303['frequency'])

    ph2co_321 = {
             'frequency': 218.76007*u.GHz,
             'energy_upper': kb_cgs*68.11081*u.K,
             'einstein_A': 10**-3.80235/u.s,
    }
    ph2co_321.update(ph2co)
    ph2co_321['dnu'] = (1*u.km/u.s/constants.c * ph2co_321['frequency'])

    ph2co_322 = {
             'frequency': 218.47563*u.GHz,
             'energy_upper': kb_cgs*68.0937*u.K,
             'einstein_A': 10**-3.80373/u.s,
    }
    ph2co_322.update(ph2co)
    ph2co_322['dnu'] = (1*u.km/u.s/constants.c * ph2co_322['frequency'])

    print("tau303 = {0}".format(line_tau(**ph2co_303)))
    print("tau321 = {0}".format(line_tau(**ph2co_321)))
    print("tau322 = {0}".format(line_tau(**ph2co_322)))
    print("r303/r321 = {0}".format(line_brightness(**ph2co_321)/line_brightness(**ph2co_303)))
    print("r303/r322 = {0}".format(line_brightness(**ph2co_322)/line_brightness(**ph2co_303)))

    # CDMS Q
    import requests
    import bs4
    url = 'http://cdms.ph1.uni-koeln.de/cdms/tap/'
    rslt = requests.post(url+"/sync", data={'REQUEST':"doQuery", 'LANG': 'VSS2', 'FORMAT':'XSAMS', 'QUERY':"SELECT SPECIES WHERE MoleculeStoichiometricFormula='CH2O'"})
    bb = bs4.BeautifulSoup(rslt.content, 'html5lib')
    h = [x for x in bb.findAll('molecule') if x.ordinarystructuralformula.value.text=='H2CO'][0]
    tem_, Q_ = h.partitionfunction.findAll('datalist')
    tem = [float(x) for x in tem_.text.split()]
    Q = [float(x) for x in Q_.text.split()]

    del ph2co_303['tex']
    del ph2co_303['partition_function']
    T_303 = np.array([line_brightness(tex=tex*u.K, partition_function=pf,
                                      **ph2co_303).value for tex,pf in
                      zip(tem,Q)])

    del ph2co_321['tex']
    del ph2co_321['partition_function']
    T_321 = np.array([line_brightness(tex=tex*u.K, partition_function=pf,
                                      **ph2co_321).value for tex,pf in
                      zip(tem,Q)])

    del ph2co_322['tex']
    del ph2co_322['partition_function']
    T_322 = np.array([line_brightness(tex=tex*u.K, partition_function=pf,
                                      **ph2co_322).value for tex,pf in
                      zip(tem,Q)])

    import pylab as pl

    pl.clf()
    pl.subplot(2,1,1)
    pl.plot(tem, T_321, label='$3_{2,1}-2_{2,0}$')
    pl.plot(tem, T_322, label='$3_{2,2}-2_{2,1}$')
    pl.plot(tem, T_303, label='$3_{0,3}-2_{0,2}$')
    pl.xlim(0,200)
    pl.subplot(2,1,2)
    pl.plot(tem, T_321/T_303, label='321/303')
    pl.plot(tem, T_322/T_303, label='322/303')
    pl.xlim(0,200)

    pl.draw(); pl.show()
<|endoftext|>